Total coverage: 130558 (7%)of 1972507 











































































































 8102 











  818 





























 8585 









 1181 










  904 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _LINUX_FILE_REF_H
#define _LINUX_FILE_REF_H

#include <linux/atomic.h>
#include <linux/preempt.h>
#include <linux/types.h>

/*
 * file_ref is a reference count implementation specifically for use by
 * files. It takes inspiration from rcuref but differs in key aspects
 * such as support for SLAB_TYPESAFE_BY_RCU type caches.
 *
 * FILE_REF_ONEREF                FILE_REF_MAXREF
 * 0x0000000000000000UL      0x7FFFFFFFFFFFFFFFUL
 * <-------------------valid ------------------->
 *
 *                       FILE_REF_SATURATED
 * 0x8000000000000000UL 0xA000000000000000UL 0xBFFFFFFFFFFFFFFFUL
 * <-----------------------saturation zone---------------------->
 *
 * FILE_REF_RELEASED                   FILE_REF_DEAD
 * 0xC000000000000000UL         0xE000000000000000UL
 * <-------------------dead zone------------------->
 *
 * FILE_REF_NOREF
 * 0xFFFFFFFFFFFFFFFFUL
 */

#ifdef CONFIG_64BIT
#define FILE_REF_ONEREF                0x0000000000000000UL
#define FILE_REF_MAXREF                0x7FFFFFFFFFFFFFFFUL
#define FILE_REF_SATURATED        0xA000000000000000UL
#define FILE_REF_RELEASED        0xC000000000000000UL
#define FILE_REF_DEAD                0xE000000000000000UL
#define FILE_REF_NOREF                0xFFFFFFFFFFFFFFFFUL
#else
#define FILE_REF_ONEREF                0x00000000U
#define FILE_REF_MAXREF                0x7FFFFFFFU
#define FILE_REF_SATURATED        0xA0000000U
#define FILE_REF_RELEASED        0xC0000000U
#define FILE_REF_DEAD                0xE0000000U
#define FILE_REF_NOREF                0xFFFFFFFFU
#endif

typedef struct {
#ifdef CONFIG_64BIT
        atomic64_t refcnt;
#else
        atomic_t refcnt;
#endif
} file_ref_t;

/**
 * file_ref_init - Initialize a file reference count
 * @ref: Pointer to the reference count
 * @cnt: The initial reference count typically '1'
 */
static inline void file_ref_init(file_ref_t *ref, unsigned long cnt)
{
        atomic_long_set(&ref->refcnt, cnt - 1);
}

bool __file_ref_put(file_ref_t *ref, unsigned long cnt);

/**
 * file_ref_get - Acquire one reference on a file
 * @ref: Pointer to the reference count
 *
 * Similar to atomic_inc_not_zero() but saturates at FILE_REF_MAXREF.
 *
 * Provides full memory ordering.
 *
 * Return: False if the attempt to acquire a reference failed. This happens
 *         when the last reference has been put already. True if a reference
 *         was successfully acquired
 */
static __always_inline __must_check bool file_ref_get(file_ref_t *ref)
{
        /*
         * Unconditionally increase the reference count with full
         * ordering. The saturation and dead zones provide enough
         * tolerance for this.
         *
         * If this indicates negative the file in question the fail can
         * be freed and immediately reused due to SLAB_TYPSAFE_BY_RCU.
         * Hence, unconditionally altering the file reference count to
         * e.g., reset the file reference count back to the middle of
         * the deadzone risk end up marking someone else's file as dead
         * behind their back.
         *
         * It would be possible to do a careful:
         *
         * cnt = atomic_long_inc_return();
         * if (likely(cnt >= 0))
         *        return true;
         *
         * and then something like:
         *
         * if (cnt >= FILE_REF_RELEASE)
         *        atomic_long_try_cmpxchg(&ref->refcnt, &cnt, FILE_REF_DEAD),
         *
         * to set the value back to the middle of the deadzone. But it's
         * practically impossible to go from FILE_REF_DEAD to
         * FILE_REF_ONEREF. It would need 2305843009213693952/2^61
         * file_ref_get()s to resurrect such a dead file.
         */
        return !atomic_long_add_negative(1, &ref->refcnt);
}

/**
 * file_ref_inc - Acquire one reference on a file
 * @ref: Pointer to the reference count
 *
 * Acquire an additional reference on a file. Warns if the caller didn't
 * already hold a reference.
 */
static __always_inline void file_ref_inc(file_ref_t *ref)
{
        long prior = atomic_long_fetch_inc_relaxed(&ref->refcnt);
        WARN_ONCE(prior < 0, "file_ref_inc() on a released file reference");
}

/**
 * file_ref_put -- Release a file reference
 * @ref:        Pointer to the reference count
 *
 * Provides release memory ordering, such that prior loads and stores
 * are done before, and provides an acquire ordering on success such
 * that free() must come after.
 *
 * Return: True if this was the last reference with no future references
 *         possible. This signals the caller that it can safely release
 *         the object which is protected by the reference counter.
 *         False if there are still active references or the put() raced
 *         with a concurrent get()/put() pair. Caller is not allowed to
 *         release the protected object.
 */
static __always_inline __must_check bool file_ref_put(file_ref_t *ref)
{
        long cnt;

        /*
         * While files are SLAB_TYPESAFE_BY_RCU and thus file_ref_put()
         * calls don't risk UAFs when a file is recyclyed, it is still
         * vulnerable to UAFs caused by freeing the whole slab page once
         * it becomes unused. Prevent file_ref_put() from being
         * preempted protects against this.
         */
        guard(preempt)();
        /*
         * Unconditionally decrease the reference count. The saturation
         * and dead zones provide enough tolerance for this. If this
         * fails then we need to handle the last reference drop and
         * cases inside the saturation and dead zones.
         */
        cnt = atomic_long_dec_return(&ref->refcnt);
        if (cnt >= 0)
                return false;
        return __file_ref_put(ref, cnt);
}

/**
 * file_ref_read - Read the number of file references
 * @ref: Pointer to the reference count
 *
 * Return: The number of held references (0 ... N)
 */
static inline unsigned long file_ref_read(file_ref_t *ref)
{
        unsigned long c = atomic_long_read(&ref->refcnt);

        /* Return 0 if within the DEAD zone. */
        return c >= FILE_REF_RELEASED ? 0 : c + 1;
}

#endif





























    3 







  140 
    1 





  141 


  140 























   42 


   85 

   44 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
#ifndef _LINUX_RSEQ_H
#define _LINUX_RSEQ_H

#ifdef CONFIG_RSEQ

#include <linux/preempt.h>
#include <linux/sched.h>

/*
 * Map the event mask on the user-space ABI enum rseq_cs_flags
 * for direct mask checks.
 */
enum rseq_event_mask_bits {
        RSEQ_EVENT_PREEMPT_BIT        = RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
        RSEQ_EVENT_SIGNAL_BIT        = RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
        RSEQ_EVENT_MIGRATE_BIT        = RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
};

enum rseq_event_mask {
        RSEQ_EVENT_PREEMPT        = (1U << RSEQ_EVENT_PREEMPT_BIT),
        RSEQ_EVENT_SIGNAL        = (1U << RSEQ_EVENT_SIGNAL_BIT),
        RSEQ_EVENT_MIGRATE        = (1U << RSEQ_EVENT_MIGRATE_BIT),
};

static inline void rseq_set_notify_resume(struct task_struct *t)
{
        if (t->rseq)
                set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
}

void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);

static inline void rseq_handle_notify_resume(struct ksignal *ksig,
                                             struct pt_regs *regs)
{
        if (current->rseq)
                __rseq_handle_notify_resume(ksig, regs);
}

static inline void rseq_signal_deliver(struct ksignal *ksig,
                                       struct pt_regs *regs)
{
        preempt_disable();
        __set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
        preempt_enable();
        rseq_handle_notify_resume(ksig, regs);
}

/* rseq_preempt() requires preemption to be disabled. */
static inline void rseq_preempt(struct task_struct *t)
{
        __set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
        rseq_set_notify_resume(t);
}

/* rseq_migrate() requires preemption to be disabled. */
static inline void rseq_migrate(struct task_struct *t)
{
        __set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
        rseq_set_notify_resume(t);
}

/*
 * If parent process has a registered restartable sequences area, the
 * child inherits. Unregister rseq for a clone with CLONE_VM set.
 */
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
{
        if (clone_flags & CLONE_VM) {
                t->rseq = NULL;
                t->rseq_len = 0;
                t->rseq_sig = 0;
                t->rseq_event_mask = 0;
        } else {
                t->rseq = current->rseq;
                t->rseq_len = current->rseq_len;
                t->rseq_sig = current->rseq_sig;
                t->rseq_event_mask = current->rseq_event_mask;
        }
}

static inline void rseq_execve(struct task_struct *t)
{
        t->rseq = NULL;
        t->rseq_len = 0;
        t->rseq_sig = 0;
        t->rseq_event_mask = 0;
}

#else

static inline void rseq_set_notify_resume(struct task_struct *t)
{
}
static inline void rseq_handle_notify_resume(struct ksignal *ksig,
                                             struct pt_regs *regs)
{
}
static inline void rseq_signal_deliver(struct ksignal *ksig,
                                       struct pt_regs *regs)
{
}
static inline void rseq_preempt(struct task_struct *t)
{
}
static inline void rseq_migrate(struct task_struct *t)
{
}
static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
{
}
static inline void rseq_execve(struct task_struct *t)
{
}

#endif

#ifdef CONFIG_DEBUG_RSEQ

void rseq_syscall(struct pt_regs *regs);

#else

static inline void rseq_syscall(struct pt_regs *regs)
{
}

#endif

#endif /* _LINUX_RSEQ_H */
























































 6642 































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor network mediation definitions.
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2017 Canonical Ltd.
 */

#ifndef __AA_NET_H
#define __AA_NET_H

#include <net/sock.h>
#include <linux/path.h>

#include "apparmorfs.h"
#include "label.h"
#include "perms.h"
#include "policy.h"

#define AA_MAY_SEND                AA_MAY_WRITE
#define AA_MAY_RECEIVE                AA_MAY_READ

#define AA_MAY_SHUTDOWN                AA_MAY_DELETE

#define AA_MAY_CONNECT                AA_MAY_OPEN
#define AA_MAY_ACCEPT                0x00100000

#define AA_MAY_BIND                0x00200000
#define AA_MAY_LISTEN                0x00400000

#define AA_MAY_SETOPT                0x01000000
#define AA_MAY_GETOPT                0x02000000

#define NET_PERMS_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |    \
                        AA_MAY_SHUTDOWN | AA_MAY_BIND | AA_MAY_LISTEN |          \
                        AA_MAY_CONNECT | AA_MAY_ACCEPT | AA_MAY_SETATTR | \
                        AA_MAY_GETATTR | AA_MAY_SETOPT | AA_MAY_GETOPT)

#define NET_FS_PERMS (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CREATE |        \
                      AA_MAY_SHUTDOWN | AA_MAY_CONNECT | AA_MAY_RENAME |\
                      AA_MAY_SETATTR | AA_MAY_GETATTR | AA_MAY_CHMOD |        \
                      AA_MAY_CHOWN | AA_MAY_CHGRP | AA_MAY_LOCK |        \
                      AA_MAY_MPROT)

#define NET_PEER_MASK (AA_MAY_SEND | AA_MAY_RECEIVE | AA_MAY_CONNECT |        \
                       AA_MAY_ACCEPT)
struct aa_sk_ctx {
        struct aa_label *label;
        struct aa_label *peer;
};

static inline struct aa_sk_ctx *aa_sock(const struct sock *sk)
{
        return sk->sk_security + apparmor_blob_sizes.lbs_sock;
}

#define DEFINE_AUDIT_NET(NAME, OP, SK, F, T, P)                                  \
        struct lsm_network_audit NAME ## _net = { .sk = (SK),                  \
                                                  .family = (F)};          \
        DEFINE_AUDIT_DATA(NAME,                                                  \
                          ((SK) && (F) != AF_UNIX) ? LSM_AUDIT_DATA_NET : \
                                                     LSM_AUDIT_DATA_NONE, \
                                                     AA_CLASS_NET,        \
                          OP);                                                  \
        NAME.common.u.net = &(NAME ## _net);                                  \
        NAME.net.type = (T);                                                  \
        NAME.net.protocol = (P)

#define DEFINE_AUDIT_SK(NAME, OP, SK)                                        \
        DEFINE_AUDIT_NET(NAME, OP, SK, (SK)->sk_family, (SK)->sk_type,        \
                         (SK)->sk_protocol)


#define af_select(FAMILY, FN, DEF_FN)                \
({                                                \
        int __e;                                \
        switch ((FAMILY)) {                        \
        default:                                \
                __e = DEF_FN;                        \
        }                                        \
        __e;                                        \
})

struct aa_secmark {
        u8 audit;
        u8 deny;
        u32 secid;
        char *label;
};

extern struct aa_sfs_entry aa_sfs_entry_network[];

void audit_net_cb(struct audit_buffer *ab, void *va);
int aa_profile_af_perm(struct aa_profile *profile,
                       struct apparmor_audit_data *ad,
                       u32 request, u16 family, int type);
int aa_af_perm(const struct cred *subj_cred, struct aa_label *label,
               const char *op, u32 request, u16 family,
               int type, int protocol);
static inline int aa_profile_af_sk_perm(struct aa_profile *profile,
                                        struct apparmor_audit_data *ad,
                                        u32 request,
                                        struct sock *sk)
{
        return aa_profile_af_perm(profile, ad, request, sk->sk_family,
                                  sk->sk_type);
}
int aa_sk_perm(const char *op, u32 request, struct sock *sk);

int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
                      const char *op, u32 request,
                      struct socket *sock);

int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
                           u32 secid, const struct sock *sk);

#endif /* __AA_NET_H */















































































































































    3 







   12 







   41 
   44 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Spanning tree protocol; timer-related code
 *        Linux ethernet bridge
 *
 *        Authors:
 *        Lennert Buytenhek                <buytenh@gnu.org>
 */

#include <linux/kernel.h>
#include <linux/times.h>

#include "br_private.h"
#include "br_private_stp.h"

/* called under bridge lock */
static int br_is_designated_for_some_port(const struct net_bridge *br)
{
        struct net_bridge_port *p;

        list_for_each_entry(p, &br->port_list, list) {
                if (p->state != BR_STATE_DISABLED &&
                    !memcmp(&p->designated_bridge, &br->bridge_id, 8))
                        return 1;
        }

        return 0;
}

static void br_hello_timer_expired(struct timer_list *t)
{
        struct net_bridge *br = from_timer(br, t, hello_timer);

        br_debug(br, "hello timer expired\n");
        spin_lock(&br->lock);
        if (br->dev->flags & IFF_UP) {
                br_config_bpdu_generation(br);

                if (br->stp_enabled == BR_KERNEL_STP)
                        mod_timer(&br->hello_timer,
                                  round_jiffies(jiffies + br->hello_time));
        }
        spin_unlock(&br->lock);
}

static void br_message_age_timer_expired(struct timer_list *t)
{
        struct net_bridge_port *p = from_timer(p, t, message_age_timer);
        struct net_bridge *br = p->br;
        const bridge_id *id = &p->designated_bridge;
        int was_root;

        if (p->state == BR_STATE_DISABLED)
                return;

        br_info(br, "port %u(%s) neighbor %.2x%.2x.%pM lost\n",
                (unsigned int) p->port_no, p->dev->name,
                id->prio[0], id->prio[1], &id->addr);

        /*
         * According to the spec, the message age timer cannot be
         * running when we are the root bridge. So..  this was_root
         * check is redundant. I'm leaving it in for now, though.
         */
        spin_lock(&br->lock);
        if (p->state == BR_STATE_DISABLED)
                goto unlock;
        was_root = br_is_root_bridge(br);

        br_become_designated_port(p);
        br_configuration_update(br);
        br_port_state_selection(br);
        if (br_is_root_bridge(br) && !was_root)
                br_become_root_bridge(br);
 unlock:
        spin_unlock(&br->lock);
}

static void br_forward_delay_timer_expired(struct timer_list *t)
{
        struct net_bridge_port *p = from_timer(p, t, forward_delay_timer);
        struct net_bridge *br = p->br;

        br_debug(br, "port %u(%s) forward delay timer\n",
                 (unsigned int) p->port_no, p->dev->name);
        spin_lock(&br->lock);
        if (p->state == BR_STATE_LISTENING) {
                br_set_state(p, BR_STATE_LEARNING);
                mod_timer(&p->forward_delay_timer,
                          jiffies + br->forward_delay);
        } else if (p->state == BR_STATE_LEARNING) {
                br_set_state(p, BR_STATE_FORWARDING);
                if (br_is_designated_for_some_port(br))
                        br_topology_change_detection(br);
                netif_carrier_on(br->dev);
        }
        rcu_read_lock();
        br_ifinfo_notify(RTM_NEWLINK, NULL, p);
        rcu_read_unlock();
        spin_unlock(&br->lock);
}

static void br_tcn_timer_expired(struct timer_list *t)
{
        struct net_bridge *br = from_timer(br, t, tcn_timer);

        br_debug(br, "tcn timer expired\n");
        spin_lock(&br->lock);
        if (!br_is_root_bridge(br) && (br->dev->flags & IFF_UP)) {
                br_transmit_tcn(br);

                mod_timer(&br->tcn_timer, jiffies + br->bridge_hello_time);
        }
        spin_unlock(&br->lock);
}

static void br_topology_change_timer_expired(struct timer_list *t)
{
        struct net_bridge *br = from_timer(br, t, topology_change_timer);

        br_debug(br, "topo change timer expired\n");
        spin_lock(&br->lock);
        br->topology_change_detected = 0;
        __br_set_topology_change(br, 0);
        spin_unlock(&br->lock);
}

static void br_hold_timer_expired(struct timer_list *t)
{
        struct net_bridge_port *p = from_timer(p, t, hold_timer);

        br_debug(p->br, "port %u(%s) hold timer expired\n",
                 (unsigned int) p->port_no, p->dev->name);

        spin_lock(&p->br->lock);
        if (p->config_pending)
                br_transmit_config(p);
        spin_unlock(&p->br->lock);
}

void br_stp_timer_init(struct net_bridge *br)
{
        timer_setup(&br->hello_timer, br_hello_timer_expired, 0);
        timer_setup(&br->tcn_timer, br_tcn_timer_expired, 0);
        timer_setup(&br->topology_change_timer,
                    br_topology_change_timer_expired, 0);
}

void br_stp_port_timer_init(struct net_bridge_port *p)
{
        timer_setup(&p->message_age_timer, br_message_age_timer_expired, 0);
        timer_setup(&p->forward_delay_timer, br_forward_delay_timer_expired, 0);
        timer_setup(&p->hold_timer, br_hold_timer_expired, 0);
}

/* Report ticks left (in USER_HZ) used for API */
unsigned long br_timer_value(const struct timer_list *timer)
{
        return timer_pending(timer)
                ? jiffies_delta_to_clock_t(timer->expires - jiffies) : 0;
}





















































































































    5 





    5 





    5 
    5 

    5 







    7 






    5 

    2 



    5 


    1 


    1 
    1 




    5 







    5 

    5 

    5 











    5 
    5 








    5 

    5 




    5 





    1 
    1 

    5 
    5 




    7 




    1 

    1 
    1 

    3 












    2 


    2 





















    2 












































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "queueing.h"
#include "socket.h"
#include "timers.h"
#include "device.h"
#include "ratelimiter.h"
#include "peer.h"
#include "messages.h"

#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/if_arp.h>
#include <linux/icmp.h>
#include <linux/suspend.h>
#include <net/dst_metadata.h>
#include <net/gso.h>
#include <net/icmp.h>
#include <net/rtnetlink.h>
#include <net/ip_tunnels.h>
#include <net/addrconf.h>

static LIST_HEAD(device_list);

static int wg_open(struct net_device *dev)
{
        struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
        struct inet6_dev *dev_v6 = __in6_dev_get(dev);
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        int ret;

        if (dev_v4) {
                /* At some point we might put this check near the ip_rt_send_
                 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
                 * to the current secpath check.
                 */
                IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
                IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
        }
        if (dev_v6)
                dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;

        mutex_lock(&wg->device_update_lock);
        ret = wg_socket_init(wg, wg->incoming_port);
        if (ret < 0)
                goto out;
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_send_staged_packets(peer);
                if (peer->persistent_keepalive_interval)
                        wg_packet_send_keepalive(peer);
        }
out:
        mutex_unlock(&wg->device_update_lock);
        return ret;
}

static int wg_pm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        /* If the machine is constantly suspending and resuming, as part of
         * its normal operation rather than as a somewhat rare event, then we
         * don't actually want to clear keys.
         */
        if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) ||
            IS_ENABLED(CONFIG_PM_USERSPACE_AUTOSLEEP))
                return 0;

        if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
                return 0;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list) {
                        del_timer(&peer->timer_zero_key_material);
                        wg_noise_handshake_clear(&peer->handshake);
                        wg_noise_keypairs_clear(&peer->keypairs);
                }
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        rcu_barrier();
        return 0;
}

static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };

static int wg_vm_notification(struct notifier_block *nb, unsigned long action, void *data)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                mutex_lock(&wg->device_update_lock);
                list_for_each_entry(peer, &wg->peer_list, peer_list)
                        wg_noise_expire_current_peer_keypairs(peer);
                mutex_unlock(&wg->device_update_lock);
        }
        rtnl_unlock();
        return 0;
}

static struct notifier_block vm_notifier = { .notifier_call = wg_vm_notification };

static int wg_stop(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct wg_peer *peer;
        struct sk_buff *skb;

        mutex_lock(&wg->device_update_lock);
        list_for_each_entry(peer, &wg->peer_list, peer_list) {
                wg_packet_purge_staged_packets(peer);
                wg_timers_stop(peer);
                wg_noise_handshake_clear(&peer->handshake);
                wg_noise_keypairs_clear(&peer->keypairs);
                wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
        }
        mutex_unlock(&wg->device_update_lock);
        while ((skb = ptr_ring_consume(&wg->handshake_queue.ring)) != NULL)
                kfree_skb(skb);
        atomic_set(&wg->handshake_queue_len, 0);
        wg_socket_reinit(wg, NULL, NULL);
        return 0;
}

static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        struct sk_buff_head packets;
        struct wg_peer *peer;
        struct sk_buff *next;
        sa_family_t family;
        u32 mtu;
        int ret;

        if (unlikely(!wg_check_packet_protocol(skb))) {
                ret = -EPROTONOSUPPORT;
                net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
                goto err;
        }

        peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
        if (unlikely(!peer)) {
                ret = -ENOKEY;
                if (skb->protocol == htons(ETH_P_IP))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
                                            dev->name, &ip_hdr(skb)->daddr);
                else if (skb->protocol == htons(ETH_P_IPV6))
                        net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
                                            dev->name, &ipv6_hdr(skb)->daddr);
                goto err_icmp;
        }

        family = READ_ONCE(peer->endpoint.addr.sa_family);
        if (unlikely(family != AF_INET && family != AF_INET6)) {
                ret = -EDESTADDRREQ;
                net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
                                    dev->name, peer->internal_id);
                goto err_peer;
        }

        mtu = skb_valid_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;

        __skb_queue_head_init(&packets);
        if (!skb_is_gso(skb)) {
                skb_mark_not_on_list(skb);
        } else {
                struct sk_buff *segs = skb_gso_segment(skb, 0);

                if (IS_ERR(segs)) {
                        ret = PTR_ERR(segs);
                        goto err_peer;
                }
                dev_kfree_skb(skb);
                skb = segs;
        }

        skb_list_walk_safe(skb, skb, next) {
                skb_mark_not_on_list(skb);

                skb = skb_share_check(skb, GFP_ATOMIC);
                if (unlikely(!skb))
                        continue;

                /* We only need to keep the original dst around for icmp,
                 * so at this point we're in a position to drop it.
                 */
                skb_dst_drop(skb);

                PACKET_CB(skb)->mtu = mtu;

                __skb_queue_tail(&packets, skb);
        }

        spin_lock_bh(&peer->staged_packet_queue.lock);
        /* If the queue is getting too big, we start removing the oldest packets
         * until it's small again. We do this before adding the new packet, so
         * we don't remove GSO segments that are in excess.
         */
        while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
                dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
                DEV_STATS_INC(dev, tx_dropped);
        }
        skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
        spin_unlock_bh(&peer->staged_packet_queue.lock);

        wg_packet_send_staged_packets(peer);

        wg_peer_put(peer);
        return NETDEV_TX_OK;

err_peer:
        wg_peer_put(peer);
err_icmp:
        if (skb->protocol == htons(ETH_P_IP))
                icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
        else if (skb->protocol == htons(ETH_P_IPV6))
                icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
err:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
        return ret;
}

static const struct net_device_ops netdev_ops = {
        .ndo_open                = wg_open,
        .ndo_stop                = wg_stop,
        .ndo_start_xmit                = wg_xmit,
};

static void wg_destruct(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);

        rtnl_lock();
        list_del(&wg->device_list);
        rtnl_unlock();
        mutex_lock(&wg->device_update_lock);
        rcu_assign_pointer(wg->creating_net, NULL);
        wg->incoming_port = 0;
        wg_socket_reinit(wg, NULL, NULL);
        /* The final references are cleared in the below calls to destroy_workqueue. */
        wg_peer_remove_all(wg);
        destroy_workqueue(wg->handshake_receive_wq);
        destroy_workqueue(wg->handshake_send_wq);
        destroy_workqueue(wg->packet_crypt_wq);
        wg_packet_queue_free(&wg->handshake_queue, true);
        wg_packet_queue_free(&wg->decrypt_queue, false);
        wg_packet_queue_free(&wg->encrypt_queue, false);
        rcu_barrier(); /* Wait for all the peers to be actually freed. */
        wg_ratelimiter_uninit();
        memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
        kvfree(wg->index_hashtable);
        kvfree(wg->peer_hashtable);
        mutex_unlock(&wg->device_update_lock);

        pr_debug("%s: Interface destroyed\n", dev->name);
        free_netdev(dev);
}

static const struct device_type device_type = { .name = KBUILD_MODNAME };

static void wg_setup(struct net_device *dev)
{
        struct wg_device *wg = netdev_priv(dev);
        enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
                                    NETIF_F_SG | NETIF_F_GSO |
                                    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
        const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
                             max(sizeof(struct ipv6hdr), sizeof(struct iphdr));

        dev->netdev_ops = &netdev_ops;
        dev->header_ops = &ip_tunnel_header_ops;
        dev->hard_header_len = 0;
        dev->addr_len = 0;
        dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
        dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
        dev->type = ARPHRD_NONE;
        dev->flags = IFF_POINTOPOINT | IFF_NOARP;
        dev->priv_flags |= IFF_NO_QUEUE;
        dev->lltx = true;
        dev->features |= WG_NETDEV_FEATURES;
        dev->hw_features |= WG_NETDEV_FEATURES;
        dev->hw_enc_features |= WG_NETDEV_FEATURES;
        dev->mtu = ETH_DATA_LEN - overhead;
        dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        SET_NETDEV_DEVTYPE(dev, &device_type);

        /* We need to keep the dst around in case of icmp replies. */
        netif_keep_dst(dev);

        netif_set_tso_max_size(dev, GSO_MAX_SIZE);

        wg->dev = dev;
}

static int wg_newlink(struct net *src_net, struct net_device *dev,
                      struct nlattr *tb[], struct nlattr *data[],
                      struct netlink_ext_ack *extack)
{
        struct wg_device *wg = netdev_priv(dev);
        int ret = -ENOMEM;

        rcu_assign_pointer(wg->creating_net, src_net);
        init_rwsem(&wg->static_identity.lock);
        mutex_init(&wg->socket_update_lock);
        mutex_init(&wg->device_update_lock);
        wg_allowedips_init(&wg->peer_allowedips);
        wg_cookie_checker_init(&wg->cookie_checker, wg);
        INIT_LIST_HEAD(&wg->peer_list);
        wg->device_update_gen = 1;

        wg->peer_hashtable = wg_pubkey_hashtable_alloc();
        if (!wg->peer_hashtable)
                return ret;

        wg->index_hashtable = wg_index_hashtable_alloc();
        if (!wg->index_hashtable)
                goto err_free_peer_hashtable;

        wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
                        WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
        if (!wg->handshake_receive_wq)
                goto err_free_index_hashtable;

        wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
                        WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
        if (!wg->handshake_send_wq)
                goto err_destroy_handshake_receive;

        wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
                        WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
        if (!wg->packet_crypt_wq)
                goto err_destroy_handshake_send;

        ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_destroy_packet_crypt;

        ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
                                   MAX_QUEUED_PACKETS);
        if (ret < 0)
                goto err_free_encrypt_queue;

        ret = wg_packet_queue_init(&wg->handshake_queue, wg_packet_handshake_receive_worker,
                                   MAX_QUEUED_INCOMING_HANDSHAKES);
        if (ret < 0)
                goto err_free_decrypt_queue;

        ret = wg_ratelimiter_init();
        if (ret < 0)
                goto err_free_handshake_queue;

        ret = register_netdevice(dev);
        if (ret < 0)
                goto err_uninit_ratelimiter;

        list_add(&wg->device_list, &device_list);

        /* We wait until the end to assign priv_destructor, so that
         * register_netdevice doesn't call it for us if it fails.
         */
        dev->priv_destructor = wg_destruct;

        pr_debug("%s: Interface created\n", dev->name);
        return ret;

err_uninit_ratelimiter:
        wg_ratelimiter_uninit();
err_free_handshake_queue:
        wg_packet_queue_free(&wg->handshake_queue, false);
err_free_decrypt_queue:
        wg_packet_queue_free(&wg->decrypt_queue, false);
err_free_encrypt_queue:
        wg_packet_queue_free(&wg->encrypt_queue, false);
err_destroy_packet_crypt:
        destroy_workqueue(wg->packet_crypt_wq);
err_destroy_handshake_send:
        destroy_workqueue(wg->handshake_send_wq);
err_destroy_handshake_receive:
        destroy_workqueue(wg->handshake_receive_wq);
err_free_index_hashtable:
        kvfree(wg->index_hashtable);
err_free_peer_hashtable:
        kvfree(wg->peer_hashtable);
        return ret;
}

static struct rtnl_link_ops link_ops __read_mostly = {
        .kind                        = KBUILD_MODNAME,
        .priv_size                = sizeof(struct wg_device),
        .setup                        = wg_setup,
        .newlink                = wg_newlink,
};

static void wg_netns_pre_exit(struct net *net)
{
        struct wg_device *wg;
        struct wg_peer *peer;

        rtnl_lock();
        list_for_each_entry(wg, &device_list, device_list) {
                if (rcu_access_pointer(wg->creating_net) == net) {
                        pr_debug("%s: Creating namespace exiting\n", wg->dev->name);
                        netif_carrier_off(wg->dev);
                        mutex_lock(&wg->device_update_lock);
                        rcu_assign_pointer(wg->creating_net, NULL);
                        wg_socket_reinit(wg, NULL, NULL);
                        list_for_each_entry(peer, &wg->peer_list, peer_list)
                                wg_socket_clear_peer_endpoint_src(peer);
                        mutex_unlock(&wg->device_update_lock);
                }
        }
        rtnl_unlock();
}

static struct pernet_operations pernet_ops = {
        .pre_exit = wg_netns_pre_exit
};

int __init wg_device_init(void)
{
        int ret;

        ret = register_pm_notifier(&pm_notifier);
        if (ret)
                return ret;

        ret = register_random_vmfork_notifier(&vm_notifier);
        if (ret)
                goto error_pm;

        ret = register_pernet_device(&pernet_ops);
        if (ret)
                goto error_vm;

        ret = rtnl_link_register(&link_ops);
        if (ret)
                goto error_pernet;

        return 0;

error_pernet:
        unregister_pernet_device(&pernet_ops);
error_vm:
        unregister_random_vmfork_notifier(&vm_notifier);
error_pm:
        unregister_pm_notifier(&pm_notifier);
        return ret;
}

void wg_device_uninit(void)
{
        rtnl_link_unregister(&link_ops);
        unregister_pernet_device(&pernet_ops);
        unregister_random_vmfork_notifier(&vm_notifier);
        unregister_pm_notifier(&pm_notifier);
        rcu_barrier();
}















































































































   98 






























  247 


  245 
















 7783 



 7777 
















  460 








 1243 


 1264 
 1202 










 1712 




 2184 






  400 





  377 


















  914 
















  914 













  886 








 9923 




































































































   66 








   33 
  106 
   52 

    9 


  132 















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_UACCESS_H__
#define __LINUX_UACCESS_H__

#include <linux/fault-inject-usercopy.h>
#include <linux/instrumented.h>
#include <linux/minmax.h>
#include <linux/nospec.h>
#include <linux/sched.h>
#include <linux/thread_info.h>

#include <asm/uaccess.h>

/*
 * Architectures that support memory tagging (assigning tags to memory regions,
 * embedding these tags into addresses that point to these memory regions, and
 * checking that the memory and the pointer tags match on memory accesses)
 * redefine this macro to strip tags from pointers.
 *
 * Passing down mm_struct allows to define untagging rules on per-process
 * basis.
 *
 * It's defined as noop for architectures that don't support memory tagging.
 */
#ifndef untagged_addr
#define untagged_addr(addr) (addr)
#endif

#ifndef untagged_addr_remote
#define untagged_addr_remote(mm, addr)        ({                \
        mmap_assert_locked(mm);                                \
        untagged_addr(addr);                                \
})
#endif

#ifdef masked_user_access_begin
 #define can_do_masked_user_access() 1
#else
 #define can_do_masked_user_access() 0
 #define masked_user_access_begin(src) NULL
 #define mask_user_address(src) (src)
#endif

/*
 * Architectures should provide two primitives (raw_copy_{to,from}_user())
 * and get rid of their private instances of copy_{to,from}_user() and
 * __copy_{to,from}_user{,_inatomic}().
 *
 * raw_copy_{to,from}_user(to, from, size) should copy up to size bytes and
 * return the amount left to copy.  They should assume that access_ok() has
 * already been checked (and succeeded); they should *not* zero-pad anything.
 * No KASAN or object size checks either - those belong here.
 *
 * Both of these functions should attempt to copy size bytes starting at from
 * into the area starting at to.  They must not fetch or store anything
 * outside of those areas.  Return value must be between 0 (everything
 * copied successfully) and size (nothing copied).
 *
 * If raw_copy_{to,from}_user(to, from, size) returns N, size - N bytes starting
 * at to must become equal to the bytes fetched from the corresponding area
 * starting at from.  All data past to + size - N must be left unmodified.
 *
 * If copying succeeds, the return value must be 0.  If some data cannot be
 * fetched, it is permitted to copy less than had been fetched; the only
 * hard requirement is that not storing anything at all (i.e. returning size)
 * should happen only when nothing could be copied.  In other words, you don't
 * have to squeeze as much as possible - it is allowed, but not necessary.
 *
 * For raw_copy_from_user() to always points to kernel memory and no faults
 * on store should happen.  Interpretation of from is affected by set_fs().
 * For raw_copy_to_user() it's the other way round.
 *
 * Both can be inlined - it's up to architectures whether it wants to bother
 * with that.  They should not be used directly; they are used to implement
 * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic())
 * that are used instead.  Out of those, __... ones are inlined.  Plain
 * copy_{to,from}_user() might or might not be inlined.  If you want them
 * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER.
 *
 * NOTE: only copy_from_user() zero-pads the destination in case of short copy.
 * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything
 * at all; their callers absolutely must check the return value.
 *
 * Biarch ones should also provide raw_copy_in_user() - similar to the above,
 * but both source and destination are __user pointers (affected by set_fs()
 * as usual) and both source and destination can trigger faults.
 */

static __always_inline __must_check unsigned long
__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
{
        unsigned long res;

        instrument_copy_from_user_before(to, from, n);
        check_object_size(to, n, false);
        res = raw_copy_from_user(to, from, n);
        instrument_copy_from_user_after(to, from, n, res);
        return res;
}

static __always_inline __must_check unsigned long
__copy_from_user(void *to, const void __user *from, unsigned long n)
{
        unsigned long res;

        might_fault();
        instrument_copy_from_user_before(to, from, n);
        if (should_fail_usercopy())
                return n;
        check_object_size(to, n, false);
        res = raw_copy_from_user(to, from, n);
        instrument_copy_from_user_after(to, from, n, res);
        return res;
}

/**
 * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
 * @to:   Destination address, in user space.
 * @from: Source address, in kernel space.
 * @n:    Number of bytes to copy.
 *
 * Context: User context only.
 *
 * Copy data from kernel space to user space.  Caller must check
 * the specified block with access_ok() before calling this function.
 * The caller should also make sure he pins the user space address
 * so that we don't result in page fault and sleep.
 */
static __always_inline __must_check unsigned long
__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
{
        if (should_fail_usercopy())
                return n;
        instrument_copy_to_user(to, from, n);
        check_object_size(from, n, true);
        return raw_copy_to_user(to, from, n);
}

static __always_inline __must_check unsigned long
__copy_to_user(void __user *to, const void *from, unsigned long n)
{
        might_fault();
        if (should_fail_usercopy())
                return n;
        instrument_copy_to_user(to, from, n);
        check_object_size(from, n, true);
        return raw_copy_to_user(to, from, n);
}

/*
 * Architectures that #define INLINE_COPY_TO_USER use this function
 * directly in the normal copy_to/from_user(), the other ones go
 * through an extern _copy_to/from_user(), which expands the same code
 * here.
 *
 * Rust code always uses the extern definition.
 */
static inline __must_check unsigned long
_inline_copy_from_user(void *to, const void __user *from, unsigned long n)
{
        unsigned long res = n;
        might_fault();
        if (should_fail_usercopy())
                goto fail;
        if (can_do_masked_user_access())
                from = mask_user_address(from);
        else {
                if (!access_ok(from, n))
                        goto fail;
                /*
                 * Ensure that bad access_ok() speculation will not
                 * lead to nasty side effects *after* the copy is
                 * finished:
                 */
                barrier_nospec();
        }
        instrument_copy_from_user_before(to, from, n);
        res = raw_copy_from_user(to, from, n);
        instrument_copy_from_user_after(to, from, n, res);
        if (likely(!res))
                return 0;
fail:
        memset(to + (n - res), 0, res);
        return res;
}
extern __must_check unsigned long
_copy_from_user(void *, const void __user *, unsigned long);

static inline __must_check unsigned long
_inline_copy_to_user(void __user *to, const void *from, unsigned long n)
{
        might_fault();
        if (should_fail_usercopy())
                return n;
        if (access_ok(to, n)) {
                instrument_copy_to_user(to, from, n);
                n = raw_copy_to_user(to, from, n);
        }
        return n;
}
extern __must_check unsigned long
_copy_to_user(void __user *, const void *, unsigned long);

static __always_inline unsigned long __must_check
copy_from_user(void *to, const void __user *from, unsigned long n)
{
        if (!check_copy_size(to, n, false))
                return n;
#ifdef INLINE_COPY_FROM_USER
        return _inline_copy_from_user(to, from, n);
#else
        return _copy_from_user(to, from, n);
#endif
}

static __always_inline unsigned long __must_check
copy_to_user(void __user *to, const void *from, unsigned long n)
{
        if (!check_copy_size(from, n, true))
                return n;

#ifdef INLINE_COPY_TO_USER
        return _inline_copy_to_user(to, from, n);
#else
        return _copy_to_user(to, from, n);
#endif
}

#ifndef copy_mc_to_kernel
/*
 * Without arch opt-in this generic copy_mc_to_kernel() will not handle
 * #MC (or arch equivalent) during source read.
 */
static inline unsigned long __must_check
copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
{
        memcpy(dst, src, cnt);
        return 0;
}
#endif

static __always_inline void pagefault_disabled_inc(void)
{
        current->pagefault_disabled++;
}

static __always_inline void pagefault_disabled_dec(void)
{
        current->pagefault_disabled--;
}

/*
 * These routines enable/disable the pagefault handler. If disabled, it will
 * not take any locks and go straight to the fixup table.
 *
 * User access methods will not sleep when called from a pagefault_disabled()
 * environment.
 */
static inline void pagefault_disable(void)
{
        pagefault_disabled_inc();
        /*
         * make sure to have issued the store before a pagefault
         * can hit.
         */
        barrier();
}

static inline void pagefault_enable(void)
{
        /*
         * make sure to issue those last loads/stores before enabling
         * the pagefault handler again.
         */
        barrier();
        pagefault_disabled_dec();
}

/*
 * Is the pagefault handler disabled? If so, user access methods will not sleep.
 */
static inline bool pagefault_disabled(void)
{
        return current->pagefault_disabled != 0;
}

/*
 * The pagefault handler is in general disabled by pagefault_disable() or
 * when in irq context (via in_atomic()).
 *
 * This function should only be used by the fault handlers. Other users should
 * stick to pagefault_disabled().
 * Please NEVER use preempt_disable() to disable the fault handler. With
 * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
 * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
 */
#define faulthandler_disabled() (pagefault_disabled() || in_atomic())

#ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS

/**
 * probe_subpage_writeable: probe the user range for write faults at sub-page
 *                            granularity (e.g. arm64 MTE)
 * @uaddr: start of address range
 * @size: size of address range
 *
 * Returns 0 on success, the number of bytes not probed on fault.
 *
 * It is expected that the caller checked for the write permission of each
 * page in the range either by put_user() or GUP. The architecture port can
 * implement a more efficient get_user() probing if the same sub-page faults
 * are triggered by either a read or a write.
 */
static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size)
{
        return 0;
}

#endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */

#ifndef ARCH_HAS_NOCACHE_UACCESS

static inline __must_check unsigned long
__copy_from_user_inatomic_nocache(void *to, const void __user *from,
                                  unsigned long n)
{
        return __copy_from_user_inatomic(to, from, n);
}

#endif                /* ARCH_HAS_NOCACHE_UACCESS */

extern __must_check int check_zeroed_user(const void __user *from, size_t size);

/**
 * copy_struct_from_user: copy a struct from userspace
 * @dst:   Destination address, in kernel space. This buffer must be @ksize
 *         bytes long.
 * @ksize: Size of @dst struct.
 * @src:   Source address, in userspace.
 * @usize: (Alleged) size of @src struct.
 *
 * Copies a struct from userspace to kernel space, in a way that guarantees
 * backwards-compatibility for struct syscall arguments (as long as future
 * struct extensions are made such that all new fields are *appended* to the
 * old struct, and zeroed-out new fields have the same meaning as the old
 * struct).
 *
 * @ksize is just sizeof(*dst), and @usize should've been passed by userspace.
 * The recommended usage is something like the following:
 *
 *   SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize)
 *   {
 *      int err;
 *      struct foo karg = {};
 *
 *      if (usize > PAGE_SIZE)
 *        return -E2BIG;
 *      if (usize < FOO_SIZE_VER0)
 *        return -EINVAL;
 *
 *      err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize);
 *      if (err)
 *        return err;
 *
 *      // ...
 *   }
 *
 * There are three cases to consider:
 *  * If @usize == @ksize, then it's copied verbatim.
 *  * If @usize < @ksize, then the userspace has passed an old struct to a
 *    newer kernel. The rest of the trailing bytes in @dst (@ksize - @usize)
 *    are to be zero-filled.
 *  * If @usize > @ksize, then the userspace has passed a new struct to an
 *    older kernel. The trailing bytes unknown to the kernel (@usize - @ksize)
 *    are checked to ensure they are zeroed, otherwise -E2BIG is returned.
 *
 * Returns (in all cases, some data may have been copied):
 *  * -E2BIG:  (@usize > @ksize) and there are non-zero trailing bytes in @src.
 *  * -EFAULT: access to userspace failed.
 */
static __always_inline __must_check int
copy_struct_from_user(void *dst, size_t ksize, const void __user *src,
                      size_t usize)
{
        size_t size = min(ksize, usize);
        size_t rest = max(ksize, usize) - size;

        /* Double check if ksize is larger than a known object size. */
        if (WARN_ON_ONCE(ksize > __builtin_object_size(dst, 1)))
                return -E2BIG;

        /* Deal with trailing bytes. */
        if (usize < ksize) {
                memset(dst + size, 0, rest);
        } else if (usize > ksize) {
                int ret = check_zeroed_user(src + size, rest);
                if (ret <= 0)
                        return ret ?: -E2BIG;
        }
        /* Copy the interoperable parts of the struct. */
        if (copy_from_user(dst, src, size))
                return -EFAULT;
        return 0;
}

/**
 * copy_struct_to_user: copy a struct to userspace
 * @dst:   Destination address, in userspace. This buffer must be @ksize
 *         bytes long.
 * @usize: (Alleged) size of @dst struct.
 * @src:   Source address, in kernel space.
 * @ksize: Size of @src struct.
 * @ignored_trailing: Set to %true if there was a non-zero byte in @src that
 * userspace cannot see because they are using an smaller struct.
 *
 * Copies a struct from kernel space to userspace, in a way that guarantees
 * backwards-compatibility for struct syscall arguments (as long as future
 * struct extensions are made such that all new fields are *appended* to the
 * old struct, and zeroed-out new fields have the same meaning as the old
 * struct).
 *
 * Some syscalls may wish to make sure that userspace knows about everything in
 * the struct, and if there is a non-zero value that userspce doesn't know
 * about, they want to return an error (such as -EMSGSIZE) or have some other
 * fallback (such as adding a "you're missing some information" flag). If
 * @ignored_trailing is non-%NULL, it will be set to %true if there was a
 * non-zero byte that could not be copied to userspace (ie. was past @usize).
 *
 * While unconditionally returning an error in this case is the simplest
 * solution, for maximum backward compatibility you should try to only return
 * -EMSGSIZE if the user explicitly requested the data that couldn't be copied.
 * Note that structure sizes can change due to header changes and simple
 * recompilations without code changes(!), so if you care about
 * @ignored_trailing you probably want to make sure that any new field data is
 * associated with a flag. Otherwise you might assume that a program knows
 * about data it does not.
 *
 * @ksize is just sizeof(*src), and @usize should've been passed by userspace.
 * The recommended usage is something like the following:
 *
 *   SYSCALL_DEFINE2(foobar, struct foo __user *, uarg, size_t, usize)
 *   {
 *      int err;
 *      bool ignored_trailing;
 *      struct foo karg = {};
 *
 *      if (usize > PAGE_SIZE)
 *                return -E2BIG;
 *      if (usize < FOO_SIZE_VER0)
 *                return -EINVAL;
 *
 *      // ... modify karg somehow ...
 *
 *      err = copy_struct_to_user(uarg, usize, &karg, sizeof(karg),
 *                                  &ignored_trailing);
 *      if (err)
 *                return err;
 *      if (ignored_trailing)
 *                return -EMSGSIZE:
 *
 *      // ...
 *   }
 *
 * There are three cases to consider:
 *  * If @usize == @ksize, then it's copied verbatim.
 *  * If @usize < @ksize, then the kernel is trying to pass userspace a newer
 *    struct than it supports. Thus we only copy the interoperable portions
 *    (@usize) and ignore the rest (but @ignored_trailing is set to %true if
 *    any of the trailing (@ksize - @usize) bytes are non-zero).
 *  * If @usize > @ksize, then the kernel is trying to pass userspace an older
 *    struct than userspace supports. In order to make sure the
 *    unknown-to-the-kernel fields don't contain garbage values, we zero the
 *    trailing (@usize - @ksize) bytes.
 *
 * Returns (in all cases, some data may have been copied):
 *  * -EFAULT: access to userspace failed.
 */
static __always_inline __must_check int
copy_struct_to_user(void __user *dst, size_t usize, const void *src,
                    size_t ksize, bool *ignored_trailing)
{
        size_t size = min(ksize, usize);
        size_t rest = max(ksize, usize) - size;

        /* Double check if ksize is larger than a known object size. */
        if (WARN_ON_ONCE(ksize > __builtin_object_size(src, 1)))
                return -E2BIG;

        /* Deal with trailing bytes. */
        if (usize > ksize) {
                if (clear_user(dst + size, rest))
                        return -EFAULT;
        }
        if (ignored_trailing)
                *ignored_trailing = ksize < usize &&
                        memchr_inv(src + size, 0, rest) != NULL;
        /* Copy the interoperable parts of the struct. */
        if (copy_to_user(dst, src, size))
                return -EFAULT;
        return 0;
}

bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);

long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
long notrace copy_to_kernel_nofault(void *dst, const void *src, size_t size);

long copy_from_user_nofault(void *dst, const void __user *src, size_t size);
long notrace copy_to_user_nofault(void __user *dst, const void *src,
                size_t size);

long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr,
                long count);

long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
                long count);
long strnlen_user_nofault(const void __user *unsafe_addr, long count);

#ifndef __get_kernel_nofault
#define __get_kernel_nofault(dst, src, type, label)        \
do {                                                        \
        type __user *p = (type __force __user *)(src);        \
        type data;                                        \
        if (__get_user(data, p))                        \
                goto label;                                \
        *(type *)dst = data;                                \
} while (0)

#define __put_kernel_nofault(dst, src, type, label)        \
do {                                                        \
        type __user *p = (type __force __user *)(dst);        \
        type data = *(type *)src;                        \
        if (__put_user(data, p))                        \
                goto label;                                \
} while (0)
#endif

/**
 * get_kernel_nofault(): safely attempt to read from a location
 * @val: read into this variable
 * @ptr: address to read from
 *
 * Returns 0 on success, or -EFAULT.
 */
#define get_kernel_nofault(val, ptr) ({                                \
        const typeof(val) *__gk_ptr = (ptr);                        \
        copy_from_kernel_nofault(&(val), __gk_ptr, sizeof(val));\
})

#ifndef user_access_begin
#define user_access_begin(ptr,len) access_ok(ptr, len)
#define user_access_end() do { } while (0)
#define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
#define unsafe_get_user(x,p,e) unsafe_op_wrap(__get_user(x,p),e)
#define unsafe_put_user(x,p,e) unsafe_op_wrap(__put_user(x,p),e)
#define unsafe_copy_to_user(d,s,l,e) unsafe_op_wrap(__copy_to_user(d,s,l),e)
#define unsafe_copy_from_user(d,s,l,e) unsafe_op_wrap(__copy_from_user(d,s,l),e)
static inline unsigned long user_access_save(void) { return 0UL; }
static inline void user_access_restore(unsigned long flags) { }
#endif
#ifndef user_write_access_begin
#define user_write_access_begin user_access_begin
#define user_write_access_end user_access_end
#endif
#ifndef user_read_access_begin
#define user_read_access_begin user_access_begin
#define user_read_access_end user_access_end
#endif

#ifdef CONFIG_HARDENED_USERCOPY
void __noreturn usercopy_abort(const char *name, const char *detail,
                               bool to_user, unsigned long offset,
                               unsigned long len);
#endif

#endif                /* __LINUX_UACCESS_H__ */






















































































































































































































































































































































































































































































































































































































































































































































































   25 


   25 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
// SPDX-License-Identifier: GPL-2.0-only
/*
 * VHT handling
 *
 * Portions of this file
 * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
 * Copyright (C) 2018 - 2024 Intel Corporation
 */

#include <linux/ieee80211.h>
#include <linux/export.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "rate.h"


static void __check_vhtcap_disable(struct ieee80211_sub_if_data *sdata,
                                   struct ieee80211_sta_vht_cap *vht_cap,
                                   u32 flag)
{
        __le32 le_flag = cpu_to_le32(flag);

        if (sdata->u.mgd.vht_capa_mask.vht_cap_info & le_flag &&
            !(sdata->u.mgd.vht_capa.vht_cap_info & le_flag))
                vht_cap->cap &= ~flag;
}

void ieee80211_apply_vhtcap_overrides(struct ieee80211_sub_if_data *sdata,
                                      struct ieee80211_sta_vht_cap *vht_cap)
{
        int i;
        u16 rxmcs_mask, rxmcs_cap, rxmcs_n, txmcs_mask, txmcs_cap, txmcs_n;

        if (!vht_cap->vht_supported)
                return;

        if (sdata->vif.type != NL80211_IFTYPE_STATION)
                return;

        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_RXLDPC);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_SHORT_GI_80);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_SHORT_GI_160);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_TXSTBC);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN);
        __check_vhtcap_disable(sdata, vht_cap,
                               IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN);

        /* Allow user to decrease AMPDU length exponent */
        if (sdata->u.mgd.vht_capa_mask.vht_cap_info &
            cpu_to_le32(IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK)) {
                u32 cap, n;

                n = le32_to_cpu(sdata->u.mgd.vht_capa.vht_cap_info) &
                        IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
                n >>= IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;
                cap = vht_cap->cap & IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
                cap >>= IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;

                if (n < cap) {
                        vht_cap->cap &=
                                ~IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
                        vht_cap->cap |=
                                n << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT;
                }
        }

        /* Allow the user to decrease MCSes */
        rxmcs_mask =
                le16_to_cpu(sdata->u.mgd.vht_capa_mask.supp_mcs.rx_mcs_map);
        rxmcs_n = le16_to_cpu(sdata->u.mgd.vht_capa.supp_mcs.rx_mcs_map);
        rxmcs_n &= rxmcs_mask;
        rxmcs_cap = le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map);

        txmcs_mask =
                le16_to_cpu(sdata->u.mgd.vht_capa_mask.supp_mcs.tx_mcs_map);
        txmcs_n = le16_to_cpu(sdata->u.mgd.vht_capa.supp_mcs.tx_mcs_map);
        txmcs_n &= txmcs_mask;
        txmcs_cap = le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map);
        for (i = 0; i < 8; i++) {
                u8 m, n, c;

                m = (rxmcs_mask >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;
                n = (rxmcs_n >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;
                c = (rxmcs_cap >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                if (m && ((c != IEEE80211_VHT_MCS_NOT_SUPPORTED && n < c) ||
                          n == IEEE80211_VHT_MCS_NOT_SUPPORTED)) {
                        rxmcs_cap &= ~(3 << 2*i);
                        rxmcs_cap |= (rxmcs_n & (3 << 2*i));
                }

                m = (txmcs_mask >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;
                n = (txmcs_n >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;
                c = (txmcs_cap >> 2*i) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                if (m && ((c != IEEE80211_VHT_MCS_NOT_SUPPORTED && n < c) ||
                          n == IEEE80211_VHT_MCS_NOT_SUPPORTED)) {
                        txmcs_cap &= ~(3 << 2*i);
                        txmcs_cap |= (txmcs_n & (3 << 2*i));
                }
        }
        vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(rxmcs_cap);
        vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(txmcs_cap);
}

void
ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata,
                                    struct ieee80211_supported_band *sband,
                                    const struct ieee80211_vht_cap *vht_cap_ie,
                                    const struct ieee80211_vht_cap *vht_cap_ie2,
                                    struct link_sta_info *link_sta)
{
        struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
        struct ieee80211_sta_vht_cap own_cap;
        u32 cap_info, i;
        bool have_80mhz;
        u32 mpdu_len;

        memset(vht_cap, 0, sizeof(*vht_cap));

        if (!link_sta->pub->ht_cap.ht_supported)
                return;

        if (!vht_cap_ie || !sband->vht_cap.vht_supported)
                return;

        /* Allow VHT if at least one channel on the sband supports 80 MHz */
        have_80mhz = false;
        for (i = 0; i < sband->n_channels; i++) {
                if (sband->channels[i].flags & (IEEE80211_CHAN_DISABLED |
                                                IEEE80211_CHAN_NO_80MHZ))
                        continue;

                have_80mhz = true;
                break;
        }

        if (!have_80mhz)
                return;

        /*
         * A VHT STA must support 40 MHz, but if we verify that here
         * then we break a few things - some APs (e.g. Netgear R6300v2
         * and others based on the BCM4360 chipset) will unset this
         * capability bit when operating in 20 MHz.
         */

        vht_cap->vht_supported = true;

        own_cap = sband->vht_cap;
        /*
         * If user has specified capability overrides, take care
         * of that if the station we're setting up is the AP that
         * we advertised a restricted capability set to. Override
         * our own capabilities and then use those below.
         */
        if (sdata->vif.type == NL80211_IFTYPE_STATION &&
            !test_sta_flag(link_sta->sta, WLAN_STA_TDLS_PEER))
                ieee80211_apply_vhtcap_overrides(sdata, &own_cap);

        /* take some capabilities as-is */
        cap_info = le32_to_cpu(vht_cap_ie->vht_cap_info);
        vht_cap->cap = cap_info;
        vht_cap->cap &= IEEE80211_VHT_CAP_RXLDPC |
                        IEEE80211_VHT_CAP_VHT_TXOP_PS |
                        IEEE80211_VHT_CAP_HTC_VHT |
                        IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK |
                        IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_UNSOL_MFB |
                        IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB |
                        IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN |
                        IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN;

        vht_cap->cap |= min_t(u32, cap_info & IEEE80211_VHT_CAP_MAX_MPDU_MASK,
                              own_cap.cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK);

        /* and some based on our own capabilities */
        switch (own_cap.cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK) {
        case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ:
                vht_cap->cap |= cap_info &
                                IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ;
                break;
        case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ:
                vht_cap->cap |= cap_info &
                                IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
                break;
        default:
                /* nothing */
                break;
        }

        /* symmetric capabilities */
        vht_cap->cap |= cap_info & own_cap.cap &
                        (IEEE80211_VHT_CAP_SHORT_GI_80 |
                         IEEE80211_VHT_CAP_SHORT_GI_160);

        /* remaining ones */
        if (own_cap.cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE)
                vht_cap->cap |= cap_info &
                                (IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE |
                                 IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK);

        if (own_cap.cap & IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE)
                vht_cap->cap |= cap_info &
                                (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
                                 IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK);

        if (own_cap.cap & IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE)
                vht_cap->cap |= cap_info &
                                IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE;

        if (own_cap.cap & IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE)
                vht_cap->cap |= cap_info &
                                IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE;

        if (own_cap.cap & IEEE80211_VHT_CAP_TXSTBC)
                vht_cap->cap |= cap_info & IEEE80211_VHT_CAP_RXSTBC_MASK;

        if (own_cap.cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
                vht_cap->cap |= cap_info & IEEE80211_VHT_CAP_TXSTBC;

        /* Copy peer MCS info, the driver might need them. */
        memcpy(&vht_cap->vht_mcs, &vht_cap_ie->supp_mcs,
               sizeof(struct ieee80211_vht_mcs_info));

        /* copy EXT_NSS_BW Support value or remove the capability */
        if (ieee80211_hw_check(&sdata->local->hw, SUPPORTS_VHT_EXT_NSS_BW))
                vht_cap->cap |= (cap_info & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK);
        else
                vht_cap->vht_mcs.tx_highest &=
                        ~cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE);

        /* but also restrict MCSes */
        for (i = 0; i < 8; i++) {
                u16 own_rx, own_tx, peer_rx, peer_tx;

                own_rx = le16_to_cpu(own_cap.vht_mcs.rx_mcs_map);
                own_rx = (own_rx >> i * 2) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                own_tx = le16_to_cpu(own_cap.vht_mcs.tx_mcs_map);
                own_tx = (own_tx >> i * 2) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                peer_rx = le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map);
                peer_rx = (peer_rx >> i * 2) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                peer_tx = le16_to_cpu(vht_cap->vht_mcs.tx_mcs_map);
                peer_tx = (peer_tx >> i * 2) & IEEE80211_VHT_MCS_NOT_SUPPORTED;

                if (peer_tx != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
                        if (own_rx == IEEE80211_VHT_MCS_NOT_SUPPORTED)
                                peer_tx = IEEE80211_VHT_MCS_NOT_SUPPORTED;
                        else if (own_rx < peer_tx)
                                peer_tx = own_rx;
                }

                if (peer_rx != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
                        if (own_tx == IEEE80211_VHT_MCS_NOT_SUPPORTED)
                                peer_rx = IEEE80211_VHT_MCS_NOT_SUPPORTED;
                        else if (own_tx < peer_rx)
                                peer_rx = own_tx;
                }

                vht_cap->vht_mcs.rx_mcs_map &=
                        ~cpu_to_le16(IEEE80211_VHT_MCS_NOT_SUPPORTED << i * 2);
                vht_cap->vht_mcs.rx_mcs_map |= cpu_to_le16(peer_rx << i * 2);

                vht_cap->vht_mcs.tx_mcs_map &=
                        ~cpu_to_le16(IEEE80211_VHT_MCS_NOT_SUPPORTED << i * 2);
                vht_cap->vht_mcs.tx_mcs_map |= cpu_to_le16(peer_tx << i * 2);
        }

        /*
         * This is a workaround for VHT-enabled STAs which break the spec
         * and have the VHT-MCS Rx map filled in with value 3 for all eight
         * spatial streams, an example is AR9462.
         *
         * As per spec, in section 22.1.1 Introduction to the VHT PHY
         * A VHT STA shall support at least single spatial stream VHT-MCSs
         * 0 to 7 (transmit and receive) in all supported channel widths.
         */
        if (vht_cap->vht_mcs.rx_mcs_map == cpu_to_le16(0xFFFF)) {
                vht_cap->vht_supported = false;
                sdata_info(sdata,
                           "Ignoring VHT IE from %pM (link:%pM) due to invalid rx_mcs_map\n",
                           link_sta->sta->addr, link_sta->addr);
                return;
        }

        /* finally set up the bandwidth */
        switch (vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK) {
        case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ:
        case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ:
                link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
                break;
        default:
                link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_80;

                if (!(vht_cap->vht_mcs.tx_highest &
                                cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE)))
                        break;

                /*
                 * If this is non-zero, then it does support 160 MHz after all,
                 * in one form or the other. We don't distinguish here (or even
                 * above) between 160 and 80+80 yet.
                 */
                if (cap_info & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)
                        link_sta->cur_max_bandwidth =
                                IEEE80211_STA_RX_BW_160;
        }

        link_sta->pub->bandwidth = ieee80211_sta_cur_vht_bw(link_sta);

        /*
         * Work around the Cisco 9115 FW 17.3 bug by taking the min of
         * both reported MPDU lengths.
         */
        mpdu_len = vht_cap->cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK;
        if (vht_cap_ie2)
                mpdu_len = min_t(u32, mpdu_len,
                                 le32_get_bits(vht_cap_ie2->vht_cap_info,
                                               IEEE80211_VHT_CAP_MAX_MPDU_MASK));

        /*
         * FIXME - should the amsdu len be per link? store per link
         * and maintain a minimum?
         */
        switch (mpdu_len) {
        case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
                link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454;
                break;
        case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991:
                link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991;
                break;
        case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895:
        default:
                link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895;
                break;
        }

        ieee80211_sta_recalc_aggregates(&link_sta->sta->sta);
}

/* FIXME: move this to some better location - parses HE/EHT now */
enum ieee80211_sta_rx_bandwidth
_ieee80211_sta_cap_rx_bw(struct link_sta_info *link_sta,
                         struct cfg80211_chan_def *chandef)
{
        unsigned int link_id = link_sta->link_id;
        struct ieee80211_sub_if_data *sdata = link_sta->sta->sdata;
        struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
        struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
        struct ieee80211_sta_eht_cap *eht_cap = &link_sta->pub->eht_cap;
        u32 cap_width;

        if (he_cap->has_he) {
                enum nl80211_band band;
                u8 info;

                if (chandef) {
                        band = chandef->chan->band;
                } else {
                        struct ieee80211_bss_conf *link_conf;

                        rcu_read_lock();
                        link_conf = rcu_dereference(sdata->vif.link_conf[link_id]);
                        band = link_conf->chanreq.oper.chan->band;
                        rcu_read_unlock();
                }

                if (eht_cap->has_eht && band == NL80211_BAND_6GHZ) {
                        info = eht_cap->eht_cap_elem.phy_cap_info[0];

                        if (info & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
                                return IEEE80211_STA_RX_BW_320;
                }

                info = he_cap->he_cap_elem.phy_cap_info[0];

                if (band == NL80211_BAND_2GHZ) {
                        if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
                                return IEEE80211_STA_RX_BW_40;
                        return IEEE80211_STA_RX_BW_20;
                }

                if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G ||
                    info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
                        return IEEE80211_STA_RX_BW_160;

                if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
                        return IEEE80211_STA_RX_BW_80;

                return IEEE80211_STA_RX_BW_20;
        }

        if (!vht_cap->vht_supported)
                return link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
                                IEEE80211_STA_RX_BW_40 :
                                IEEE80211_STA_RX_BW_20;

        cap_width = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;

        if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ ||
            cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
                return IEEE80211_STA_RX_BW_160;

        /*
         * If this is non-zero, then it does support 160 MHz after all,
         * in one form or the other. We don't distinguish here (or even
         * above) between 160 and 80+80 yet.
         */
        if (vht_cap->cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)
                return IEEE80211_STA_RX_BW_160;

        return IEEE80211_STA_RX_BW_80;
}

enum nl80211_chan_width
ieee80211_sta_cap_chan_bw(struct link_sta_info *link_sta)
{
        struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
        u32 cap_width;

        if (!vht_cap->vht_supported) {
                if (!link_sta->pub->ht_cap.ht_supported)
                        return NL80211_CHAN_WIDTH_20_NOHT;

                return link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
                                NL80211_CHAN_WIDTH_40 : NL80211_CHAN_WIDTH_20;
        }

        cap_width = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;

        if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ)
                return NL80211_CHAN_WIDTH_160;
        else if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
                return NL80211_CHAN_WIDTH_80P80;

        return NL80211_CHAN_WIDTH_80;
}

enum nl80211_chan_width
ieee80211_sta_rx_bw_to_chan_width(struct link_sta_info *link_sta)
{
        enum ieee80211_sta_rx_bandwidth cur_bw =
                link_sta->pub->bandwidth;
        struct ieee80211_sta_vht_cap *vht_cap =
                &link_sta->pub->vht_cap;
        u32 cap_width;

        switch (cur_bw) {
        case IEEE80211_STA_RX_BW_20:
                if (!link_sta->pub->ht_cap.ht_supported)
                        return NL80211_CHAN_WIDTH_20_NOHT;
                else
                        return NL80211_CHAN_WIDTH_20;
        case IEEE80211_STA_RX_BW_40:
                return NL80211_CHAN_WIDTH_40;
        case IEEE80211_STA_RX_BW_80:
                return NL80211_CHAN_WIDTH_80;
        case IEEE80211_STA_RX_BW_160:
                cap_width =
                        vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;

                if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ)
                        return NL80211_CHAN_WIDTH_160;

                return NL80211_CHAN_WIDTH_80P80;
        default:
                return NL80211_CHAN_WIDTH_20;
        }
}

/* FIXME: rename/move - this deals with everything not just VHT */
enum ieee80211_sta_rx_bandwidth
_ieee80211_sta_cur_vht_bw(struct link_sta_info *link_sta,
                          struct cfg80211_chan_def *chandef)
{
        struct sta_info *sta = link_sta->sta;
        enum nl80211_chan_width bss_width;
        enum ieee80211_sta_rx_bandwidth bw;

        if (chandef) {
                bss_width = chandef->width;
        } else {
                struct ieee80211_bss_conf *link_conf;

                rcu_read_lock();
                link_conf = rcu_dereference(sta->sdata->vif.link_conf[link_sta->link_id]);
                if (WARN_ON_ONCE(!link_conf)) {
                        rcu_read_unlock();
                        return IEEE80211_STA_RX_BW_20;
                }
                bss_width = link_conf->chanreq.oper.width;
                rcu_read_unlock();
        }

        bw = _ieee80211_sta_cap_rx_bw(link_sta, chandef);
        bw = min(bw, link_sta->cur_max_bandwidth);

        /* Don't consider AP's bandwidth for TDLS peers, section 11.23.1 of
         * IEEE80211-2016 specification makes higher bandwidth operation
         * possible on the TDLS link if the peers have wider bandwidth
         * capability.
         *
         * However, in this case, and only if the TDLS peer is authorized,
         * limit to the tdls_chandef so that the configuration here isn't
         * wider than what's actually requested on the channel context.
         */
        if (test_sta_flag(sta, WLAN_STA_TDLS_PEER) &&
            test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW) &&
            test_sta_flag(sta, WLAN_STA_AUTHORIZED) &&
            sta->tdls_chandef.chan)
                bw = min(bw, ieee80211_chan_width_to_rx_bw(sta->tdls_chandef.width));
        else
                bw = min(bw, ieee80211_chan_width_to_rx_bw(bss_width));

        return bw;
}

void ieee80211_sta_init_nss(struct link_sta_info *link_sta)
{
        u8 ht_rx_nss = 0, vht_rx_nss = 0, he_rx_nss = 0, eht_rx_nss = 0, rx_nss;
        bool support_160;

        if (link_sta->pub->eht_cap.has_eht) {
                int i;
                const u8 *rx_nss_mcs = (void *)&link_sta->pub->eht_cap.eht_mcs_nss_supp;

                /* get the max nss for EHT over all possible bandwidths and mcs */
                for (i = 0; i < sizeof(struct ieee80211_eht_mcs_nss_supp); i++)
                        eht_rx_nss = max_t(u8, eht_rx_nss,
                                           u8_get_bits(rx_nss_mcs[i],
                                                       IEEE80211_EHT_MCS_NSS_RX));
        }

        if (link_sta->pub->he_cap.has_he) {
                int i;
                u8 rx_mcs_80 = 0, rx_mcs_160 = 0;
                const struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
                u16 mcs_160_map =
                        le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
                u16 mcs_80_map = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);

                for (i = 7; i >= 0; i--) {
                        u8 mcs_160 = (mcs_160_map >> (2 * i)) & 3;

                        if (mcs_160 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
                                rx_mcs_160 = i + 1;
                                break;
                        }
                }
                for (i = 7; i >= 0; i--) {
                        u8 mcs_80 = (mcs_80_map >> (2 * i)) & 3;

                        if (mcs_80 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
                                rx_mcs_80 = i + 1;
                                break;
                        }
                }

                support_160 = he_cap->he_cap_elem.phy_cap_info[0] &
                              IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;

                if (support_160)
                        he_rx_nss = min(rx_mcs_80, rx_mcs_160);
                else
                        he_rx_nss = rx_mcs_80;
        }

        if (link_sta->pub->ht_cap.ht_supported) {
                if (link_sta->pub->ht_cap.mcs.rx_mask[0])
                        ht_rx_nss++;
                if (link_sta->pub->ht_cap.mcs.rx_mask[1])
                        ht_rx_nss++;
                if (link_sta->pub->ht_cap.mcs.rx_mask[2])
                        ht_rx_nss++;
                if (link_sta->pub->ht_cap.mcs.rx_mask[3])
                        ht_rx_nss++;
                /* FIXME: consider rx_highest? */
        }

        if (link_sta->pub->vht_cap.vht_supported) {
                int i;
                u16 rx_mcs_map;

                rx_mcs_map = le16_to_cpu(link_sta->pub->vht_cap.vht_mcs.rx_mcs_map);

                for (i = 7; i >= 0; i--) {
                        u8 mcs = (rx_mcs_map >> (2 * i)) & 3;

                        if (mcs != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
                                vht_rx_nss = i + 1;
                                break;
                        }
                }
                /* FIXME: consider rx_highest? */
        }

        rx_nss = max(vht_rx_nss, ht_rx_nss);
        rx_nss = max(he_rx_nss, rx_nss);
        rx_nss = max(eht_rx_nss, rx_nss);
        rx_nss = max_t(u8, 1, rx_nss);
        link_sta->capa_nss = rx_nss;

        /* that shouldn't be set yet, but we can handle it anyway */
        if (link_sta->op_mode_nss)
                link_sta->pub->rx_nss =
                        min_t(u8, rx_nss, link_sta->op_mode_nss);
        else
                link_sta->pub->rx_nss = rx_nss;
}

u32 __ieee80211_vht_handle_opmode(struct ieee80211_sub_if_data *sdata,
                                  struct link_sta_info *link_sta,
                                  u8 opmode, enum nl80211_band band)
{
        enum ieee80211_sta_rx_bandwidth new_bw;
        struct sta_opmode_info sta_opmode = {};
        u32 changed = 0;
        u8 nss;

        /* ignore - no support for BF yet */
        if (opmode & IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF)
                return 0;

        nss = opmode & IEEE80211_OPMODE_NOTIF_RX_NSS_MASK;
        nss >>= IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT;
        nss += 1;

        if (link_sta->op_mode_nss != nss) {
                if (nss <= link_sta->capa_nss) {
                        link_sta->op_mode_nss = nss;

                        if (nss != link_sta->pub->rx_nss) {
                                link_sta->pub->rx_nss = nss;
                                changed |= IEEE80211_RC_NSS_CHANGED;
                                sta_opmode.rx_nss = link_sta->pub->rx_nss;
                                sta_opmode.changed |= STA_OPMODE_N_SS_CHANGED;
                        }
                } else {
                        pr_warn_ratelimited("Ignoring NSS change in VHT Operating Mode Notification from %pM with invalid nss %d",
                                            link_sta->pub->addr, nss);
                }
        }

        switch (opmode & IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK) {
        case IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ:
                /* ignore IEEE80211_OPMODE_NOTIF_BW_160_80P80 must not be set */
                link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_20;
                break;
        case IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ:
                /* ignore IEEE80211_OPMODE_NOTIF_BW_160_80P80 must not be set */
                link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_40;
                break;
        case IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ:
                if (opmode & IEEE80211_OPMODE_NOTIF_BW_160_80P80)
                        link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
                else
                        link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_80;
                break;
        case IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ:
                /* legacy only, no longer used by newer spec */
                link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
                break;
        }

        new_bw = ieee80211_sta_cur_vht_bw(link_sta);
        if (new_bw != link_sta->pub->bandwidth) {
                link_sta->pub->bandwidth = new_bw;
                sta_opmode.bw = ieee80211_sta_rx_bw_to_chan_width(link_sta);
                changed |= IEEE80211_RC_BW_CHANGED;
                sta_opmode.changed |= STA_OPMODE_MAX_BW_CHANGED;
        }

        if (sta_opmode.changed)
                cfg80211_sta_opmode_change_notify(sdata->dev, link_sta->addr,
                                                  &sta_opmode, GFP_KERNEL);

        return changed;
}

void ieee80211_process_mu_groups(struct ieee80211_sub_if_data *sdata,
                                 struct ieee80211_link_data *link,
                                 struct ieee80211_mgmt *mgmt)
{
        struct ieee80211_bss_conf *link_conf = link->conf;

        if (!link_conf->mu_mimo_owner)
                return;

        if (!memcmp(mgmt->u.action.u.vht_group_notif.position,
                    link_conf->mu_group.position, WLAN_USER_POSITION_LEN) &&
            !memcmp(mgmt->u.action.u.vht_group_notif.membership,
                    link_conf->mu_group.membership, WLAN_MEMBERSHIP_LEN))
                return;

        memcpy(link_conf->mu_group.membership,
               mgmt->u.action.u.vht_group_notif.membership,
               WLAN_MEMBERSHIP_LEN);
        memcpy(link_conf->mu_group.position,
               mgmt->u.action.u.vht_group_notif.position,
               WLAN_USER_POSITION_LEN);

        ieee80211_link_info_change_notify(sdata, link,
                                          BSS_CHANGED_MU_GROUPS);
}

void ieee80211_update_mu_groups(struct ieee80211_vif *vif, unsigned int link_id,
                                const u8 *membership, const u8 *position)
{
        struct ieee80211_bss_conf *link_conf;

        rcu_read_lock();
        link_conf = rcu_dereference(vif->link_conf[link_id]);

        if (!WARN_ON_ONCE(!link_conf || !link_conf->mu_mimo_owner)) {
                memcpy(link_conf->mu_group.membership, membership,
                       WLAN_MEMBERSHIP_LEN);
                memcpy(link_conf->mu_group.position, position,
                       WLAN_USER_POSITION_LEN);
        }
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(ieee80211_update_mu_groups);

void ieee80211_vht_handle_opmode(struct ieee80211_sub_if_data *sdata,
                                 struct link_sta_info *link_sta,
                                 u8 opmode, enum nl80211_band band)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_supported_band *sband = local->hw.wiphy->bands[band];

        u32 changed = __ieee80211_vht_handle_opmode(sdata, link_sta,
                                                    opmode, band);

        if (changed > 0) {
                ieee80211_recalc_min_chandef(sdata, link_sta->link_id);
                rate_control_rate_update(local, sband, link_sta, changed);
        }
}

void ieee80211_get_vht_mask_from_cap(__le16 vht_cap,
                                     u16 vht_mask[NL80211_VHT_NSS_MAX])
{
        int i;
        u16 mask, cap = le16_to_cpu(vht_cap);

        for (i = 0; i < NL80211_VHT_NSS_MAX; i++) {
                mask = (cap >> i * 2) & IEEE80211_VHT_MCS_NOT_SUPPORTED;
                switch (mask) {
                case IEEE80211_VHT_MCS_SUPPORT_0_7:
                        vht_mask[i] = 0x00FF;
                        break;
                case IEEE80211_VHT_MCS_SUPPORT_0_8:
                        vht_mask[i] = 0x01FF;
                        break;
                case IEEE80211_VHT_MCS_SUPPORT_0_9:
                        vht_mask[i] = 0x03FF;
                        break;
                case IEEE80211_VHT_MCS_NOT_SUPPORTED:
                default:
                        vht_mask[i] = 0;
                        break;
                }
        }
}

















   10 







   10 
    9 

   10 
    1 
   10 

   10 


   10 







   11 
   11 


   11 



    3 






    3 







   11 








    4 



















    4 






    4 



    4 










    8 



    1 


    7 














   13 




   13 






   10 


   10 
   10 



   13 
   11 


   13 
    4 


   13 
    8 


   13 
    4 


   13 




   13 









   15 










   13 
   13 

   13 


   13 




   13 


   13 


   15 
   13 







   16 







   15 


   14 
   13 


   16 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/module.h>
#include <linux/sock_diag.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/packet_diag.h>
#include <linux/percpu.h>
#include <net/net_namespace.h>
#include <net/sock.h>

#include "internal.h"

static int pdiag_put_info(const struct packet_sock *po, struct sk_buff *nlskb)
{
        struct packet_diag_info pinfo;

        pinfo.pdi_index = po->ifindex;
        pinfo.pdi_version = po->tp_version;
        pinfo.pdi_reserve = po->tp_reserve;
        pinfo.pdi_copy_thresh = READ_ONCE(po->copy_thresh);
        pinfo.pdi_tstamp = READ_ONCE(po->tp_tstamp);

        pinfo.pdi_flags = 0;
        if (packet_sock_flag(po, PACKET_SOCK_RUNNING))
                pinfo.pdi_flags |= PDI_RUNNING;
        if (packet_sock_flag(po, PACKET_SOCK_AUXDATA))
                pinfo.pdi_flags |= PDI_AUXDATA;
        if (packet_sock_flag(po, PACKET_SOCK_ORIGDEV))
                pinfo.pdi_flags |= PDI_ORIGDEV;
        if (READ_ONCE(po->vnet_hdr_sz))
                pinfo.pdi_flags |= PDI_VNETHDR;
        if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS))
                pinfo.pdi_flags |= PDI_LOSS;

        return nla_put(nlskb, PACKET_DIAG_INFO, sizeof(pinfo), &pinfo);
}

static int pdiag_put_mclist(const struct packet_sock *po, struct sk_buff *nlskb)
{
        struct nlattr *mca;
        struct packet_mclist *ml;

        mca = nla_nest_start_noflag(nlskb, PACKET_DIAG_MCLIST);
        if (!mca)
                return -EMSGSIZE;

        rtnl_lock();
        for (ml = po->mclist; ml; ml = ml->next) {
                struct packet_diag_mclist *dml;

                dml = nla_reserve_nohdr(nlskb, sizeof(*dml));
                if (!dml) {
                        rtnl_unlock();
                        nla_nest_cancel(nlskb, mca);
                        return -EMSGSIZE;
                }

                dml->pdmc_index = ml->ifindex;
                dml->pdmc_type = ml->type;
                dml->pdmc_alen = ml->alen;
                dml->pdmc_count = ml->count;
                BUILD_BUG_ON(sizeof(dml->pdmc_addr) != sizeof(ml->addr));
                memcpy(dml->pdmc_addr, ml->addr, sizeof(ml->addr));
        }

        rtnl_unlock();
        nla_nest_end(nlskb, mca);

        return 0;
}

static int pdiag_put_ring(struct packet_ring_buffer *ring, int ver, int nl_type,
                struct sk_buff *nlskb)
{
        struct packet_diag_ring pdr;

        if (!ring->pg_vec)
                return 0;

        pdr.pdr_block_size = ring->pg_vec_pages << PAGE_SHIFT;
        pdr.pdr_block_nr = ring->pg_vec_len;
        pdr.pdr_frame_size = ring->frame_size;
        pdr.pdr_frame_nr = ring->frame_max + 1;

        if (ver > TPACKET_V2) {
                pdr.pdr_retire_tmo = ring->prb_bdqc.retire_blk_tov;
                pdr.pdr_sizeof_priv = ring->prb_bdqc.blk_sizeof_priv;
                pdr.pdr_features = ring->prb_bdqc.feature_req_word;
        } else {
                pdr.pdr_retire_tmo = 0;
                pdr.pdr_sizeof_priv = 0;
                pdr.pdr_features = 0;
        }

        return nla_put(nlskb, nl_type, sizeof(pdr), &pdr);
}

static int pdiag_put_rings_cfg(struct packet_sock *po, struct sk_buff *skb)
{
        int ret;

        mutex_lock(&po->pg_vec_lock);
        ret = pdiag_put_ring(&po->rx_ring, po->tp_version,
                        PACKET_DIAG_RX_RING, skb);
        if (!ret)
                ret = pdiag_put_ring(&po->tx_ring, po->tp_version,
                                PACKET_DIAG_TX_RING, skb);
        mutex_unlock(&po->pg_vec_lock);

        return ret;
}

static int pdiag_put_fanout(struct packet_sock *po, struct sk_buff *nlskb)
{
        int ret = 0;

        mutex_lock(&fanout_mutex);
        if (po->fanout) {
                u32 val;

                val = (u32)po->fanout->id | ((u32)po->fanout->type << 16);
                ret = nla_put_u32(nlskb, PACKET_DIAG_FANOUT, val);
        }
        mutex_unlock(&fanout_mutex);

        return ret;
}

static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
                        struct packet_diag_req *req,
                        bool may_report_filterinfo,
                        struct user_namespace *user_ns,
                        u32 portid, u32 seq, u32 flags, int sk_ino)
{
        struct nlmsghdr *nlh;
        struct packet_diag_msg *rp;
        struct packet_sock *po = pkt_sk(sk);

        nlh = nlmsg_put(skb, portid, seq, SOCK_DIAG_BY_FAMILY, sizeof(*rp), flags);
        if (!nlh)
                return -EMSGSIZE;

        rp = nlmsg_data(nlh);
        rp->pdiag_family = AF_PACKET;
        rp->pdiag_type = sk->sk_type;
        rp->pdiag_num = ntohs(READ_ONCE(po->num));
        rp->pdiag_ino = sk_ino;
        sock_diag_save_cookie(sk, rp->pdiag_cookie);

        if ((req->pdiag_show & PACKET_SHOW_INFO) &&
                        pdiag_put_info(po, skb))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_INFO) &&
            nla_put_u32(skb, PACKET_DIAG_UID,
                        from_kuid_munged(user_ns, sock_i_uid(sk))))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_MCLIST) &&
                        pdiag_put_mclist(po, skb))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_RING_CFG) &&
                        pdiag_put_rings_cfg(po, skb))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_FANOUT) &&
                        pdiag_put_fanout(po, skb))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_MEMINFO) &&
            sock_diag_put_meminfo(sk, skb, PACKET_DIAG_MEMINFO))
                goto out_nlmsg_trim;

        if ((req->pdiag_show & PACKET_SHOW_FILTER) &&
            sock_diag_put_filterinfo(may_report_filterinfo, sk, skb,
                                     PACKET_DIAG_FILTER))
                goto out_nlmsg_trim;

        nlmsg_end(skb, nlh);
        return 0;

out_nlmsg_trim:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int packet_diag_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        int num = 0, s_num = cb->args[0];
        struct packet_diag_req *req;
        struct net *net;
        struct sock *sk;
        bool may_report_filterinfo;

        net = sock_net(skb->sk);
        req = nlmsg_data(cb->nlh);
        may_report_filterinfo = netlink_net_capable(cb->skb, CAP_NET_ADMIN);

        mutex_lock(&net->packet.sklist_lock);
        sk_for_each(sk, &net->packet.sklist) {
                if (!net_eq(sock_net(sk), net))
                        continue;
                if (num < s_num)
                        goto next;

                if (sk_diag_fill(sk, skb, req,
                                 may_report_filterinfo,
                                 sk_user_ns(NETLINK_CB(cb->skb).sk),
                                 NETLINK_CB(cb->skb).portid,
                                 cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                 sock_i_ino(sk)) < 0)
                        goto done;
next:
                num++;
        }
done:
        mutex_unlock(&net->packet.sklist_lock);
        cb->args[0] = num;

        return skb->len;
}

static int packet_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h)
{
        int hdrlen = sizeof(struct packet_diag_req);
        struct net *net = sock_net(skb->sk);
        struct packet_diag_req *req;

        if (nlmsg_len(h) < hdrlen)
                return -EINVAL;

        req = nlmsg_data(h);
        /* Make it possible to support protocol filtering later */
        if (req->sdiag_protocol)
                return -EINVAL;

        if (h->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .dump = packet_diag_dump,
                };
                return netlink_dump_start(net->diag_nlsk, skb, h, &c);
        } else
                return -EOPNOTSUPP;
}

static const struct sock_diag_handler packet_diag_handler = {
        .owner = THIS_MODULE,
        .family = AF_PACKET,
        .dump = packet_diag_handler_dump,
};

static int __init packet_diag_init(void)
{
        return sock_diag_register(&packet_diag_handler);
}

static void __exit packet_diag_exit(void)
{
        sock_diag_unregister(&packet_diag_handler);
}

module_init(packet_diag_init);
module_exit(packet_diag_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("PACKET socket monitoring via SOCK_DIAG");
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 17 /* AF_PACKET */);



























































  109 


   36 





























  203 











  204 




  203 








   37 







   38 






  204 











  203 
  203 
  203 

  204 




  204 


  203 

  200 

   38 







   86 


   85 


   86 









  109 




  108 
  108 



















































































    1 





































































































    1 









    1 

































    1 











    1 













































    1 
































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * This file contains sctp stream maniuplation primitives and helpers.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    Xin Long <lucien.xin@gmail.com>
 */

#include <linux/list.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>
#include <net/sctp/stream_sched.h>

static void sctp_stream_shrink_out(struct sctp_stream *stream, __u16 outcnt)
{
        struct sctp_association *asoc;
        struct sctp_chunk *ch, *temp;
        struct sctp_outq *outq;

        asoc = container_of(stream, struct sctp_association, stream);
        outq = &asoc->outqueue;

        list_for_each_entry_safe(ch, temp, &outq->out_chunk_list, list) {
                __u16 sid = sctp_chunk_stream_no(ch);

                if (sid < outcnt)
                        continue;

                sctp_sched_dequeue_common(outq, ch);
                /* No need to call dequeue_done here because
                 * the chunks are not scheduled by now.
                 */

                /* Mark as failed send. */
                sctp_chunk_fail(ch, (__force __u32)SCTP_ERROR_INV_STRM);
                if (asoc->peer.prsctp_capable &&
                    SCTP_PR_PRIO_ENABLED(ch->sinfo.sinfo_flags))
                        asoc->sent_cnt_removable--;

                sctp_chunk_free(ch);
        }
}

static void sctp_stream_free_ext(struct sctp_stream *stream, __u16 sid)
{
        struct sctp_sched_ops *sched;

        if (!SCTP_SO(stream, sid)->ext)
                return;

        sched = sctp_sched_ops_from_stream(stream);
        sched->free_sid(stream, sid);
        kfree(SCTP_SO(stream, sid)->ext);
        SCTP_SO(stream, sid)->ext = NULL;
}

/* Migrates chunks from stream queues to new stream queues if needed,
 * but not across associations. Also, removes those chunks to streams
 * higher than the new max.
 */
static void sctp_stream_outq_migrate(struct sctp_stream *stream,
                                     struct sctp_stream *new, __u16 outcnt)
{
        int i;

        if (stream->outcnt > outcnt)
                sctp_stream_shrink_out(stream, outcnt);

        if (new) {
                /* Here we actually move the old ext stuff into the new
                 * buffer, because we want to keep it. Then
                 * sctp_stream_update will swap ->out pointers.
                 */
                for (i = 0; i < outcnt; i++) {
                        sctp_stream_free_ext(new, i);
                        SCTP_SO(new, i)->ext = SCTP_SO(stream, i)->ext;
                        SCTP_SO(stream, i)->ext = NULL;
                }
        }

        for (i = outcnt; i < stream->outcnt; i++)
                sctp_stream_free_ext(stream, i);
}

static int sctp_stream_alloc_out(struct sctp_stream *stream, __u16 outcnt,
                                 gfp_t gfp)
{
        int ret;

        if (outcnt <= stream->outcnt)
                goto out;

        ret = genradix_prealloc(&stream->out, outcnt, gfp);
        if (ret)
                return ret;

out:
        stream->outcnt = outcnt;
        return 0;
}

static int sctp_stream_alloc_in(struct sctp_stream *stream, __u16 incnt,
                                gfp_t gfp)
{
        int ret;

        if (incnt <= stream->incnt)
                goto out;

        ret = genradix_prealloc(&stream->in, incnt, gfp);
        if (ret)
                return ret;

out:
        stream->incnt = incnt;
        return 0;
}

int sctp_stream_init(struct sctp_stream *stream, __u16 outcnt, __u16 incnt,
                     gfp_t gfp)
{
        struct sctp_sched_ops *sched = sctp_sched_ops_from_stream(stream);
        int i, ret = 0;

        gfp |= __GFP_NOWARN;

        /* Initial stream->out size may be very big, so free it and alloc
         * a new one with new outcnt to save memory if needed.
         */
        if (outcnt == stream->outcnt)
                goto handle_in;

        /* Filter out chunks queued on streams that won't exist anymore */
        sched->unsched_all(stream);
        sctp_stream_outq_migrate(stream, NULL, outcnt);
        sched->sched_all(stream);

        ret = sctp_stream_alloc_out(stream, outcnt, gfp);
        if (ret)
                return ret;

        for (i = 0; i < stream->outcnt; i++)
                SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;

handle_in:
        sctp_stream_interleave_init(stream);
        if (!incnt)
                return 0;

        return sctp_stream_alloc_in(stream, incnt, gfp);
}

int sctp_stream_init_ext(struct sctp_stream *stream, __u16 sid)
{
        struct sctp_stream_out_ext *soute;
        int ret;

        soute = kzalloc(sizeof(*soute), GFP_KERNEL);
        if (!soute)
                return -ENOMEM;
        SCTP_SO(stream, sid)->ext = soute;

        ret = sctp_sched_init_sid(stream, sid, GFP_KERNEL);
        if (ret) {
                kfree(SCTP_SO(stream, sid)->ext);
                SCTP_SO(stream, sid)->ext = NULL;
        }

        return ret;
}

void sctp_stream_free(struct sctp_stream *stream)
{
        struct sctp_sched_ops *sched = sctp_sched_ops_from_stream(stream);
        int i;

        sched->unsched_all(stream);
        for (i = 0; i < stream->outcnt; i++)
                sctp_stream_free_ext(stream, i);
        genradix_free(&stream->out);
        genradix_free(&stream->in);
}

void sctp_stream_clear(struct sctp_stream *stream)
{
        int i;

        for (i = 0; i < stream->outcnt; i++) {
                SCTP_SO(stream, i)->mid = 0;
                SCTP_SO(stream, i)->mid_uo = 0;
        }

        for (i = 0; i < stream->incnt; i++)
                SCTP_SI(stream, i)->mid = 0;
}

void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new)
{
        struct sctp_sched_ops *sched = sctp_sched_ops_from_stream(stream);

        sched->unsched_all(stream);
        sctp_stream_outq_migrate(stream, new, new->outcnt);
        sctp_stream_free(stream);

        stream->out = new->out;
        stream->in  = new->in;
        stream->outcnt = new->outcnt;
        stream->incnt  = new->incnt;

        sched->sched_all(stream);

        new->out.tree.root = NULL;
        new->in.tree.root  = NULL;
        new->outcnt = 0;
        new->incnt  = 0;
}

static int sctp_send_reconf(struct sctp_association *asoc,
                            struct sctp_chunk *chunk)
{
        int retval = 0;

        retval = sctp_primitive_RECONF(asoc->base.net, asoc, chunk);
        if (retval)
                sctp_chunk_free(chunk);

        return retval;
}

static bool sctp_stream_outq_is_empty(struct sctp_stream *stream,
                                      __u16 str_nums, __be16 *str_list)
{
        struct sctp_association *asoc;
        __u16 i;

        asoc = container_of(stream, struct sctp_association, stream);
        if (!asoc->outqueue.out_qlen)
                return true;

        if (!str_nums)
                return false;

        for (i = 0; i < str_nums; i++) {
                __u16 sid = ntohs(str_list[i]);

                if (SCTP_SO(stream, sid)->ext &&
                    !list_empty(&SCTP_SO(stream, sid)->ext->outq))
                        return false;
        }

        return true;
}

int sctp_send_reset_streams(struct sctp_association *asoc,
                            struct sctp_reset_streams *params)
{
        struct sctp_stream *stream = &asoc->stream;
        __u16 i, str_nums, *str_list;
        struct sctp_chunk *chunk;
        int retval = -EINVAL;
        __be16 *nstr_list;
        bool out, in;

        if (!asoc->peer.reconf_capable ||
            !(asoc->strreset_enable & SCTP_ENABLE_RESET_STREAM_REQ)) {
                retval = -ENOPROTOOPT;
                goto out;
        }

        if (asoc->strreset_outstanding) {
                retval = -EINPROGRESS;
                goto out;
        }

        out = params->srs_flags & SCTP_STREAM_RESET_OUTGOING;
        in  = params->srs_flags & SCTP_STREAM_RESET_INCOMING;
        if (!out && !in)
                goto out;

        str_nums = params->srs_number_streams;
        str_list = params->srs_stream_list;
        if (str_nums) {
                int param_len = 0;

                if (out) {
                        for (i = 0; i < str_nums; i++)
                                if (str_list[i] >= stream->outcnt)
                                        goto out;

                        param_len = str_nums * sizeof(__u16) +
                                    sizeof(struct sctp_strreset_outreq);
                }

                if (in) {
                        for (i = 0; i < str_nums; i++)
                                if (str_list[i] >= stream->incnt)
                                        goto out;

                        param_len += str_nums * sizeof(__u16) +
                                     sizeof(struct sctp_strreset_inreq);
                }

                if (param_len > SCTP_MAX_CHUNK_LEN -
                                sizeof(struct sctp_reconf_chunk))
                        goto out;
        }

        nstr_list = kcalloc(str_nums, sizeof(__be16), GFP_KERNEL);
        if (!nstr_list) {
                retval = -ENOMEM;
                goto out;
        }

        for (i = 0; i < str_nums; i++)
                nstr_list[i] = htons(str_list[i]);

        if (out && !sctp_stream_outq_is_empty(stream, str_nums, nstr_list)) {
                kfree(nstr_list);
                retval = -EAGAIN;
                goto out;
        }

        chunk = sctp_make_strreset_req(asoc, str_nums, nstr_list, out, in);

        kfree(nstr_list);

        if (!chunk) {
                retval = -ENOMEM;
                goto out;
        }

        if (out) {
                if (str_nums)
                        for (i = 0; i < str_nums; i++)
                                SCTP_SO(stream, str_list[i])->state =
                                                       SCTP_STREAM_CLOSED;
                else
                        for (i = 0; i < stream->outcnt; i++)
                                SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED;
        }

        asoc->strreset_chunk = chunk;
        sctp_chunk_hold(asoc->strreset_chunk);

        retval = sctp_send_reconf(asoc, chunk);
        if (retval) {
                sctp_chunk_put(asoc->strreset_chunk);
                asoc->strreset_chunk = NULL;
                if (!out)
                        goto out;

                if (str_nums)
                        for (i = 0; i < str_nums; i++)
                                SCTP_SO(stream, str_list[i])->state =
                                                       SCTP_STREAM_OPEN;
                else
                        for (i = 0; i < stream->outcnt; i++)
                                SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;

                goto out;
        }

        asoc->strreset_outstanding = out + in;

out:
        return retval;
}

int sctp_send_reset_assoc(struct sctp_association *asoc)
{
        struct sctp_stream *stream = &asoc->stream;
        struct sctp_chunk *chunk = NULL;
        int retval;
        __u16 i;

        if (!asoc->peer.reconf_capable ||
            !(asoc->strreset_enable & SCTP_ENABLE_RESET_ASSOC_REQ))
                return -ENOPROTOOPT;

        if (asoc->strreset_outstanding)
                return -EINPROGRESS;

        if (!sctp_outq_is_empty(&asoc->outqueue))
                return -EAGAIN;

        chunk = sctp_make_strreset_tsnreq(asoc);
        if (!chunk)
                return -ENOMEM;

        /* Block further xmit of data until this request is completed */
        for (i = 0; i < stream->outcnt; i++)
                SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED;

        asoc->strreset_chunk = chunk;
        sctp_chunk_hold(asoc->strreset_chunk);

        retval = sctp_send_reconf(asoc, chunk);
        if (retval) {
                sctp_chunk_put(asoc->strreset_chunk);
                asoc->strreset_chunk = NULL;

                for (i = 0; i < stream->outcnt; i++)
                        SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;

                return retval;
        }

        asoc->strreset_outstanding = 1;

        return 0;
}

int sctp_send_add_streams(struct sctp_association *asoc,
                          struct sctp_add_streams *params)
{
        struct sctp_stream *stream = &asoc->stream;
        struct sctp_chunk *chunk = NULL;
        int retval;
        __u32 outcnt, incnt;
        __u16 out, in;

        if (!asoc->peer.reconf_capable ||
            !(asoc->strreset_enable & SCTP_ENABLE_CHANGE_ASSOC_REQ)) {
                retval = -ENOPROTOOPT;
                goto out;
        }

        if (asoc->strreset_outstanding) {
                retval = -EINPROGRESS;
                goto out;
        }

        out = params->sas_outstrms;
        in  = params->sas_instrms;
        outcnt = stream->outcnt + out;
        incnt = stream->incnt + in;
        if (outcnt > SCTP_MAX_STREAM || incnt > SCTP_MAX_STREAM ||
            (!out && !in)) {
                retval = -EINVAL;
                goto out;
        }

        if (out) {
                retval = sctp_stream_alloc_out(stream, outcnt, GFP_KERNEL);
                if (retval)
                        goto out;
        }

        chunk = sctp_make_strreset_addstrm(asoc, out, in);
        if (!chunk) {
                retval = -ENOMEM;
                goto out;
        }

        asoc->strreset_chunk = chunk;
        sctp_chunk_hold(asoc->strreset_chunk);

        retval = sctp_send_reconf(asoc, chunk);
        if (retval) {
                sctp_chunk_put(asoc->strreset_chunk);
                asoc->strreset_chunk = NULL;
                goto out;
        }

        asoc->strreset_outstanding = !!out + !!in;

out:
        return retval;
}

static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param(
                        struct sctp_association *asoc, __be32 resp_seq,
                        __be16 type)
{
        struct sctp_chunk *chunk = asoc->strreset_chunk;
        struct sctp_reconf_chunk *hdr;
        union sctp_params param;

        if (!chunk)
                return NULL;

        hdr = (struct sctp_reconf_chunk *)chunk->chunk_hdr;
        sctp_walk_params(param, hdr) {
                /* sctp_strreset_tsnreq is actually the basic structure
                 * of all stream reconf params, so it's safe to use it
                 * to access request_seq.
                 */
                struct sctp_strreset_tsnreq *req = param.v;

                if ((!resp_seq || req->request_seq == resp_seq) &&
                    (!type || type == req->param_hdr.type))
                        return param.v;
        }

        return NULL;
}

static void sctp_update_strreset_result(struct sctp_association *asoc,
                                        __u32 result)
{
        asoc->strreset_result[1] = asoc->strreset_result[0];
        asoc->strreset_result[0] = result;
}

struct sctp_chunk *sctp_process_strreset_outreq(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        struct sctp_strreset_outreq *outreq = param.v;
        struct sctp_stream *stream = &asoc->stream;
        __u32 result = SCTP_STRRESET_DENIED;
        __be16 *str_p = NULL;
        __u32 request_seq;
        __u16 i, nums;

        request_seq = ntohl(outreq->request_seq);

        if (ntohl(outreq->send_reset_at_tsn) >
            sctp_tsnmap_get_ctsn(&asoc->peer.tsn_map)) {
                result = SCTP_STRRESET_IN_PROGRESS;
                goto err;
        }

        if (TSN_lt(asoc->strreset_inseq, request_seq) ||
            TSN_lt(request_seq, asoc->strreset_inseq - 2)) {
                result = SCTP_STRRESET_ERR_BAD_SEQNO;
                goto err;
        } else if (TSN_lt(request_seq, asoc->strreset_inseq)) {
                i = asoc->strreset_inseq - request_seq - 1;
                result = asoc->strreset_result[i];
                goto err;
        }
        asoc->strreset_inseq++;

        /* Check strreset_enable after inseq inc, as sender cannot tell
         * the peer doesn't enable strreset after receiving response with
         * result denied, as well as to keep consistent with bsd.
         */
        if (!(asoc->strreset_enable & SCTP_ENABLE_RESET_STREAM_REQ))
                goto out;

        nums = (ntohs(param.p->length) - sizeof(*outreq)) / sizeof(__u16);
        str_p = outreq->list_of_streams;
        for (i = 0; i < nums; i++) {
                if (ntohs(str_p[i]) >= stream->incnt) {
                        result = SCTP_STRRESET_ERR_WRONG_SSN;
                        goto out;
                }
        }

        if (asoc->strreset_chunk) {
                if (!sctp_chunk_lookup_strreset_param(
                                asoc, outreq->response_seq,
                                SCTP_PARAM_RESET_IN_REQUEST)) {
                        /* same process with outstanding isn't 0 */
                        result = SCTP_STRRESET_ERR_IN_PROGRESS;
                        goto out;
                }

                asoc->strreset_outstanding--;
                asoc->strreset_outseq++;

                if (!asoc->strreset_outstanding) {
                        struct sctp_transport *t;

                        t = asoc->strreset_chunk->transport;
                        if (del_timer(&t->reconf_timer))
                                sctp_transport_put(t);

                        sctp_chunk_put(asoc->strreset_chunk);
                        asoc->strreset_chunk = NULL;
                }
        }

        if (nums)
                for (i = 0; i < nums; i++)
                        SCTP_SI(stream, ntohs(str_p[i]))->mid = 0;
        else
                for (i = 0; i < stream->incnt; i++)
                        SCTP_SI(stream, i)->mid = 0;

        result = SCTP_STRRESET_PERFORMED;

        *evp = sctp_ulpevent_make_stream_reset_event(asoc,
                SCTP_STREAM_RESET_INCOMING_SSN, nums, str_p, GFP_ATOMIC);

out:
        sctp_update_strreset_result(asoc, result);
err:
        return sctp_make_strreset_resp(asoc, result, request_seq);
}

struct sctp_chunk *sctp_process_strreset_inreq(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        struct sctp_strreset_inreq *inreq = param.v;
        struct sctp_stream *stream = &asoc->stream;
        __u32 result = SCTP_STRRESET_DENIED;
        struct sctp_chunk *chunk = NULL;
        __u32 request_seq;
        __u16 i, nums;
        __be16 *str_p;

        request_seq = ntohl(inreq->request_seq);
        if (TSN_lt(asoc->strreset_inseq, request_seq) ||
            TSN_lt(request_seq, asoc->strreset_inseq - 2)) {
                result = SCTP_STRRESET_ERR_BAD_SEQNO;
                goto err;
        } else if (TSN_lt(request_seq, asoc->strreset_inseq)) {
                i = asoc->strreset_inseq - request_seq - 1;
                result = asoc->strreset_result[i];
                if (result == SCTP_STRRESET_PERFORMED)
                        return NULL;
                goto err;
        }
        asoc->strreset_inseq++;

        if (!(asoc->strreset_enable & SCTP_ENABLE_RESET_STREAM_REQ))
                goto out;

        if (asoc->strreset_outstanding) {
                result = SCTP_STRRESET_ERR_IN_PROGRESS;
                goto out;
        }

        nums = (ntohs(param.p->length) - sizeof(*inreq)) / sizeof(__u16);
        str_p = inreq->list_of_streams;
        for (i = 0; i < nums; i++) {
                if (ntohs(str_p[i]) >= stream->outcnt) {
                        result = SCTP_STRRESET_ERR_WRONG_SSN;
                        goto out;
                }
        }

        if (!sctp_stream_outq_is_empty(stream, nums, str_p)) {
                result = SCTP_STRRESET_IN_PROGRESS;
                asoc->strreset_inseq--;
                goto err;
        }

        chunk = sctp_make_strreset_req(asoc, nums, str_p, 1, 0);
        if (!chunk)
                goto out;

        if (nums)
                for (i = 0; i < nums; i++)
                        SCTP_SO(stream, ntohs(str_p[i]))->state =
                                               SCTP_STREAM_CLOSED;
        else
                for (i = 0; i < stream->outcnt; i++)
                        SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED;

        asoc->strreset_chunk = chunk;
        asoc->strreset_outstanding = 1;
        sctp_chunk_hold(asoc->strreset_chunk);

        result = SCTP_STRRESET_PERFORMED;

out:
        sctp_update_strreset_result(asoc, result);
err:
        if (!chunk)
                chunk =  sctp_make_strreset_resp(asoc, result, request_seq);

        return chunk;
}

struct sctp_chunk *sctp_process_strreset_tsnreq(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        __u32 init_tsn = 0, next_tsn = 0, max_tsn_seen;
        struct sctp_strreset_tsnreq *tsnreq = param.v;
        struct sctp_stream *stream = &asoc->stream;
        __u32 result = SCTP_STRRESET_DENIED;
        __u32 request_seq;
        __u16 i;

        request_seq = ntohl(tsnreq->request_seq);
        if (TSN_lt(asoc->strreset_inseq, request_seq) ||
            TSN_lt(request_seq, asoc->strreset_inseq - 2)) {
                result = SCTP_STRRESET_ERR_BAD_SEQNO;
                goto err;
        } else if (TSN_lt(request_seq, asoc->strreset_inseq)) {
                i = asoc->strreset_inseq - request_seq - 1;
                result = asoc->strreset_result[i];
                if (result == SCTP_STRRESET_PERFORMED) {
                        next_tsn = asoc->ctsn_ack_point + 1;
                        init_tsn =
                                sctp_tsnmap_get_ctsn(&asoc->peer.tsn_map) + 1;
                }
                goto err;
        }

        if (!sctp_outq_is_empty(&asoc->outqueue)) {
                result = SCTP_STRRESET_IN_PROGRESS;
                goto err;
        }

        asoc->strreset_inseq++;

        if (!(asoc->strreset_enable & SCTP_ENABLE_RESET_ASSOC_REQ))
                goto out;

        if (asoc->strreset_outstanding) {
                result = SCTP_STRRESET_ERR_IN_PROGRESS;
                goto out;
        }

        /* G4: The same processing as though a FWD-TSN chunk (as defined in
         *     [RFC3758]) with all streams affected and a new cumulative TSN
         *     ACK of the Receiver's Next TSN minus 1 were received MUST be
         *     performed.
         */
        max_tsn_seen = sctp_tsnmap_get_max_tsn_seen(&asoc->peer.tsn_map);
        asoc->stream.si->report_ftsn(&asoc->ulpq, max_tsn_seen);

        /* G1: Compute an appropriate value for the Receiver's Next TSN -- the
         *     TSN that the peer should use to send the next DATA chunk.  The
         *     value SHOULD be the smallest TSN not acknowledged by the
         *     receiver of the request plus 2^31.
         */
        init_tsn = sctp_tsnmap_get_ctsn(&asoc->peer.tsn_map) + (1 << 31);
        sctp_tsnmap_init(&asoc->peer.tsn_map, SCTP_TSN_MAP_INITIAL,
                         init_tsn, GFP_ATOMIC);

        /* G3: The same processing as though a SACK chunk with no gap report
         *     and a cumulative TSN ACK of the Sender's Next TSN minus 1 were
         *     received MUST be performed.
         */
        sctp_outq_free(&asoc->outqueue);

        /* G2: Compute an appropriate value for the local endpoint's next TSN,
         *     i.e., the next TSN assigned by the receiver of the SSN/TSN reset
         *     chunk.  The value SHOULD be the highest TSN sent by the receiver
         *     of the request plus 1.
         */
        next_tsn = asoc->next_tsn;
        asoc->ctsn_ack_point = next_tsn - 1;
        asoc->adv_peer_ack_point = asoc->ctsn_ack_point;

        /* G5:  The next expected and outgoing SSNs MUST be reset to 0 for all
         *      incoming and outgoing streams.
         */
        for (i = 0; i < stream->outcnt; i++) {
                SCTP_SO(stream, i)->mid = 0;
                SCTP_SO(stream, i)->mid_uo = 0;
        }
        for (i = 0; i < stream->incnt; i++)
                SCTP_SI(stream, i)->mid = 0;

        result = SCTP_STRRESET_PERFORMED;

        *evp = sctp_ulpevent_make_assoc_reset_event(asoc, 0, init_tsn,
                                                    next_tsn, GFP_ATOMIC);

out:
        sctp_update_strreset_result(asoc, result);
err:
        return sctp_make_strreset_tsnresp(asoc, result, request_seq,
                                          next_tsn, init_tsn);
}

struct sctp_chunk *sctp_process_strreset_addstrm_out(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        struct sctp_strreset_addstrm *addstrm = param.v;
        struct sctp_stream *stream = &asoc->stream;
        __u32 result = SCTP_STRRESET_DENIED;
        __u32 request_seq, incnt;
        __u16 in, i;

        request_seq = ntohl(addstrm->request_seq);
        if (TSN_lt(asoc->strreset_inseq, request_seq) ||
            TSN_lt(request_seq, asoc->strreset_inseq - 2)) {
                result = SCTP_STRRESET_ERR_BAD_SEQNO;
                goto err;
        } else if (TSN_lt(request_seq, asoc->strreset_inseq)) {
                i = asoc->strreset_inseq - request_seq - 1;
                result = asoc->strreset_result[i];
                goto err;
        }
        asoc->strreset_inseq++;

        if (!(asoc->strreset_enable & SCTP_ENABLE_CHANGE_ASSOC_REQ))
                goto out;

        in = ntohs(addstrm->number_of_streams);
        incnt = stream->incnt + in;
        if (!in || incnt > SCTP_MAX_STREAM)
                goto out;

        if (sctp_stream_alloc_in(stream, incnt, GFP_ATOMIC))
                goto out;

        if (asoc->strreset_chunk) {
                if (!sctp_chunk_lookup_strreset_param(
                        asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS)) {
                        /* same process with outstanding isn't 0 */
                        result = SCTP_STRRESET_ERR_IN_PROGRESS;
                        goto out;
                }

                asoc->strreset_outstanding--;
                asoc->strreset_outseq++;

                if (!asoc->strreset_outstanding) {
                        struct sctp_transport *t;

                        t = asoc->strreset_chunk->transport;
                        if (del_timer(&t->reconf_timer))
                                sctp_transport_put(t);

                        sctp_chunk_put(asoc->strreset_chunk);
                        asoc->strreset_chunk = NULL;
                }
        }

        stream->incnt = incnt;

        result = SCTP_STRRESET_PERFORMED;

        *evp = sctp_ulpevent_make_stream_change_event(asoc,
                0, ntohs(addstrm->number_of_streams), 0, GFP_ATOMIC);

out:
        sctp_update_strreset_result(asoc, result);
err:
        return sctp_make_strreset_resp(asoc, result, request_seq);
}

struct sctp_chunk *sctp_process_strreset_addstrm_in(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        struct sctp_strreset_addstrm *addstrm = param.v;
        struct sctp_stream *stream = &asoc->stream;
        __u32 result = SCTP_STRRESET_DENIED;
        struct sctp_chunk *chunk = NULL;
        __u32 request_seq, outcnt;
        __u16 out, i;
        int ret;

        request_seq = ntohl(addstrm->request_seq);
        if (TSN_lt(asoc->strreset_inseq, request_seq) ||
            TSN_lt(request_seq, asoc->strreset_inseq - 2)) {
                result = SCTP_STRRESET_ERR_BAD_SEQNO;
                goto err;
        } else if (TSN_lt(request_seq, asoc->strreset_inseq)) {
                i = asoc->strreset_inseq - request_seq - 1;
                result = asoc->strreset_result[i];
                if (result == SCTP_STRRESET_PERFORMED)
                        return NULL;
                goto err;
        }
        asoc->strreset_inseq++;

        if (!(asoc->strreset_enable & SCTP_ENABLE_CHANGE_ASSOC_REQ))
                goto out;

        if (asoc->strreset_outstanding) {
                result = SCTP_STRRESET_ERR_IN_PROGRESS;
                goto out;
        }

        out = ntohs(addstrm->number_of_streams);
        outcnt = stream->outcnt + out;
        if (!out || outcnt > SCTP_MAX_STREAM)
                goto out;

        ret = sctp_stream_alloc_out(stream, outcnt, GFP_ATOMIC);
        if (ret)
                goto out;

        chunk = sctp_make_strreset_addstrm(asoc, out, 0);
        if (!chunk)
                goto out;

        asoc->strreset_chunk = chunk;
        asoc->strreset_outstanding = 1;
        sctp_chunk_hold(asoc->strreset_chunk);

        stream->outcnt = outcnt;

        result = SCTP_STRRESET_PERFORMED;

out:
        sctp_update_strreset_result(asoc, result);
err:
        if (!chunk)
                chunk = sctp_make_strreset_resp(asoc, result, request_seq);

        return chunk;
}

struct sctp_chunk *sctp_process_strreset_resp(
                                struct sctp_association *asoc,
                                union sctp_params param,
                                struct sctp_ulpevent **evp)
{
        struct sctp_stream *stream = &asoc->stream;
        struct sctp_strreset_resp *resp = param.v;
        struct sctp_transport *t;
        __u16 i, nums, flags = 0;
        struct sctp_paramhdr *req;
        __u32 result;

        req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0);
        if (!req)
                return NULL;

        result = ntohl(resp->result);
        if (result != SCTP_STRRESET_PERFORMED) {
                /* if in progress, do nothing but retransmit */
                if (result == SCTP_STRRESET_IN_PROGRESS)
                        return NULL;
                else if (result == SCTP_STRRESET_DENIED)
                        flags = SCTP_STREAM_RESET_DENIED;
                else
                        flags = SCTP_STREAM_RESET_FAILED;
        }

        if (req->type == SCTP_PARAM_RESET_OUT_REQUEST) {
                struct sctp_strreset_outreq *outreq;
                __be16 *str_p;

                outreq = (struct sctp_strreset_outreq *)req;
                str_p = outreq->list_of_streams;
                nums = (ntohs(outreq->param_hdr.length) - sizeof(*outreq)) /
                       sizeof(__u16);

                if (result == SCTP_STRRESET_PERFORMED) {
                        struct sctp_stream_out *sout;
                        if (nums) {
                                for (i = 0; i < nums; i++) {
                                        sout = SCTP_SO(stream, ntohs(str_p[i]));
                                        sout->mid = 0;
                                        sout->mid_uo = 0;
                                }
                        } else {
                                for (i = 0; i < stream->outcnt; i++) {
                                        sout = SCTP_SO(stream, i);
                                        sout->mid = 0;
                                        sout->mid_uo = 0;
                                }
                        }
                }

                flags |= SCTP_STREAM_RESET_OUTGOING_SSN;

                for (i = 0; i < stream->outcnt; i++)
                        SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;

                *evp = sctp_ulpevent_make_stream_reset_event(asoc, flags,
                        nums, str_p, GFP_ATOMIC);
        } else if (req->type == SCTP_PARAM_RESET_IN_REQUEST) {
                struct sctp_strreset_inreq *inreq;
                __be16 *str_p;

                /* if the result is performed, it's impossible for inreq */
                if (result == SCTP_STRRESET_PERFORMED)
                        return NULL;

                inreq = (struct sctp_strreset_inreq *)req;
                str_p = inreq->list_of_streams;
                nums = (ntohs(inreq->param_hdr.length) - sizeof(*inreq)) /
                       sizeof(__u16);

                flags |= SCTP_STREAM_RESET_INCOMING_SSN;

                *evp = sctp_ulpevent_make_stream_reset_event(asoc, flags,
                        nums, str_p, GFP_ATOMIC);
        } else if (req->type == SCTP_PARAM_RESET_TSN_REQUEST) {
                struct sctp_strreset_resptsn *resptsn;
                __u32 stsn, rtsn;

                /* check for resptsn, as sctp_verify_reconf didn't do it*/
                if (ntohs(param.p->length) != sizeof(*resptsn))
                        return NULL;

                resptsn = (struct sctp_strreset_resptsn *)resp;
                stsn = ntohl(resptsn->senders_next_tsn);
                rtsn = ntohl(resptsn->receivers_next_tsn);

                if (result == SCTP_STRRESET_PERFORMED) {
                        __u32 mtsn = sctp_tsnmap_get_max_tsn_seen(
                                                &asoc->peer.tsn_map);
                        LIST_HEAD(temp);

                        asoc->stream.si->report_ftsn(&asoc->ulpq, mtsn);

                        sctp_tsnmap_init(&asoc->peer.tsn_map,
                                         SCTP_TSN_MAP_INITIAL,
                                         stsn, GFP_ATOMIC);

                        /* Clean up sacked and abandoned queues only. As the
                         * out_chunk_list may not be empty, splice it to temp,
                         * then get it back after sctp_outq_free is done.
                         */
                        list_splice_init(&asoc->outqueue.out_chunk_list, &temp);
                        sctp_outq_free(&asoc->outqueue);
                        list_splice_init(&temp, &asoc->outqueue.out_chunk_list);

                        asoc->next_tsn = rtsn;
                        asoc->ctsn_ack_point = asoc->next_tsn - 1;
                        asoc->adv_peer_ack_point = asoc->ctsn_ack_point;

                        for (i = 0; i < stream->outcnt; i++) {
                                SCTP_SO(stream, i)->mid = 0;
                                SCTP_SO(stream, i)->mid_uo = 0;
                        }
                        for (i = 0; i < stream->incnt; i++)
                                SCTP_SI(stream, i)->mid = 0;
                }

                for (i = 0; i < stream->outcnt; i++)
                        SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;

                *evp = sctp_ulpevent_make_assoc_reset_event(asoc, flags,
                        stsn, rtsn, GFP_ATOMIC);
        } else if (req->type == SCTP_PARAM_RESET_ADD_OUT_STREAMS) {
                struct sctp_strreset_addstrm *addstrm;
                __u16 number;

                addstrm = (struct sctp_strreset_addstrm *)req;
                nums = ntohs(addstrm->number_of_streams);
                number = stream->outcnt - nums;

                if (result == SCTP_STRRESET_PERFORMED) {
                        for (i = number; i < stream->outcnt; i++)
                                SCTP_SO(stream, i)->state = SCTP_STREAM_OPEN;
                } else {
                        sctp_stream_shrink_out(stream, number);
                        stream->outcnt = number;
                }

                *evp = sctp_ulpevent_make_stream_change_event(asoc, flags,
                        0, nums, GFP_ATOMIC);
        } else if (req->type == SCTP_PARAM_RESET_ADD_IN_STREAMS) {
                struct sctp_strreset_addstrm *addstrm;

                /* if the result is performed, it's impossible for addstrm in
                 * request.
                 */
                if (result == SCTP_STRRESET_PERFORMED)
                        return NULL;

                addstrm = (struct sctp_strreset_addstrm *)req;
                nums = ntohs(addstrm->number_of_streams);

                *evp = sctp_ulpevent_make_stream_change_event(asoc, flags,
                        nums, 0, GFP_ATOMIC);
        }

        asoc->strreset_outstanding--;
        asoc->strreset_outseq++;

        /* remove everything for this reconf request */
        if (!asoc->strreset_outstanding) {
                t = asoc->strreset_chunk->transport;
                if (del_timer(&t->reconf_timer))
                        sctp_transport_put(t);

                sctp_chunk_put(asoc->strreset_chunk);
                asoc->strreset_chunk = NULL;
        }

        return NULL;
}

















































   14 
   12 





























    8 



    8 
    8 




    3 
    3 


























    6 









    6 
















































































































































































































































































    3 

























   21 

   21 

   18 

   15 

   13 
   12 








   10 





   14 














    1 





    1 





   10 





    9 

    1 




    8 
    1 




    7 

    8 




    8 









    8 
    6 
    8 









    8 


   10 

   12 






    3 


    7 


































    3 

    7 








    1 

















    3 





    2 




    1 



    3 






    5 





    4 



    2 







    2 







    4 




    5 







    1 



    1 











    6 









    7 

    4 



    3 


    2 



    2 
    2 










    1 









    8 











    1 

    1 

















    7 









    6 

    5 


    4 


    3 


    1 


    1 












    1 

    1 




    4 
    7 





    3 




    1 


    2 
    1 




    1 








































    3 










    3 










    3 




    3 













    2 

    2 


    2 
    1 








    1 








































































































































































































    6 












    6 


    3 



    3 






    2 



    2 


    2 







    1 

















    1 





    3 












    4 












   12 












    9 


    7 






    9 


    8 




    7 




    7 




    7 
    5 




    6 







    1 
    1 






    6 


   12 
























    1 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2010-2011 EIA Electronics,
//                         Pieter Beyens <pieter.beyens@eia.be>
// Copyright (c) 2010-2011 EIA Electronics,
//                         Kurt Van Dijck <kurt.van.dijck@eia.be>
// Copyright (c) 2018 Protonic,
//                         Robin van der Gracht <robin@protonic.nl>
// Copyright (c) 2017-2019 Pengutronix,
//                         Marc Kleine-Budde <kernel@pengutronix.de>
// Copyright (c) 2017-2019 Pengutronix,
//                         Oleksij Rempel <kernel@pengutronix.de>

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/can/can-ml.h>
#include <linux/can/core.h>
#include <linux/can/skb.h>
#include <linux/errqueue.h>
#include <linux/if_arp.h>

#include "j1939-priv.h"

#define J1939_MIN_NAMELEN CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.j1939)

/* conversion function between struct sock::sk_priority from linux and
 * j1939 priority field
 */
static inline priority_t j1939_prio(u32 sk_priority)
{
        sk_priority = min(sk_priority, 7U);

        return 7 - sk_priority;
}

static inline u32 j1939_to_sk_priority(priority_t prio)
{
        return 7 - prio;
}

/* function to see if pgn is to be evaluated */
static inline bool j1939_pgn_is_valid(pgn_t pgn)
{
        return pgn <= J1939_PGN_MAX;
}

/* test function to avoid non-zero DA placeholder for pdu1 pgn's */
static inline bool j1939_pgn_is_clean_pdu(pgn_t pgn)
{
        if (j1939_pgn_is_pdu1(pgn))
                return !(pgn & 0xff);
        else
                return true;
}

static inline void j1939_sock_pending_add(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        atomic_inc(&jsk->skb_pending);
}

static int j1939_sock_pending_get(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        return atomic_read(&jsk->skb_pending);
}

void j1939_sock_pending_del(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* atomic_dec_return returns the new value */
        if (!atomic_dec_return(&jsk->skb_pending))
                wake_up(&jsk->waitq);        /* no pending SKB's */
}

static void j1939_jsk_add(struct j1939_priv *priv, struct j1939_sock *jsk)
{
        jsk->state |= J1939_SOCK_BOUND;
        j1939_priv_get(priv);

        write_lock_bh(&priv->j1939_socks_lock);
        list_add_tail(&jsk->list, &priv->j1939_socks);
        write_unlock_bh(&priv->j1939_socks_lock);
}

static void j1939_jsk_del(struct j1939_priv *priv, struct j1939_sock *jsk)
{
        write_lock_bh(&priv->j1939_socks_lock);
        list_del_init(&jsk->list);
        write_unlock_bh(&priv->j1939_socks_lock);

        j1939_priv_put(priv);
        jsk->state &= ~J1939_SOCK_BOUND;
}

static bool j1939_sk_queue_session(struct j1939_session *session)
{
        struct j1939_sock *jsk = j1939_sk(session->sk);
        bool empty;

        spin_lock_bh(&jsk->sk_session_queue_lock);
        empty = list_empty(&jsk->sk_session_queue);
        j1939_session_get(session);
        list_add_tail(&session->sk_session_queue_entry, &jsk->sk_session_queue);
        spin_unlock_bh(&jsk->sk_session_queue_lock);
        j1939_sock_pending_add(&jsk->sk);

        return empty;
}

static struct
j1939_session *j1939_sk_get_incomplete_session(struct j1939_sock *jsk)
{
        struct j1939_session *session = NULL;

        spin_lock_bh(&jsk->sk_session_queue_lock);
        if (!list_empty(&jsk->sk_session_queue)) {
                session = list_last_entry(&jsk->sk_session_queue,
                                          struct j1939_session,
                                          sk_session_queue_entry);
                if (session->total_queued_size == session->total_message_size)
                        session = NULL;
                else
                        j1939_session_get(session);
        }
        spin_unlock_bh(&jsk->sk_session_queue_lock);

        return session;
}

static void j1939_sk_queue_drop_all(struct j1939_priv *priv,
                                    struct j1939_sock *jsk, int err)
{
        struct j1939_session *session, *tmp;

        netdev_dbg(priv->ndev, "%s: err: %i\n", __func__, err);
        spin_lock_bh(&jsk->sk_session_queue_lock);
        list_for_each_entry_safe(session, tmp, &jsk->sk_session_queue,
                                 sk_session_queue_entry) {
                list_del_init(&session->sk_session_queue_entry);
                session->err = err;
                j1939_session_put(session);
        }
        spin_unlock_bh(&jsk->sk_session_queue_lock);
}

static void j1939_sk_queue_activate_next_locked(struct j1939_session *session)
{
        struct j1939_sock *jsk;
        struct j1939_session *first;
        int err;

        /* RX-Session don't have a socket (yet) */
        if (!session->sk)
                return;

        jsk = j1939_sk(session->sk);
        lockdep_assert_held(&jsk->sk_session_queue_lock);

        err = session->err;

        first = list_first_entry_or_null(&jsk->sk_session_queue,
                                         struct j1939_session,
                                         sk_session_queue_entry);

        /* Some else has already activated the next session */
        if (first != session)
                return;

activate_next:
        list_del_init(&first->sk_session_queue_entry);
        j1939_session_put(first);
        first = list_first_entry_or_null(&jsk->sk_session_queue,
                                         struct j1939_session,
                                         sk_session_queue_entry);
        if (!first)
                return;

        if (j1939_session_activate(first)) {
                netdev_warn_once(first->priv->ndev,
                                 "%s: 0x%p: Identical session is already activated.\n",
                                 __func__, first);
                first->err = -EBUSY;
                goto activate_next;
        } else {
                /* Give receiver some time (arbitrary chosen) to recover */
                int time_ms = 0;

                if (err)
                        time_ms = 10 + get_random_u32_below(16);

                j1939_tp_schedule_txtimer(first, time_ms);
        }
}

void j1939_sk_queue_activate_next(struct j1939_session *session)
{
        struct j1939_sock *jsk;

        if (!session->sk)
                return;

        jsk = j1939_sk(session->sk);

        spin_lock_bh(&jsk->sk_session_queue_lock);
        j1939_sk_queue_activate_next_locked(session);
        spin_unlock_bh(&jsk->sk_session_queue_lock);
}

static bool j1939_sk_match_dst(struct j1939_sock *jsk,
                               const struct j1939_sk_buff_cb *skcb)
{
        if ((jsk->state & J1939_SOCK_PROMISC))
                return true;

        /* Destination address filter */
        if (jsk->addr.src_name && skcb->addr.dst_name) {
                if (jsk->addr.src_name != skcb->addr.dst_name)
                        return false;
        } else {
                /* receive (all sockets) if
                 * - all packages that match our bind() address
                 * - all broadcast on a socket if SO_BROADCAST
                 *   is set
                 */
                if (j1939_address_is_unicast(skcb->addr.da)) {
                        if (jsk->addr.sa != skcb->addr.da)
                                return false;
                } else if (!sock_flag(&jsk->sk, SOCK_BROADCAST)) {
                        /* receiving broadcast without SO_BROADCAST
                         * flag is not allowed
                         */
                        return false;
                }
        }

        /* Source address filter */
        if (jsk->state & J1939_SOCK_CONNECTED) {
                /* receive (all sockets) if
                 * - all packages that match our connect() name or address
                 */
                if (jsk->addr.dst_name && skcb->addr.src_name) {
                        if (jsk->addr.dst_name != skcb->addr.src_name)
                                return false;
                } else {
                        if (jsk->addr.da != skcb->addr.sa)
                                return false;
                }
        }

        /* PGN filter */
        if (j1939_pgn_is_valid(jsk->pgn_rx_filter) &&
            jsk->pgn_rx_filter != skcb->addr.pgn)
                return false;

        return true;
}

/* matches skb control buffer (addr) with a j1939 filter */
static bool j1939_sk_match_filter(struct j1939_sock *jsk,
                                  const struct j1939_sk_buff_cb *skcb)
{
        const struct j1939_filter *f;
        int nfilter;

        spin_lock_bh(&jsk->filters_lock);

        f = jsk->filters;
        nfilter = jsk->nfilters;

        if (!nfilter)
                /* receive all when no filters are assigned */
                goto filter_match_found;

        for (; nfilter; ++f, --nfilter) {
                if ((skcb->addr.pgn & f->pgn_mask) != f->pgn)
                        continue;
                if ((skcb->addr.sa & f->addr_mask) != f->addr)
                        continue;
                if ((skcb->addr.src_name & f->name_mask) != f->name)
                        continue;
                goto filter_match_found;
        }

        spin_unlock_bh(&jsk->filters_lock);
        return false;

filter_match_found:
        spin_unlock_bh(&jsk->filters_lock);
        return true;
}

static bool j1939_sk_recv_match_one(struct j1939_sock *jsk,
                                    const struct j1939_sk_buff_cb *skcb)
{
        if (!(jsk->state & J1939_SOCK_BOUND))
                return false;

        if (!j1939_sk_match_dst(jsk, skcb))
                return false;

        if (!j1939_sk_match_filter(jsk, skcb))
                return false;

        return true;
}

static void j1939_sk_recv_one(struct j1939_sock *jsk, struct sk_buff *oskb)
{
        const struct j1939_sk_buff_cb *oskcb = j1939_skb_to_cb(oskb);
        struct j1939_sk_buff_cb *skcb;
        struct sk_buff *skb;

        if (oskb->sk == &jsk->sk)
                return;

        if (!j1939_sk_recv_match_one(jsk, oskcb))
                return;

        skb = skb_clone(oskb, GFP_ATOMIC);
        if (!skb) {
                pr_warn("skb clone failed\n");
                return;
        }
        can_skb_set_owner(skb, oskb->sk);

        skcb = j1939_skb_to_cb(skb);
        skcb->msg_flags &= ~(MSG_DONTROUTE);
        if (skb->sk)
                skcb->msg_flags |= MSG_DONTROUTE;

        if (sock_queue_rcv_skb(&jsk->sk, skb) < 0)
                kfree_skb(skb);
}

bool j1939_sk_recv_match(struct j1939_priv *priv, struct j1939_sk_buff_cb *skcb)
{
        struct j1939_sock *jsk;
        bool match = false;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                match = j1939_sk_recv_match_one(jsk, skcb);
                if (match)
                        break;
        }
        read_unlock_bh(&priv->j1939_socks_lock);

        return match;
}

void j1939_sk_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sock *jsk;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                j1939_sk_recv_one(jsk, skb);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
}

static void j1939_sk_sock_destruct(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* This function will be called by the generic networking code, when
         * the socket is ultimately closed (sk->sk_destruct).
         *
         * The race between
         * - processing a received CAN frame
         *   (can_receive -> j1939_can_recv)
         *   and accessing j1939_priv
         * ... and ...
         * - closing a socket
         *   (j1939_can_rx_unregister -> can_rx_unregister)
         *   and calling the final j1939_priv_put()
         *
         * is avoided by calling the final j1939_priv_put() from this
         * RCU deferred cleanup call.
         */
        if (jsk->priv) {
                j1939_priv_put(jsk->priv);
                jsk->priv = NULL;
        }

        /* call generic CAN sock destruct */
        can_sock_destruct(sk);
}

static int j1939_sk_init(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* Ensure that "sk" is first member in "struct j1939_sock", so that we
         * can skip it during memset().
         */
        BUILD_BUG_ON(offsetof(struct j1939_sock, sk) != 0);
        memset((void *)jsk + sizeof(jsk->sk), 0x0,
               sizeof(*jsk) - sizeof(jsk->sk));

        INIT_LIST_HEAD(&jsk->list);
        init_waitqueue_head(&jsk->waitq);
        jsk->sk.sk_priority = j1939_to_sk_priority(6);
        jsk->sk.sk_reuse = 1; /* per default */
        jsk->addr.sa = J1939_NO_ADDR;
        jsk->addr.da = J1939_NO_ADDR;
        jsk->addr.pgn = J1939_NO_PGN;
        jsk->pgn_rx_filter = J1939_NO_PGN;
        atomic_set(&jsk->skb_pending, 0);
        spin_lock_init(&jsk->sk_session_queue_lock);
        INIT_LIST_HEAD(&jsk->sk_session_queue);
        spin_lock_init(&jsk->filters_lock);

        /* j1939_sk_sock_destruct() depends on SOCK_RCU_FREE flag */
        sock_set_flag(sk, SOCK_RCU_FREE);
        sk->sk_destruct = j1939_sk_sock_destruct;
        sk->sk_protocol = CAN_J1939;

        return 0;
}

static int j1939_sk_sanity_check(struct sockaddr_can *addr, int len)
{
        if (!addr)
                return -EDESTADDRREQ;
        if (len < J1939_MIN_NAMELEN)
                return -EINVAL;
        if (addr->can_family != AF_CAN)
                return -EINVAL;
        if (!addr->can_ifindex)
                return -ENODEV;
        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn) &&
            !j1939_pgn_is_clean_pdu(addr->can_addr.j1939.pgn))
                return -EINVAL;

        return 0;
}

static int j1939_sk_bind(struct socket *sock, struct sockaddr *uaddr, int len)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct j1939_sock *jsk = j1939_sk(sock->sk);
        struct j1939_priv *priv;
        struct sock *sk;
        struct net *net;
        int ret = 0;

        ret = j1939_sk_sanity_check(addr, len);
        if (ret)
                return ret;

        lock_sock(sock->sk);

        priv = jsk->priv;
        sk = sock->sk;
        net = sock_net(sk);

        /* Already bound to an interface? */
        if (jsk->state & J1939_SOCK_BOUND) {
                /* A re-bind() to a different interface is not
                 * supported.
                 */
                if (jsk->ifindex != addr->can_ifindex) {
                        ret = -EINVAL;
                        goto out_release_sock;
                }

                /* drop old references */
                j1939_jsk_del(priv, jsk);
                j1939_local_ecu_put(priv, jsk->addr.src_name, jsk->addr.sa);
        } else {
                struct can_ml_priv *can_ml;
                struct net_device *ndev;

                ndev = dev_get_by_index(net, addr->can_ifindex);
                if (!ndev) {
                        ret = -ENODEV;
                        goto out_release_sock;
                }

                can_ml = can_get_ml_priv(ndev);
                if (!can_ml) {
                        dev_put(ndev);
                        ret = -ENODEV;
                        goto out_release_sock;
                }

                if (!(ndev->flags & IFF_UP)) {
                        dev_put(ndev);
                        ret = -ENETDOWN;
                        goto out_release_sock;
                }

                priv = j1939_netdev_start(ndev);
                dev_put(ndev);
                if (IS_ERR(priv)) {
                        ret = PTR_ERR(priv);
                        goto out_release_sock;
                }

                jsk->ifindex = addr->can_ifindex;

                /* the corresponding j1939_priv_put() is called via
                 * sk->sk_destruct, which points to j1939_sk_sock_destruct()
                 */
                j1939_priv_get(priv);
                jsk->priv = priv;
        }

        /* set default transmit pgn */
        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                jsk->pgn_rx_filter = addr->can_addr.j1939.pgn;
        jsk->addr.src_name = addr->can_addr.j1939.name;
        jsk->addr.sa = addr->can_addr.j1939.addr;

        /* get new references */
        ret = j1939_local_ecu_get(priv, jsk->addr.src_name, jsk->addr.sa);
        if (ret) {
                j1939_netdev_stop(priv);
                goto out_release_sock;
        }

        j1939_jsk_add(priv, jsk);

 out_release_sock: /* fall through */
        release_sock(sock->sk);

        return ret;
}

static int j1939_sk_connect(struct socket *sock, struct sockaddr *uaddr,
                            int len, int flags)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct j1939_sock *jsk = j1939_sk(sock->sk);
        int ret = 0;

        ret = j1939_sk_sanity_check(addr, len);
        if (ret)
                return ret;

        lock_sock(sock->sk);

        /* bind() before connect() is mandatory */
        if (!(jsk->state & J1939_SOCK_BOUND)) {
                ret = -EINVAL;
                goto out_release_sock;
        }

        /* A connect() to a different interface is not supported. */
        if (jsk->ifindex != addr->can_ifindex) {
                ret = -EINVAL;
                goto out_release_sock;
        }

        if (!addr->can_addr.j1939.name &&
            addr->can_addr.j1939.addr == J1939_NO_ADDR &&
            !sock_flag(&jsk->sk, SOCK_BROADCAST)) {
                /* broadcast, but SO_BROADCAST not set */
                ret = -EACCES;
                goto out_release_sock;
        }

        jsk->addr.dst_name = addr->can_addr.j1939.name;
        jsk->addr.da = addr->can_addr.j1939.addr;

        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                jsk->addr.pgn = addr->can_addr.j1939.pgn;

        jsk->state |= J1939_SOCK_CONNECTED;

 out_release_sock: /* fall through */
        release_sock(sock->sk);

        return ret;
}

static void j1939_sk_sock2sockaddr_can(struct sockaddr_can *addr,
                                       const struct j1939_sock *jsk, int peer)
{
        /* There are two holes (2 bytes and 3 bytes) to clear to avoid
         * leaking kernel information to user space.
         */
        memset(addr, 0, J1939_MIN_NAMELEN);

        addr->can_family = AF_CAN;
        addr->can_ifindex = jsk->ifindex;
        addr->can_addr.j1939.pgn = jsk->addr.pgn;
        if (peer) {
                addr->can_addr.j1939.name = jsk->addr.dst_name;
                addr->can_addr.j1939.addr = jsk->addr.da;
        } else {
                addr->can_addr.j1939.name = jsk->addr.src_name;
                addr->can_addr.j1939.addr = jsk->addr.sa;
        }
}

static int j1939_sk_getname(struct socket *sock, struct sockaddr *uaddr,
                            int peer)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int ret = 0;

        lock_sock(sk);

        if (peer && !(jsk->state & J1939_SOCK_CONNECTED)) {
                ret = -EADDRNOTAVAIL;
                goto failure;
        }

        j1939_sk_sock2sockaddr_can(addr, jsk, peer);
        ret = J1939_MIN_NAMELEN;

 failure:
        release_sock(sk);

        return ret;
}

static int j1939_sk_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk;

        if (!sk)
                return 0;

        lock_sock(sk);
        jsk = j1939_sk(sk);

        if (jsk->state & J1939_SOCK_BOUND) {
                struct j1939_priv *priv = jsk->priv;

                if (wait_event_interruptible(jsk->waitq,
                                             !j1939_sock_pending_get(&jsk->sk))) {
                        j1939_cancel_active_session(priv, sk);
                        j1939_sk_queue_drop_all(priv, jsk, ESHUTDOWN);
                }

                j1939_jsk_del(priv, jsk);

                j1939_local_ecu_put(priv, jsk->addr.src_name,
                                    jsk->addr.sa);

                j1939_netdev_stop(priv);
        }

        kfree(jsk->filters);
        sock_orphan(sk);
        sock->sk = NULL;

        release_sock(sk);
        sock_put(sk);

        return 0;
}

static int j1939_sk_setsockopt_flag(struct j1939_sock *jsk, sockptr_t optval,
                                    unsigned int optlen, int flag)
{
        int tmp;

        if (optlen != sizeof(tmp))
                return -EINVAL;
        if (copy_from_sockptr(&tmp, optval, optlen))
                return -EFAULT;
        lock_sock(&jsk->sk);
        if (tmp)
                jsk->state |= flag;
        else
                jsk->state &= ~flag;
        release_sock(&jsk->sk);
        return tmp;
}

static int j1939_sk_setsockopt(struct socket *sock, int level, int optname,
                               sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int tmp, count = 0, ret = 0;
        struct j1939_filter *filters = NULL, *ofilters;

        if (level != SOL_CAN_J1939)
                return -EINVAL;

        switch (optname) {
        case SO_J1939_FILTER:
                if (!sockptr_is_null(optval) && optlen != 0) {
                        struct j1939_filter *f;
                        int c;

                        if (optlen % sizeof(*filters) != 0)
                                return -EINVAL;

                        if (optlen > J1939_FILTER_MAX *
                            sizeof(struct j1939_filter))
                                return -EINVAL;

                        count = optlen / sizeof(*filters);
                        filters = memdup_sockptr(optval, optlen);
                        if (IS_ERR(filters))
                                return PTR_ERR(filters);

                        for (f = filters, c = count; c; f++, c--) {
                                f->name &= f->name_mask;
                                f->pgn &= f->pgn_mask;
                                f->addr &= f->addr_mask;
                        }
                }

                lock_sock(&jsk->sk);
                spin_lock_bh(&jsk->filters_lock);
                ofilters = jsk->filters;
                jsk->filters = filters;
                jsk->nfilters = count;
                spin_unlock_bh(&jsk->filters_lock);
                release_sock(&jsk->sk);
                kfree(ofilters);
                return 0;
        case SO_J1939_PROMISC:
                return j1939_sk_setsockopt_flag(jsk, optval, optlen,
                                                J1939_SOCK_PROMISC);
        case SO_J1939_ERRQUEUE:
                ret = j1939_sk_setsockopt_flag(jsk, optval, optlen,
                                               J1939_SOCK_ERRQUEUE);
                if (ret < 0)
                        return ret;

                if (!(jsk->state & J1939_SOCK_ERRQUEUE))
                        skb_queue_purge(&sk->sk_error_queue);
                return ret;
        case SO_J1939_SEND_PRIO:
                if (optlen != sizeof(tmp))
                        return -EINVAL;
                if (copy_from_sockptr(&tmp, optval, optlen))
                        return -EFAULT;
                if (tmp < 0 || tmp > 7)
                        return -EDOM;
                if (tmp < 2 && !capable(CAP_NET_ADMIN))
                        return -EPERM;
                lock_sock(&jsk->sk);
                jsk->sk.sk_priority = j1939_to_sk_priority(tmp);
                release_sock(&jsk->sk);
                return 0;
        default:
                return -ENOPROTOOPT;
        }
}

static int j1939_sk_getsockopt(struct socket *sock, int level, int optname,
                               char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int ret, ulen;
        /* set defaults for using 'int' properties */
        int tmp = 0;
        int len = sizeof(tmp);
        void *val = &tmp;

        if (level != SOL_CAN_J1939)
                return -EINVAL;
        if (get_user(ulen, optlen))
                return -EFAULT;
        if (ulen < 0)
                return -EINVAL;

        lock_sock(&jsk->sk);
        switch (optname) {
        case SO_J1939_PROMISC:
                tmp = (jsk->state & J1939_SOCK_PROMISC) ? 1 : 0;
                break;
        case SO_J1939_ERRQUEUE:
                tmp = (jsk->state & J1939_SOCK_ERRQUEUE) ? 1 : 0;
                break;
        case SO_J1939_SEND_PRIO:
                tmp = j1939_prio(jsk->sk.sk_priority);
                break;
        default:
                ret = -ENOPROTOOPT;
                goto no_copy;
        }

        /* copy to user, based on 'len' & 'val'
         * but most sockopt's are 'int' properties, and have 'len' & 'val'
         * left unchanged, but instead modified 'tmp'
         */
        if (len > ulen)
                ret = -EFAULT;
        else if (put_user(len, optlen))
                ret = -EFAULT;
        else if (copy_to_user(optval, val, len))
                ret = -EFAULT;
        else
                ret = 0;
 no_copy:
        release_sock(&jsk->sk);
        return ret;
}

static int j1939_sk_recvmsg(struct socket *sock, struct msghdr *msg,
                            size_t size, int flags)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        struct j1939_sk_buff_cb *skcb;
        int ret = 0;

        if (flags & ~(MSG_DONTWAIT | MSG_ERRQUEUE | MSG_CMSG_COMPAT))
                return -EINVAL;

        if (flags & MSG_ERRQUEUE)
                return sock_recv_errqueue(sock->sk, msg, size, SOL_CAN_J1939,
                                          SCM_J1939_ERRQUEUE);

        skb = skb_recv_datagram(sk, flags, &ret);
        if (!skb)
                return ret;

        if (size < skb->len)
                msg->msg_flags |= MSG_TRUNC;
        else
                size = skb->len;

        ret = memcpy_to_msg(msg, skb->data, size);
        if (ret < 0) {
                skb_free_datagram(sk, skb);
                return ret;
        }

        skcb = j1939_skb_to_cb(skb);
        if (j1939_address_is_valid(skcb->addr.da))
                put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_DEST_ADDR,
                         sizeof(skcb->addr.da), &skcb->addr.da);

        if (skcb->addr.dst_name)
                put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_DEST_NAME,
                         sizeof(skcb->addr.dst_name), &skcb->addr.dst_name);

        put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_PRIO,
                 sizeof(skcb->priority), &skcb->priority);

        if (msg->msg_name) {
                struct sockaddr_can *paddr = msg->msg_name;

                msg->msg_namelen = J1939_MIN_NAMELEN;
                memset(msg->msg_name, 0, msg->msg_namelen);
                paddr->can_family = AF_CAN;
                paddr->can_ifindex = skb->skb_iif;
                paddr->can_addr.j1939.name = skcb->addr.src_name;
                paddr->can_addr.j1939.addr = skcb->addr.sa;
                paddr->can_addr.j1939.pgn = skcb->addr.pgn;
        }

        sock_recv_cmsgs(msg, sk, skb);
        msg->msg_flags |= skcb->msg_flags;
        skb_free_datagram(sk, skb);

        return size;
}

static struct sk_buff *j1939_sk_alloc_skb(struct net_device *ndev,
                                          struct sock *sk,
                                          struct msghdr *msg, size_t size,
                                          int *errcode)
{
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_sk_buff_cb *skcb;
        struct sk_buff *skb;
        int ret;

        skb = sock_alloc_send_skb(sk,
                                  size +
                                  sizeof(struct can_frame) -
                                  sizeof(((struct can_frame *)NULL)->data) +
                                  sizeof(struct can_skb_priv),
                                  msg->msg_flags & MSG_DONTWAIT, &ret);
        if (!skb)
                goto failure;

        can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = ndev->ifindex;
        can_skb_prv(skb)->skbcnt = 0;
        skb_reserve(skb, offsetof(struct can_frame, data));

        ret = memcpy_from_msg(skb_put(skb, size), msg, size);
        if (ret < 0)
                goto free_skb;

        skb->dev = ndev;

        skcb = j1939_skb_to_cb(skb);
        memset(skcb, 0, sizeof(*skcb));
        skcb->addr = jsk->addr;
        skcb->priority = j1939_prio(READ_ONCE(sk->sk_priority));

        if (msg->msg_name) {
                struct sockaddr_can *addr = msg->msg_name;

                if (addr->can_addr.j1939.name ||
                    addr->can_addr.j1939.addr != J1939_NO_ADDR) {
                        skcb->addr.dst_name = addr->can_addr.j1939.name;
                        skcb->addr.da = addr->can_addr.j1939.addr;
                }
                if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                        skcb->addr.pgn = addr->can_addr.j1939.pgn;
        }

        *errcode = ret;
        return skb;

free_skb:
        kfree_skb(skb);
failure:
        *errcode = ret;
        return NULL;
}

static size_t j1939_sk_opt_stats_get_size(enum j1939_sk_errqueue_type type)
{
        switch (type) {
        case J1939_ERRQUEUE_RX_RTS:
                return
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_TOTAL_SIZE */
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_PGN */
                        nla_total_size(sizeof(u64)) + /* J1939_NLA_SRC_NAME */
                        nla_total_size(sizeof(u64)) + /* J1939_NLA_DEST_NAME */
                        nla_total_size(sizeof(u8)) +  /* J1939_NLA_SRC_ADDR */
                        nla_total_size(sizeof(u8)) +  /* J1939_NLA_DEST_ADDR */
                        0;
        default:
                return
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_BYTES_ACKED */
                        0;
        }
}

static struct sk_buff *
j1939_sk_get_timestamping_opt_stats(struct j1939_session *session,
                                    enum j1939_sk_errqueue_type type)
{
        struct sk_buff *stats;
        u32 size;

        stats = alloc_skb(j1939_sk_opt_stats_get_size(type), GFP_ATOMIC);
        if (!stats)
                return NULL;

        if (session->skcb.addr.type == J1939_SIMPLE)
                size = session->total_message_size;
        else
                size = min(session->pkt.tx_acked * 7,
                           session->total_message_size);

        switch (type) {
        case J1939_ERRQUEUE_RX_RTS:
                nla_put_u32(stats, J1939_NLA_TOTAL_SIZE,
                            session->total_message_size);
                nla_put_u32(stats, J1939_NLA_PGN,
                            session->skcb.addr.pgn);
                nla_put_u64_64bit(stats, J1939_NLA_SRC_NAME,
                                  session->skcb.addr.src_name, J1939_NLA_PAD);
                nla_put_u64_64bit(stats, J1939_NLA_DEST_NAME,
                                  session->skcb.addr.dst_name, J1939_NLA_PAD);
                nla_put_u8(stats, J1939_NLA_SRC_ADDR,
                           session->skcb.addr.sa);
                nla_put_u8(stats, J1939_NLA_DEST_ADDR,
                           session->skcb.addr.da);
                break;
        default:
                nla_put_u32(stats, J1939_NLA_BYTES_ACKED, size);
        }

        return stats;
}

static void __j1939_sk_errqueue(struct j1939_session *session, struct sock *sk,
                                enum j1939_sk_errqueue_type type)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sock *jsk;
        struct sock_exterr_skb *serr;
        struct sk_buff *skb;
        char *state = "UNK";
        u32 tsflags;
        int err;

        jsk = j1939_sk(sk);

        if (!(jsk->state & J1939_SOCK_ERRQUEUE))
                return;

        tsflags = READ_ONCE(sk->sk_tsflags);
        switch (type) {
        case J1939_ERRQUEUE_TX_ACK:
                if (!(tsflags & SOF_TIMESTAMPING_TX_ACK))
                        return;
                break;
        case J1939_ERRQUEUE_TX_SCHED:
                if (!(tsflags & SOF_TIMESTAMPING_TX_SCHED))
                        return;
                break;
        case J1939_ERRQUEUE_TX_ABORT:
                break;
        case J1939_ERRQUEUE_RX_RTS:
                fallthrough;
        case J1939_ERRQUEUE_RX_DPO:
                fallthrough;
        case J1939_ERRQUEUE_RX_ABORT:
                if (!(tsflags & SOF_TIMESTAMPING_RX_SOFTWARE))
                        return;
                break;
        default:
                netdev_err(priv->ndev, "Unknown errqueue type %i\n", type);
        }

        skb = j1939_sk_get_timestamping_opt_stats(session, type);
        if (!skb)
                return;

        skb->tstamp = ktime_get_real();

        BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));

        serr = SKB_EXT_ERR(skb);
        memset(serr, 0, sizeof(*serr));
        switch (type) {
        case J1939_ERRQUEUE_TX_ACK:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
                serr->ee.ee_info = SCM_TSTAMP_ACK;
                state = "TX ACK";
                break;
        case J1939_ERRQUEUE_TX_SCHED:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
                serr->ee.ee_info = SCM_TSTAMP_SCHED;
                state = "TX SCH";
                break;
        case J1939_ERRQUEUE_TX_ABORT:
                serr->ee.ee_errno = session->err;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_TX_ABORT;
                state = "TX ABT";
                break;
        case J1939_ERRQUEUE_RX_RTS:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_RTS;
                state = "RX RTS";
                break;
        case J1939_ERRQUEUE_RX_DPO:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_DPO;
                state = "RX DPO";
                break;
        case J1939_ERRQUEUE_RX_ABORT:
                serr->ee.ee_errno = session->err;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_ABORT;
                state = "RX ABT";
                break;
        }

        serr->opt_stats = true;
        if (tsflags & SOF_TIMESTAMPING_OPT_ID)
                serr->ee.ee_data = session->tskey;

        netdev_dbg(session->priv->ndev, "%s: 0x%p tskey: %i, state: %s\n",
                   __func__, session, session->tskey, state);
        err = sock_queue_err_skb(sk, skb);

        if (err)
                kfree_skb(skb);
};

void j1939_sk_errqueue(struct j1939_session *session,
                       enum j1939_sk_errqueue_type type)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sock *jsk;

        if (session->sk) {
                /* send TX notifications to the socket of origin  */
                __j1939_sk_errqueue(session, session->sk, type);
                return;
        }

        /* spread RX notifications to all sockets subscribed to this session */
        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                if (j1939_sk_recv_match_one(jsk, &session->skcb))
                        __j1939_sk_errqueue(session, &jsk->sk, type);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
};

void j1939_sk_send_loop_abort(struct sock *sk, int err)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        if (jsk->state & J1939_SOCK_ERRQUEUE)
                return;

        sk->sk_err = err;

        sk_error_report(sk);
}

static int j1939_sk_send_loop(struct j1939_priv *priv,  struct sock *sk,
                              struct msghdr *msg, size_t size)

{
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_session *session = j1939_sk_get_incomplete_session(jsk);
        struct sk_buff *skb;
        size_t segment_size, todo_size;
        int ret = 0;

        if (session &&
            session->total_message_size != session->total_queued_size + size) {
                j1939_session_put(session);
                return -EIO;
        }

        todo_size = size;

        while (todo_size) {
                struct j1939_sk_buff_cb *skcb;

                segment_size = min_t(size_t, J1939_MAX_TP_PACKET_SIZE,
                                     todo_size);

                /* Allocate skb for one segment */
                skb = j1939_sk_alloc_skb(priv->ndev, sk, msg, segment_size,
                                         &ret);
                if (ret)
                        break;

                skcb = j1939_skb_to_cb(skb);

                if (!session) {
                        /* at this point the size should be full size
                         * of the session
                         */
                        skcb->offset = 0;
                        session = j1939_tp_send(priv, skb, size);
                        if (IS_ERR(session)) {
                                ret = PTR_ERR(session);
                                goto kfree_skb;
                        }
                        if (j1939_sk_queue_session(session)) {
                                /* try to activate session if we a
                                 * fist in the queue
                                 */
                                if (!j1939_session_activate(session)) {
                                        j1939_tp_schedule_txtimer(session, 0);
                                } else {
                                        ret = -EBUSY;
                                        session->err = ret;
                                        j1939_sk_queue_drop_all(priv, jsk,
                                                                EBUSY);
                                        break;
                                }
                        }
                } else {
                        skcb->offset = session->total_queued_size;
                        j1939_session_skb_queue(session, skb);
                }

                todo_size -= segment_size;
                session->total_queued_size += segment_size;
        }

        switch (ret) {
        case 0: /* OK */
                if (todo_size)
                        netdev_warn(priv->ndev,
                                    "no error found and not completely queued?! %zu\n",
                                    todo_size);
                ret = size;
                break;
        case -ERESTARTSYS:
                ret = -EINTR;
                fallthrough;
        case -EAGAIN: /* OK */
                if (todo_size != size)
                        ret = size - todo_size;
                break;
        default: /* ERROR */
                break;
        }

        if (session)
                j1939_session_put(session);

        return ret;

 kfree_skb:
        kfree_skb(skb);
        return ret;
}

static int j1939_sk_sendmsg(struct socket *sock, struct msghdr *msg,
                            size_t size)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_priv *priv;
        int ifindex;
        int ret;

        lock_sock(sock->sk);
        /* various socket state tests */
        if (!(jsk->state & J1939_SOCK_BOUND)) {
                ret = -EBADFD;
                goto sendmsg_done;
        }

        priv = jsk->priv;
        ifindex = jsk->ifindex;

        if (!jsk->addr.src_name && jsk->addr.sa == J1939_NO_ADDR) {
                /* no source address assigned yet */
                ret = -EBADFD;
                goto sendmsg_done;
        }

        /* deal with provided destination address info */
        if (msg->msg_name) {
                struct sockaddr_can *addr = msg->msg_name;

                if (msg->msg_namelen < J1939_MIN_NAMELEN) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (addr->can_family != AF_CAN) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (addr->can_ifindex && addr->can_ifindex != ifindex) {
                        ret = -EBADFD;
                        goto sendmsg_done;
                }

                if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn) &&
                    !j1939_pgn_is_clean_pdu(addr->can_addr.j1939.pgn)) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (!addr->can_addr.j1939.name &&
                    addr->can_addr.j1939.addr == J1939_NO_ADDR &&
                    !sock_flag(sk, SOCK_BROADCAST)) {
                        /* broadcast, but SO_BROADCAST not set */
                        ret = -EACCES;
                        goto sendmsg_done;
                }
        } else {
                if (!jsk->addr.dst_name && jsk->addr.da == J1939_NO_ADDR &&
                    !sock_flag(sk, SOCK_BROADCAST)) {
                        /* broadcast, but SO_BROADCAST not set */
                        ret = -EACCES;
                        goto sendmsg_done;
                }
        }

        ret = j1939_sk_send_loop(priv, sk, msg, size);

sendmsg_done:
        release_sock(sock->sk);

        return ret;
}

void j1939_sk_netdev_event_netdown(struct j1939_priv *priv)
{
        struct j1939_sock *jsk;
        int error_code = ENETDOWN;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                jsk->sk.sk_err = error_code;
                if (!sock_flag(&jsk->sk, SOCK_DEAD))
                        sk_error_report(&jsk->sk);

                j1939_sk_queue_drop_all(priv, jsk, error_code);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
}

static int j1939_sk_no_ioctlcmd(struct socket *sock, unsigned int cmd,
                                unsigned long arg)
{
        /* no ioctls for socket layer -> hand it down to NIC layer */
        return -ENOIOCTLCMD;
}

static const struct proto_ops j1939_ops = {
        .family = PF_CAN,
        .release = j1939_sk_release,
        .bind = j1939_sk_bind,
        .connect = j1939_sk_connect,
        .socketpair = sock_no_socketpair,
        .accept = sock_no_accept,
        .getname = j1939_sk_getname,
        .poll = datagram_poll,
        .ioctl = j1939_sk_no_ioctlcmd,
        .listen = sock_no_listen,
        .shutdown = sock_no_shutdown,
        .setsockopt = j1939_sk_setsockopt,
        .getsockopt = j1939_sk_getsockopt,
        .sendmsg = j1939_sk_sendmsg,
        .recvmsg = j1939_sk_recvmsg,
        .mmap = sock_no_mmap,
};

static struct proto j1939_proto __read_mostly = {
        .name = "CAN_J1939",
        .owner = THIS_MODULE,
        .obj_size = sizeof(struct j1939_sock),
        .init = j1939_sk_init,
};

const struct can_proto j1939_can_proto = {
        .type = SOCK_DGRAM,
        .protocol = CAN_J1939,
        .ops = &j1939_ops,
        .prot = &j1939_proto,
};




















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
// SPDX-License-Identifier: GPL-2.0-only
/*
 * BPF JIT compiler
 *
 * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
 * Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 */
#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <linux/bpf.h>
#include <linux/memory.h>
#include <linux/sort.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
#include <asm/nospec-branch.h>
#include <asm/text-patching.h>
#include <asm/unwind.h>
#include <asm/cfi.h>

static bool all_callee_regs_used[4] = {true, true, true, true};

static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
{
        if (len == 1)
                *ptr = bytes;
        else if (len == 2)
                *(u16 *)ptr = bytes;
        else {
                *(u32 *)ptr = bytes;
                barrier();
        }
        return ptr + len;
}

#define EMIT(bytes, len) \
        do { prog = emit_code(prog, bytes, len); } while (0)

#define EMIT1(b1)                EMIT(b1, 1)
#define EMIT2(b1, b2)                EMIT((b1) + ((b2) << 8), 2)
#define EMIT3(b1, b2, b3)        EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
#define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)

#define EMIT1_off32(b1, off) \
        do { EMIT1(b1); EMIT(off, 4); } while (0)
#define EMIT2_off32(b1, b2, off) \
        do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
#define EMIT3_off32(b1, b2, b3, off) \
        do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
#define EMIT4_off32(b1, b2, b3, b4, off) \
        do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#ifdef CONFIG_X86_KERNEL_IBT
#define EMIT_ENDBR()                EMIT(gen_endbr(), 4)
#define EMIT_ENDBR_POISON()        EMIT(gen_endbr_poison(), 4)
#else
#define EMIT_ENDBR()
#define EMIT_ENDBR_POISON()
#endif

static bool is_imm8(int value)
{
        return value <= 127 && value >= -128;
}

/*
 * Let us limit the positive offset to be <= 123.
 * This is to ensure eventual jit convergence For the following patterns:
 * ...
 * pass4, final_proglen=4391:
 *   ...
 *   20e:    48 85 ff                test   rdi,rdi
 *   211:    74 7d                   je     0x290
 *   213:    48 8b 77 00             mov    rsi,QWORD PTR [rdi+0x0]
 *   ...
 *   289:    48 85 ff                test   rdi,rdi
 *   28c:    74 17                   je     0x2a5
 *   28e:    e9 7f ff ff ff          jmp    0x212
 *   293:    bf 03 00 00 00          mov    edi,0x3
 * Note that insn at 0x211 is 2-byte cond jump insn for offset 0x7d (-125)
 * and insn at 0x28e is 5-byte jmp insn with offset -129.
 *
 * pass5, final_proglen=4392:
 *   ...
 *   20e:    48 85 ff                test   rdi,rdi
 *   211:    0f 84 80 00 00 00       je     0x297
 *   217:    48 8b 77 00             mov    rsi,QWORD PTR [rdi+0x0]
 *   ...
 *   28d:    48 85 ff                test   rdi,rdi
 *   290:    74 1a                   je     0x2ac
 *   292:    eb 84                   jmp    0x218
 *   294:    bf 03 00 00 00          mov    edi,0x3
 * Note that insn at 0x211 is 6-byte cond jump insn now since its offset
 * becomes 0x80 based on previous round (0x293 - 0x213 = 0x80).
 * At the same time, insn at 0x292 is a 2-byte insn since its offset is
 * -124.
 *
 * pass6 will repeat the same code as in pass4 and this will prevent
 * eventual convergence.
 *
 * To fix this issue, we need to break je (2->6 bytes) <-> jmp (5->2 bytes)
 * cycle in the above. In the above example je offset <= 0x7c should work.
 *
 * For other cases, je <-> je needs offset <= 0x7b to avoid no convergence
 * issue. For jmp <-> je and jmp <-> jmp cases, jmp offset <= 0x7c should
 * avoid no convergence issue.
 *
 * Overall, let us limit the positive offset for 8bit cond/uncond jmp insn
 * to maximum 123 (0x7b). This way, the jit pass can eventually converge.
 */
static bool is_imm8_jmp_offset(int value)
{
        return value <= 123 && value >= -128;
}

static bool is_simm32(s64 value)
{
        return value == (s64)(s32)value;
}

static bool is_uimm32(u64 value)
{
        return value == (u64)(u32)value;
}

/* mov dst, src */
#define EMIT_mov(DST, SRC)                                                                 \
        do {                                                                                 \
                if (DST != SRC)                                                                 \
                        EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
        } while (0)

static int bpf_size_to_x86_bytes(int bpf_size)
{
        if (bpf_size == BPF_W)
                return 4;
        else if (bpf_size == BPF_H)
                return 2;
        else if (bpf_size == BPF_B)
                return 1;
        else if (bpf_size == BPF_DW)
                return 4; /* imm32 */
        else
                return 0;
}

/*
 * List of x86 cond jumps opcodes (. + s8)
 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
 */
#define X86_JB  0x72
#define X86_JAE 0x73
#define X86_JE  0x74
#define X86_JNE 0x75
#define X86_JBE 0x76
#define X86_JA  0x77
#define X86_JL  0x7C
#define X86_JGE 0x7D
#define X86_JLE 0x7E
#define X86_JG  0x7F

/* Pick a register outside of BPF range for JIT internal work */
#define AUX_REG (MAX_BPF_JIT_REG + 1)
#define X86_REG_R9 (MAX_BPF_JIT_REG + 2)
#define X86_REG_R12 (MAX_BPF_JIT_REG + 3)

/*
 * The following table maps BPF registers to x86-64 registers.
 *
 * x86-64 register R12 is unused, since if used as base address
 * register in load/store instructions, it always needs an
 * extra byte of encoding and is callee saved.
 *
 * x86-64 register R9 is not used by BPF programs, but can be used by BPF
 * trampoline. x86-64 register R10 is used for blinding (if enabled).
 */
static const int reg2hex[] = {
        [BPF_REG_0] = 0,  /* RAX */
        [BPF_REG_1] = 7,  /* RDI */
        [BPF_REG_2] = 6,  /* RSI */
        [BPF_REG_3] = 2,  /* RDX */
        [BPF_REG_4] = 1,  /* RCX */
        [BPF_REG_5] = 0,  /* R8  */
        [BPF_REG_6] = 3,  /* RBX callee saved */
        [BPF_REG_7] = 5,  /* R13 callee saved */
        [BPF_REG_8] = 6,  /* R14 callee saved */
        [BPF_REG_9] = 7,  /* R15 callee saved */
        [BPF_REG_FP] = 5, /* RBP readonly */
        [BPF_REG_AX] = 2, /* R10 temp register */
        [AUX_REG] = 3,    /* R11 temp register */
        [X86_REG_R9] = 1, /* R9 register, 6th function argument */
        [X86_REG_R12] = 4, /* R12 callee saved */
};

static const int reg2pt_regs[] = {
        [BPF_REG_0] = offsetof(struct pt_regs, ax),
        [BPF_REG_1] = offsetof(struct pt_regs, di),
        [BPF_REG_2] = offsetof(struct pt_regs, si),
        [BPF_REG_3] = offsetof(struct pt_regs, dx),
        [BPF_REG_4] = offsetof(struct pt_regs, cx),
        [BPF_REG_5] = offsetof(struct pt_regs, r8),
        [BPF_REG_6] = offsetof(struct pt_regs, bx),
        [BPF_REG_7] = offsetof(struct pt_regs, r13),
        [BPF_REG_8] = offsetof(struct pt_regs, r14),
        [BPF_REG_9] = offsetof(struct pt_regs, r15),
};

/*
 * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
 * which need extra byte of encoding.
 * rax,rcx,...,rbp have simpler encoding
 */
static bool is_ereg(u32 reg)
{
        return (1 << reg) & (BIT(BPF_REG_5) |
                             BIT(AUX_REG) |
                             BIT(BPF_REG_7) |
                             BIT(BPF_REG_8) |
                             BIT(BPF_REG_9) |
                             BIT(X86_REG_R9) |
                             BIT(X86_REG_R12) |
                             BIT(BPF_REG_AX));
}

/*
 * is_ereg_8l() == true if BPF register 'reg' is mapped to access x86-64
 * lower 8-bit registers dil,sil,bpl,spl,r8b..r15b, which need extra byte
 * of encoding. al,cl,dl,bl have simpler encoding.
 */
static bool is_ereg_8l(u32 reg)
{
        return is_ereg(reg) ||
            (1 << reg) & (BIT(BPF_REG_1) |
                          BIT(BPF_REG_2) |
                          BIT(BPF_REG_FP));
}

static bool is_axreg(u32 reg)
{
        return reg == BPF_REG_0;
}

/* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
static u8 add_1mod(u8 byte, u32 reg)
{
        if (is_ereg(reg))
                byte |= 1;
        return byte;
}

static u8 add_2mod(u8 byte, u32 r1, u32 r2)
{
        if (is_ereg(r1))
                byte |= 1;
        if (is_ereg(r2))
                byte |= 4;
        return byte;
}

static u8 add_3mod(u8 byte, u32 r1, u32 r2, u32 index)
{
        if (is_ereg(r1))
                byte |= 1;
        if (is_ereg(index))
                byte |= 2;
        if (is_ereg(r2))
                byte |= 4;
        return byte;
}

/* Encode 'dst_reg' register into x86-64 opcode 'byte' */
static u8 add_1reg(u8 byte, u32 dst_reg)
{
        return byte + reg2hex[dst_reg];
}

/* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */
static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
{
        return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
}

/* Some 1-byte opcodes for binary ALU operations */
static u8 simple_alu_opcodes[] = {
        [BPF_ADD] = 0x01,
        [BPF_SUB] = 0x29,
        [BPF_AND] = 0x21,
        [BPF_OR] = 0x09,
        [BPF_XOR] = 0x31,
        [BPF_LSH] = 0xE0,
        [BPF_RSH] = 0xE8,
        [BPF_ARSH] = 0xF8,
};

static void jit_fill_hole(void *area, unsigned int size)
{
        /* Fill whole space with INT3 instructions */
        memset(area, 0xcc, size);
}

int bpf_arch_text_invalidate(void *dst, size_t len)
{
        return IS_ERR_OR_NULL(text_poke_set(dst, 0xcc, len));
}

struct jit_context {
        int cleanup_addr; /* Epilogue code offset */

        /*
         * Program specific offsets of labels in the code; these rely on the
         * JIT doing at least 2 passes, recording the position on the first
         * pass, only to generate the correct offset on the second pass.
         */
        int tail_call_direct_label;
        int tail_call_indirect_label;
};

/* Maximum number of bytes emitted while JITing one eBPF insn */
#define BPF_MAX_INSN_SIZE        128
#define BPF_INSN_SAFETY                64

/* Number of bytes emit_patch() needs to generate instructions */
#define X86_PATCH_SIZE                5
/* Number of bytes that will be skipped on tailcall */
#define X86_TAIL_CALL_OFFSET        (12 + ENDBR_INSN_SIZE)

static void push_r9(u8 **pprog)
{
        u8 *prog = *pprog;

        EMIT2(0x41, 0x51);   /* push r9 */
        *pprog = prog;
}

static void pop_r9(u8 **pprog)
{
        u8 *prog = *pprog;

        EMIT2(0x41, 0x59);   /* pop r9 */
        *pprog = prog;
}

static void push_r12(u8 **pprog)
{
        u8 *prog = *pprog;

        EMIT2(0x41, 0x54);   /* push r12 */
        *pprog = prog;
}

static void push_callee_regs(u8 **pprog, bool *callee_regs_used)
{
        u8 *prog = *pprog;

        if (callee_regs_used[0])
                EMIT1(0x53);         /* push rbx */
        if (callee_regs_used[1])
                EMIT2(0x41, 0x55);   /* push r13 */
        if (callee_regs_used[2])
                EMIT2(0x41, 0x56);   /* push r14 */
        if (callee_regs_used[3])
                EMIT2(0x41, 0x57);   /* push r15 */
        *pprog = prog;
}

static void pop_r12(u8 **pprog)
{
        u8 *prog = *pprog;

        EMIT2(0x41, 0x5C);   /* pop r12 */
        *pprog = prog;
}

static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)
{
        u8 *prog = *pprog;

        if (callee_regs_used[3])
                EMIT2(0x41, 0x5F);   /* pop r15 */
        if (callee_regs_used[2])
                EMIT2(0x41, 0x5E);   /* pop r14 */
        if (callee_regs_used[1])
                EMIT2(0x41, 0x5D);   /* pop r13 */
        if (callee_regs_used[0])
                EMIT1(0x5B);         /* pop rbx */
        *pprog = prog;
}

static void emit_nops(u8 **pprog, int len)
{
        u8 *prog = *pprog;
        int i, noplen;

        while (len > 0) {
                noplen = len;

                if (noplen > ASM_NOP_MAX)
                        noplen = ASM_NOP_MAX;

                for (i = 0; i < noplen; i++)
                        EMIT1(x86_nops[noplen][i]);
                len -= noplen;
        }

        *pprog = prog;
}

/*
 * Emit the various CFI preambles, see asm/cfi.h and the comments about FineIBT
 * in arch/x86/kernel/alternative.c
 */

static void emit_fineibt(u8 **pprog, u32 hash)
{
        u8 *prog = *pprog;

        EMIT_ENDBR();
        EMIT3_off32(0x41, 0x81, 0xea, hash);                /* subl $hash, %r10d        */
        EMIT2(0x74, 0x07);                                /* jz.d8 +7                */
        EMIT2(0x0f, 0x0b);                                /* ud2                        */
        EMIT1(0x90);                                        /* nop                        */
        EMIT_ENDBR_POISON();

        *pprog = prog;
}

static void emit_kcfi(u8 **pprog, u32 hash)
{
        u8 *prog = *pprog;

        EMIT1_off32(0xb8, hash);                        /* movl $hash, %eax        */
#ifdef CONFIG_CALL_PADDING
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
        EMIT1(0x90);
#endif
        EMIT_ENDBR();

        *pprog = prog;
}

static void emit_cfi(u8 **pprog, u32 hash)
{
        u8 *prog = *pprog;

        switch (cfi_mode) {
        case CFI_FINEIBT:
                emit_fineibt(&prog, hash);
                break;

        case CFI_KCFI:
                emit_kcfi(&prog, hash);
                break;

        default:
                EMIT_ENDBR();
                break;
        }

        *pprog = prog;
}

static void emit_prologue_tail_call(u8 **pprog, bool is_subprog)
{
        u8 *prog = *pprog;

        if (!is_subprog) {
                /* cmp rax, MAX_TAIL_CALL_CNT */
                EMIT4(0x48, 0x83, 0xF8, MAX_TAIL_CALL_CNT);
                EMIT2(X86_JA, 6);        /* ja 6 */
                /* rax is tail_call_cnt if <= MAX_TAIL_CALL_CNT.
                 * case1: entry of main prog.
                 * case2: tail callee of main prog.
                 */
                EMIT1(0x50);             /* push rax */
                /* Make rax as tail_call_cnt_ptr. */
                EMIT3(0x48, 0x89, 0xE0); /* mov rax, rsp */
                EMIT2(0xEB, 1);          /* jmp 1 */
                /* rax is tail_call_cnt_ptr if > MAX_TAIL_CALL_CNT.
                 * case: tail callee of subprog.
                 */
                EMIT1(0x50);             /* push rax */
                /* push tail_call_cnt_ptr */
                EMIT1(0x50);             /* push rax */
        } else { /* is_subprog */
                /* rax is tail_call_cnt_ptr. */
                EMIT1(0x50);             /* push rax */
                EMIT1(0x50);             /* push rax */
        }

        *pprog = prog;
}

/*
 * Emit x86-64 prologue code for BPF program.
 * bpf_tail_call helper will skip the first X86_TAIL_CALL_OFFSET bytes
 * while jumping to another program
 */
static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,
                          bool tail_call_reachable, bool is_subprog,
                          bool is_exception_cb)
{
        u8 *prog = *pprog;

        emit_cfi(&prog, is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash);
        /* BPF trampoline can be made to work without these nops,
         * but let's waste 5 bytes for now and optimize later
         */
        emit_nops(&prog, X86_PATCH_SIZE);
        if (!ebpf_from_cbpf) {
                if (tail_call_reachable && !is_subprog)
                        /* When it's the entry of the whole tailcall context,
                         * zeroing rax means initialising tail_call_cnt.
                         */
                        EMIT3(0x48, 0x31, 0xC0); /* xor rax, rax */
                else
                        /* Keep the same instruction layout. */
                        emit_nops(&prog, 3);     /* nop3 */
        }
        /* Exception callback receives FP as third parameter */
        if (is_exception_cb) {
                EMIT3(0x48, 0x89, 0xF4); /* mov rsp, rsi */
                EMIT3(0x48, 0x89, 0xD5); /* mov rbp, rdx */
                /* The main frame must have exception_boundary as true, so we
                 * first restore those callee-saved regs from stack, before
                 * reusing the stack frame.
                 */
                pop_callee_regs(&prog, all_callee_regs_used);
                pop_r12(&prog);
                /* Reset the stack frame. */
                EMIT3(0x48, 0x89, 0xEC); /* mov rsp, rbp */
        } else {
                EMIT1(0x55);             /* push rbp */
                EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
        }

        /* X86_TAIL_CALL_OFFSET is here */
        EMIT_ENDBR();

        /* sub rsp, rounded_stack_depth */
        if (stack_depth)
                EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
        if (tail_call_reachable)
                emit_prologue_tail_call(&prog, is_subprog);
        *pprog = prog;
}

static int emit_patch(u8 **pprog, void *func, void *ip, u8 opcode)
{
        u8 *prog = *pprog;
        s64 offset;

        offset = func - (ip + X86_PATCH_SIZE);
        if (!is_simm32(offset)) {
                pr_err("Target call %p is out of range\n", func);
                return -ERANGE;
        }
        EMIT1_off32(opcode, offset);
        *pprog = prog;
        return 0;
}

static int emit_call(u8 **pprog, void *func, void *ip)
{
        return emit_patch(pprog, func, ip, 0xE8);
}

static int emit_rsb_call(u8 **pprog, void *func, void *ip)
{
        OPTIMIZER_HIDE_VAR(func);
        ip += x86_call_depth_emit_accounting(pprog, func, ip);
        return emit_patch(pprog, func, ip, 0xE8);
}

static int emit_jump(u8 **pprog, void *func, void *ip)
{
        return emit_patch(pprog, func, ip, 0xE9);
}

static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
                                void *old_addr, void *new_addr)
{
        const u8 *nop_insn = x86_nops[5];
        u8 old_insn[X86_PATCH_SIZE];
        u8 new_insn[X86_PATCH_SIZE];
        u8 *prog;
        int ret;

        memcpy(old_insn, nop_insn, X86_PATCH_SIZE);
        if (old_addr) {
                prog = old_insn;
                ret = t == BPF_MOD_CALL ?
                      emit_call(&prog, old_addr, ip) :
                      emit_jump(&prog, old_addr, ip);
                if (ret)
                        return ret;
        }

        memcpy(new_insn, nop_insn, X86_PATCH_SIZE);
        if (new_addr) {
                prog = new_insn;
                ret = t == BPF_MOD_CALL ?
                      emit_call(&prog, new_addr, ip) :
                      emit_jump(&prog, new_addr, ip);
                if (ret)
                        return ret;
        }

        ret = -EBUSY;
        mutex_lock(&text_mutex);
        if (memcmp(ip, old_insn, X86_PATCH_SIZE))
                goto out;
        ret = 1;
        if (memcmp(ip, new_insn, X86_PATCH_SIZE)) {
                text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL);
                ret = 0;
        }
out:
        mutex_unlock(&text_mutex);
        return ret;
}

int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
                       void *old_addr, void *new_addr)
{
        if (!is_kernel_text((long)ip) &&
            !is_bpf_text_address((long)ip))
                /* BPF poking in modules is not supported */
                return -EINVAL;

        /*
         * See emit_prologue(), for IBT builds the trampoline hook is preceded
         * with an ENDBR instruction.
         */
        if (is_endbr(*(u32 *)ip))
                ip += ENDBR_INSN_SIZE;

        return __bpf_arch_text_poke(ip, t, old_addr, new_addr);
}

#define EMIT_LFENCE()        EMIT3(0x0F, 0xAE, 0xE8)

static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
{
        u8 *prog = *pprog;

        if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
                EMIT_LFENCE();
                EMIT2(0xFF, 0xE0 + reg);
        } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
                OPTIMIZER_HIDE_VAR(reg);
                if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
                        emit_jump(&prog, &__x86_indirect_jump_thunk_array[reg], ip);
                else
                        emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
        } else {
                EMIT2(0xFF, 0xE0 + reg);        /* jmp *%\reg */
                if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) || IS_ENABLED(CONFIG_MITIGATION_SLS))
                        EMIT1(0xCC);                /* int3 */
        }

        *pprog = prog;
}

static void emit_return(u8 **pprog, u8 *ip)
{
        u8 *prog = *pprog;

        if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) {
                emit_jump(&prog, x86_return_thunk, ip);
        } else {
                EMIT1(0xC3);                /* ret */
                if (IS_ENABLED(CONFIG_MITIGATION_SLS))
                        EMIT1(0xCC);        /* int3 */
        }

        *pprog = prog;
}

#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack)        (-16 - round_up(stack, 8))

/*
 * Generate the following code:
 *
 * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
 *   if (index >= array->map.max_entries)
 *     goto out;
 *   if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
 *     goto out;
 *   prog = array->ptrs[index];
 *   if (prog == NULL)
 *     goto out;
 *   goto *(prog->bpf_func + prologue_size);
 * out:
 */
static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog,
                                        u8 **pprog, bool *callee_regs_used,
                                        u32 stack_depth, u8 *ip,
                                        struct jit_context *ctx)
{
        int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack_depth);
        u8 *prog = *pprog, *start = *pprog;
        int offset;

        /*
         * rdi - pointer to ctx
         * rsi - pointer to bpf_array
         * rdx - index in bpf_array
         */

        /*
         * if (index >= array->map.max_entries)
         *        goto out;
         */
        EMIT2(0x89, 0xD2);                        /* mov edx, edx */
        EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
              offsetof(struct bpf_array, map.max_entries));

        offset = ctx->tail_call_indirect_label - (prog + 2 - start);
        EMIT2(X86_JBE, offset);                   /* jbe out */

        /*
         * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
         *        goto out;
         */
        EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */
        EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */

        offset = ctx->tail_call_indirect_label - (prog + 2 - start);
        EMIT2(X86_JAE, offset);                   /* jae out */

        /* prog = array->ptrs[index]; */
        EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6,       /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */
                    offsetof(struct bpf_array, ptrs));

        /*
         * if (prog == NULL)
         *        goto out;
         */
        EMIT3(0x48, 0x85, 0xC9);                  /* test rcx,rcx */

        offset = ctx->tail_call_indirect_label - (prog + 2 - start);
        EMIT2(X86_JE, offset);                    /* je out */

        /* Inc tail_call_cnt if the slot is populated. */
        EMIT4(0x48, 0x83, 0x00, 0x01);            /* add qword ptr [rax], 1 */

        if (bpf_prog->aux->exception_boundary) {
                pop_callee_regs(&prog, all_callee_regs_used);
                pop_r12(&prog);
        } else {
                pop_callee_regs(&prog, callee_regs_used);
                if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
                        pop_r12(&prog);
        }

        /* Pop tail_call_cnt_ptr. */
        EMIT1(0x58);                              /* pop rax */
        /* Pop tail_call_cnt, if it's main prog.
         * Pop tail_call_cnt_ptr, if it's subprog.
         */
        EMIT1(0x58);                              /* pop rax */
        if (stack_depth)
                EMIT3_off32(0x48, 0x81, 0xC4,     /* add rsp, sd */
                            round_up(stack_depth, 8));

        /* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */
        EMIT4(0x48, 0x8B, 0x49,                   /* mov rcx, qword ptr [rcx + 32] */
              offsetof(struct bpf_prog, bpf_func));
        EMIT4(0x48, 0x83, 0xC1,                   /* add rcx, X86_TAIL_CALL_OFFSET */
              X86_TAIL_CALL_OFFSET);
        /*
         * Now we're ready to jump into next BPF program
         * rdi == ctx (1st arg)
         * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET
         */
        emit_indirect_jump(&prog, 1 /* rcx */, ip + (prog - start));

        /* out: */
        ctx->tail_call_indirect_label = prog - start;
        *pprog = prog;
}

static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog,
                                      struct bpf_jit_poke_descriptor *poke,
                                      u8 **pprog, u8 *ip,
                                      bool *callee_regs_used, u32 stack_depth,
                                      struct jit_context *ctx)
{
        int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack_depth);
        u8 *prog = *pprog, *start = *pprog;
        int offset;

        /*
         * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
         *        goto out;
         */
        EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off);   /* mov rax, qword ptr [rbp - tcc_ptr_off] */
        EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT);   /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */

        offset = ctx->tail_call_direct_label - (prog + 2 - start);
        EMIT2(X86_JAE, offset);                       /* jae out */

        poke->tailcall_bypass = ip + (prog - start);
        poke->adj_off = X86_TAIL_CALL_OFFSET;
        poke->tailcall_target = ip + ctx->tail_call_direct_label - X86_PATCH_SIZE;
        poke->bypass_addr = (u8 *)poke->tailcall_target + X86_PATCH_SIZE;

        emit_jump(&prog, (u8 *)poke->tailcall_target + X86_PATCH_SIZE,
                  poke->tailcall_bypass);

        /* Inc tail_call_cnt if the slot is populated. */
        EMIT4(0x48, 0x83, 0x00, 0x01);                /* add qword ptr [rax], 1 */

        if (bpf_prog->aux->exception_boundary) {
                pop_callee_regs(&prog, all_callee_regs_used);
                pop_r12(&prog);
        } else {
                pop_callee_regs(&prog, callee_regs_used);
                if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena))
                        pop_r12(&prog);
        }

        /* Pop tail_call_cnt_ptr. */
        EMIT1(0x58);                                  /* pop rax */
        /* Pop tail_call_cnt, if it's main prog.
         * Pop tail_call_cnt_ptr, if it's subprog.
         */
        EMIT1(0x58);                                  /* pop rax */
        if (stack_depth)
                EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8));

        emit_nops(&prog, X86_PATCH_SIZE);

        /* out: */
        ctx->tail_call_direct_label = prog - start;

        *pprog = prog;
}

static void bpf_tail_call_direct_fixup(struct bpf_prog *prog)
{
        struct bpf_jit_poke_descriptor *poke;
        struct bpf_array *array;
        struct bpf_prog *target;
        int i, ret;

        for (i = 0; i < prog->aux->size_poke_tab; i++) {
                poke = &prog->aux->poke_tab[i];
                if (poke->aux && poke->aux != prog->aux)
                        continue;

                WARN_ON_ONCE(READ_ONCE(poke->tailcall_target_stable));

                if (poke->reason != BPF_POKE_REASON_TAIL_CALL)
                        continue;

                array = container_of(poke->tail_call.map, struct bpf_array, map);
                mutex_lock(&array->aux->poke_mutex);
                target = array->ptrs[poke->tail_call.key];
                if (target) {
                        ret = __bpf_arch_text_poke(poke->tailcall_target,
                                                   BPF_MOD_JUMP, NULL,
                                                   (u8 *)target->bpf_func +
                                                   poke->adj_off);
                        BUG_ON(ret < 0);
                        ret = __bpf_arch_text_poke(poke->tailcall_bypass,
                                                   BPF_MOD_JUMP,
                                                   (u8 *)poke->tailcall_target +
                                                   X86_PATCH_SIZE, NULL);
                        BUG_ON(ret < 0);
                }
                WRITE_ONCE(poke->tailcall_target_stable, true);
                mutex_unlock(&array->aux->poke_mutex);
        }
}

static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
                           u32 dst_reg, const u32 imm32)
{
        u8 *prog = *pprog;
        u8 b1, b2, b3;

        /*
         * Optimization: if imm32 is positive, use 'mov %eax, imm32'
         * (which zero-extends imm32) to save 2 bytes.
         */
        if (sign_propagate && (s32)imm32 < 0) {
                /* 'mov %rax, imm32' sign extends imm32 */
                b1 = add_1mod(0x48, dst_reg);
                b2 = 0xC7;
                b3 = 0xC0;
                EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
                goto done;
        }

        /*
         * Optimization: if imm32 is zero, use 'xor %eax, %eax'
         * to save 3 bytes.
         */
        if (imm32 == 0) {
                if (is_ereg(dst_reg))
                        EMIT1(add_2mod(0x40, dst_reg, dst_reg));
                b2 = 0x31; /* xor */
                b3 = 0xC0;
                EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
                goto done;
        }

        /* mov %eax, imm32 */
        if (is_ereg(dst_reg))
                EMIT1(add_1mod(0x40, dst_reg));
        EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
done:
        *pprog = prog;
}

static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
                           const u32 imm32_hi, const u32 imm32_lo)
{
        u64 imm64 = ((u64)imm32_hi << 32) | (u32)imm32_lo;
        u8 *prog = *pprog;

        if (is_uimm32(imm64)) {
                /*
                 * For emitting plain u32, where sign bit must not be
                 * propagated LLVM tends to load imm64 over mov32
                 * directly, so save couple of bytes by just doing
                 * 'mov %eax, imm32' instead.
                 */
                emit_mov_imm32(&prog, false, dst_reg, imm32_lo);
        } else if (is_simm32(imm64)) {
                emit_mov_imm32(&prog, true, dst_reg, imm32_lo);
        } else {
                /* movabsq rax, imm64 */
                EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
                EMIT(imm32_lo, 4);
                EMIT(imm32_hi, 4);
        }

        *pprog = prog;
}

static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
{
        u8 *prog = *pprog;

        if (is64) {
                /* mov dst, src */
                EMIT_mov(dst_reg, src_reg);
        } else {
                /* mov32 dst, src */
                if (is_ereg(dst_reg) || is_ereg(src_reg))
                        EMIT1(add_2mod(0x40, dst_reg, src_reg));
                EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
        }

        *pprog = prog;
}

static void emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg,
                           u32 src_reg)
{
        u8 *prog = *pprog;

        if (is64) {
                /* movs[b,w,l]q dst, src */
                if (num_bits == 8)
                        EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbe,
                              add_2reg(0xC0, src_reg, dst_reg));
                else if (num_bits == 16)
                        EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbf,
                              add_2reg(0xC0, src_reg, dst_reg));
                else if (num_bits == 32)
                        EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x63,
                              add_2reg(0xC0, src_reg, dst_reg));
        } else {
                /* movs[b,w]l dst, src */
                if (num_bits == 8) {
                        EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe,
                              add_2reg(0xC0, src_reg, dst_reg));
                } else if (num_bits == 16) {
                        if (is_ereg(dst_reg) || is_ereg(src_reg))
                                EMIT1(add_2mod(0x40, src_reg, dst_reg));
                        EMIT3(add_2mod(0x0f, src_reg, dst_reg), 0xbf,
                              add_2reg(0xC0, src_reg, dst_reg));
                }
        }

        *pprog = prog;
}

/* Emit the suffix (ModR/M etc) for addressing *(ptr_reg + off) and val_reg */
static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)
{
        u8 *prog = *pprog;

        if (is_imm8(off)) {
                /* 1-byte signed displacement.
                 *
                 * If off == 0 we could skip this and save one extra byte, but
                 * special case of x86 R13 which always needs an offset is not
                 * worth the hassle
                 */
                EMIT2(add_2reg(0x40, ptr_reg, val_reg), off);
        } else {
                /* 4-byte signed displacement */
                EMIT1_off32(add_2reg(0x80, ptr_reg, val_reg), off);
        }
        *pprog = prog;
}

static void emit_insn_suffix_SIB(u8 **pprog, u32 ptr_reg, u32 val_reg, u32 index_reg, int off)
{
        u8 *prog = *pprog;

        if (is_imm8(off)) {
                EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off);
        } else {
                EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off);
        }
        *pprog = prog;
}

/*
 * Emit a REX byte if it will be necessary to address these registers
 */
static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)
{
        u8 *prog = *pprog;

        if (is64)
                EMIT1(add_2mod(0x48, dst_reg, src_reg));
        else if (is_ereg(dst_reg) || is_ereg(src_reg))
                EMIT1(add_2mod(0x40, dst_reg, src_reg));
        *pprog = prog;
}

/*
 * Similar version of maybe_emit_mod() for a single register
 */
static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)
{
        u8 *prog = *pprog;

        if (is64)
                EMIT1(add_1mod(0x48, reg));
        else if (is_ereg(reg))
                EMIT1(add_1mod(0x40, reg));
        *pprog = prog;
}

/* LDX: dst_reg = *(u8*)(src_reg + off) */
static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* Emit 'movzx rax, byte ptr [rax + off]' */
                EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
                break;
        case BPF_H:
                /* Emit 'movzx rax, word ptr [rax + off]' */
                EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
                break;
        case BPF_W:
                /* Emit 'mov eax, dword ptr [rax+0x14]' */
                if (is_ereg(dst_reg) || is_ereg(src_reg))
                        EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
                else
                        EMIT1(0x8B);
                break;
        case BPF_DW:
                /* Emit 'mov rax, qword ptr [rax+0x14]' */
                EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
                break;
        }
        emit_insn_suffix(&prog, src_reg, dst_reg, off);
        *pprog = prog;
}

/* LDSX: dst_reg = *(s8*)(src_reg + off) */
static void emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* Emit 'movsx rax, byte ptr [rax + off]' */
                EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBE);
                break;
        case BPF_H:
                /* Emit 'movsx rax, word ptr [rax + off]' */
                EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBF);
                break;
        case BPF_W:
                /* Emit 'movsx rax, dword ptr [rax+0x14]' */
                EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x63);
                break;
        }
        emit_insn_suffix(&prog, src_reg, dst_reg, off);
        *pprog = prog;
}

static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* movzx rax, byte ptr [rax + r12 + off] */
                EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB6);
                break;
        case BPF_H:
                /* movzx rax, word ptr [rax + r12 + off] */
                EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB7);
                break;
        case BPF_W:
                /* mov eax, dword ptr [rax + r12 + off] */
                EMIT2(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x8B);
                break;
        case BPF_DW:
                /* mov rax, qword ptr [rax + r12 + off] */
                EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B);
                break;
        }
        emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off);
        *pprog = prog;
}

static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
        emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);
}

/* STX: *(u8*)(dst_reg + off) = src_reg */
static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* Emit 'mov byte ptr [rax + off], al' */
                if (is_ereg(dst_reg) || is_ereg_8l(src_reg))
                        /* Add extra byte for eregs or SIL,DIL,BPL in src_reg */
                        EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
                else
                        EMIT1(0x88);
                break;
        case BPF_H:
                if (is_ereg(dst_reg) || is_ereg(src_reg))
                        EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
                else
                        EMIT2(0x66, 0x89);
                break;
        case BPF_W:
                if (is_ereg(dst_reg) || is_ereg(src_reg))
                        EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
                else
                        EMIT1(0x89);
                break;
        case BPF_DW:
                EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
                break;
        }
        emit_insn_suffix(&prog, dst_reg, src_reg, off);
        *pprog = prog;
}

/* STX: *(u8*)(dst_reg + index_reg + off) = src_reg */
static void emit_stx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* mov byte ptr [rax + r12 + off], al */
                EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x88);
                break;
        case BPF_H:
                /* mov word ptr [rax + r12 + off], ax */
                EMIT3(0x66, add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89);
                break;
        case BPF_W:
                /* mov dword ptr [rax + r12 + 1], eax */
                EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89);
                break;
        case BPF_DW:
                /* mov qword ptr [rax + r12 + 1], rax */
                EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89);
                break;
        }
        emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off);
        *pprog = prog;
}

static void emit_stx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
        emit_stx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);
}

/* ST: *(u8*)(dst_reg + index_reg + off) = imm32 */
static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int off, int imm)
{
        u8 *prog = *pprog;

        switch (size) {
        case BPF_B:
                /* mov byte ptr [rax + r12 + off], imm8 */
                EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC6);
                break;
        case BPF_H:
                /* mov word ptr [rax + r12 + off], imm16 */
                EMIT3(0x66, add_3mod(0x40, dst_reg, 0, index_reg), 0xC7);
                break;
        case BPF_W:
                /* mov dword ptr [rax + r12 + 1], imm32 */
                EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC7);
                break;
        case BPF_DW:
                /* mov qword ptr [rax + r12 + 1], imm32 */
                EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7);
                break;
        }
        emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off);
        EMIT(imm, bpf_size_to_x86_bytes(size));
        *pprog = prog;
}

static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)
{
        emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
}

static int emit_atomic(u8 **pprog, u8 atomic_op,
                       u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
{
        u8 *prog = *pprog;

        EMIT1(0xF0); /* lock prefix */

        maybe_emit_mod(&prog, dst_reg, src_reg, bpf_size == BPF_DW);

        /* emit opcode */
        switch (atomic_op) {
        case BPF_ADD:
        case BPF_AND:
        case BPF_OR:
        case BPF_XOR:
                /* lock *(u32/u64*)(dst_reg + off) <op>= src_reg */
                EMIT1(simple_alu_opcodes[atomic_op]);
                break;
        case BPF_ADD | BPF_FETCH:
                /* src_reg = atomic_fetch_add(dst_reg + off, src_reg); */
                EMIT2(0x0F, 0xC1);
                break;
        case BPF_XCHG:
                /* src_reg = atomic_xchg(dst_reg + off, src_reg); */
                EMIT1(0x87);
                break;
        case BPF_CMPXCHG:
                /* r0 = atomic_cmpxchg(dst_reg + off, r0, src_reg); */
                EMIT2(0x0F, 0xB1);
                break;
        default:
                pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op);
                return -EFAULT;
        }

        emit_insn_suffix(&prog, dst_reg, src_reg, off);

        *pprog = prog;
        return 0;
}

static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
                             u32 dst_reg, u32 src_reg, u32 index_reg, int off)
{
        u8 *prog = *pprog;

        EMIT1(0xF0); /* lock prefix */
        switch (size) {
        case BPF_W:
                EMIT1(add_3mod(0x40, dst_reg, src_reg, index_reg));
                break;
        case BPF_DW:
                EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
                break;
        default:
                pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");
                return -EFAULT;
        }

        /* emit opcode */
        switch (atomic_op) {
        case BPF_ADD:
        case BPF_AND:
        case BPF_OR:
        case BPF_XOR:
                /* lock *(u32/u64*)(dst_reg + idx_reg + off) <op>= src_reg */
                EMIT1(simple_alu_opcodes[atomic_op]);
                break;
        case BPF_ADD | BPF_FETCH:
                /* src_reg = atomic_fetch_add(dst_reg + idx_reg + off, src_reg); */
                EMIT2(0x0F, 0xC1);
                break;
        case BPF_XCHG:
                /* src_reg = atomic_xchg(dst_reg + idx_reg + off, src_reg); */
                EMIT1(0x87);
                break;
        case BPF_CMPXCHG:
                /* r0 = atomic_cmpxchg(dst_reg + idx_reg + off, r0, src_reg); */
                EMIT2(0x0F, 0xB1);
                break;
        default:
                pr_err("bpf_jit: unknown atomic opcode %02x\n", atomic_op);
                return -EFAULT;
        }
        emit_insn_suffix_SIB(&prog, dst_reg, src_reg, index_reg, off);
        *pprog = prog;
        return 0;
}

#define DONT_CLEAR 1

bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)
{
        u32 reg = x->fixup >> 8;

        /* jump over faulting load and clear dest register */
        if (reg != DONT_CLEAR)
                *(unsigned long *)((void *)regs + reg) = 0;
        regs->ip += x->fixup & 0xff;
        return true;
}

static void detect_reg_usage(struct bpf_insn *insn, int insn_cnt,
                             bool *regs_used)
{
        int i;

        for (i = 1; i <= insn_cnt; i++, insn++) {
                if (insn->dst_reg == BPF_REG_6 || insn->src_reg == BPF_REG_6)
                        regs_used[0] = true;
                if (insn->dst_reg == BPF_REG_7 || insn->src_reg == BPF_REG_7)
                        regs_used[1] = true;
                if (insn->dst_reg == BPF_REG_8 || insn->src_reg == BPF_REG_8)
                        regs_used[2] = true;
                if (insn->dst_reg == BPF_REG_9 || insn->src_reg == BPF_REG_9)
                        regs_used[3] = true;
        }
}

/* emit the 3-byte VEX prefix
 *
 * r: same as rex.r, extra bit for ModRM reg field
 * x: same as rex.x, extra bit for SIB index field
 * b: same as rex.b, extra bit for ModRM r/m, or SIB base
 * m: opcode map select, encoding escape bytes e.g. 0x0f38
 * w: same as rex.w (32 bit or 64 bit) or opcode specific
 * src_reg2: additional source reg (encoded as BPF reg)
 * l: vector length (128 bit or 256 bit) or reserved
 * pp: opcode prefix (none, 0x66, 0xf2 or 0xf3)
 */
static void emit_3vex(u8 **pprog, bool r, bool x, bool b, u8 m,
                      bool w, u8 src_reg2, bool l, u8 pp)
{
        u8 *prog = *pprog;
        const u8 b0 = 0xc4; /* first byte of 3-byte VEX prefix */
        u8 b1, b2;
        u8 vvvv = reg2hex[src_reg2];

        /* reg2hex gives only the lower 3 bit of vvvv */
        if (is_ereg(src_reg2))
                vvvv |= 1 << 3;

        /*
         * 2nd byte of 3-byte VEX prefix
         * ~ means bit inverted encoding
         *
         *    7                           0
         *  +---+---+---+---+---+---+---+---+
         *  |~R |~X |~B |         m         |
         *  +---+---+---+---+---+---+---+---+
         */
        b1 = (!r << 7) | (!x << 6) | (!b << 5) | (m & 0x1f);
        /*
         * 3rd byte of 3-byte VEX prefix
         *
         *    7                           0
         *  +---+---+---+---+---+---+---+---+
         *  | W |     ~vvvv     | L |   pp  |
         *  +---+---+---+---+---+---+---+---+
         */
        b2 = (w << 7) | ((~vvvv & 0xf) << 3) | (l << 2) | (pp & 3);

        EMIT3(b0, b1, b2);
        *pprog = prog;
}

/* emit BMI2 shift instruction */
static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op)
{
        u8 *prog = *pprog;
        bool r = is_ereg(dst_reg);
        u8 m = 2; /* escape code 0f38 */

        emit_3vex(&prog, r, false, r, m, is64, src_reg, false, op);
        EMIT2(0xf7, add_2reg(0xC0, dst_reg, dst_reg));
        *pprog = prog;
}

static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr)
{
        u8 *prog = *pprog;

        /* movabs r9, priv_frame_ptr */
        emit_mov_imm64(&prog, X86_REG_R9, (__force long) priv_frame_ptr >> 32,
                       (u32) (__force long) priv_frame_ptr);

#ifdef CONFIG_SMP
        /* add <r9>, gs:[<off>] */
        EMIT2(0x65, 0x4c);
        EMIT3(0x03, 0x0c, 0x25);
        EMIT((u32)(unsigned long)&this_cpu_off, 4);
#endif

        *pprog = prog;
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

#define __LOAD_TCC_PTR(off)                        \
        EMIT3_off32(0x48, 0x8B, 0x85, off)
/* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */
#define LOAD_TAIL_CALL_CNT_PTR(stack)                                \
        __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack))

/* Memory size/value to protect private stack overflow/underflow */
#define PRIV_STACK_GUARD_SZ    8
#define PRIV_STACK_GUARD_VAL   0xEB9F12345678eb9fULL

static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
                  int oldproglen, struct jit_context *ctx, bool jmp_padding)
{
        bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
        struct bpf_insn *insn = bpf_prog->insnsi;
        bool callee_regs_used[4] = {};
        int insn_cnt = bpf_prog->len;
        bool seen_exit = false;
        u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
        void __percpu *priv_frame_ptr = NULL;
        u64 arena_vm_start, user_vm_start;
        void __percpu *priv_stack_ptr;
        int i, excnt = 0;
        int ilen, proglen = 0;
        u8 *prog = temp;
        u32 stack_depth;
        int err;

        stack_depth = bpf_prog->aux->stack_depth;
        priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
        if (priv_stack_ptr) {
                priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
                stack_depth = 0;
        }

        arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
        user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);

        detect_reg_usage(insn, insn_cnt, callee_regs_used);

        emit_prologue(&prog, stack_depth,
                      bpf_prog_was_classic(bpf_prog), tail_call_reachable,
                      bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
        /* Exception callback will clobber callee regs for its own use, and
         * restore the original callee regs from main prog's stack frame.
         */
        if (bpf_prog->aux->exception_boundary) {
                /* We also need to save r12, which is not mapped to any BPF
                 * register, as we throw after entry into the kernel, which may
                 * overwrite r12.
                 */
                push_r12(&prog);
                push_callee_regs(&prog, all_callee_regs_used);
        } else {
                if (arena_vm_start)
                        push_r12(&prog);
                push_callee_regs(&prog, callee_regs_used);
        }
        if (arena_vm_start)
                emit_mov_imm64(&prog, X86_REG_R12,
                               arena_vm_start >> 32, (u32) arena_vm_start);

        if (priv_frame_ptr)
                emit_priv_frame_ptr(&prog, priv_frame_ptr);

        ilen = prog - temp;
        if (rw_image)
                memcpy(rw_image + proglen, temp, ilen);
        proglen += ilen;
        addrs[0] = proglen;
        prog = temp;

        for (i = 1; i <= insn_cnt; i++, insn++) {
                const s32 imm32 = insn->imm;
                u32 dst_reg = insn->dst_reg;
                u32 src_reg = insn->src_reg;
                u8 b2 = 0, b3 = 0;
                u8 *start_of_ldx;
                s64 jmp_offset;
                s16 insn_off;
                u8 jmp_cond;
                u8 *func;
                int nops;

                if (priv_frame_ptr) {
                        if (src_reg == BPF_REG_FP)
                                src_reg = X86_REG_R9;

                        if (dst_reg == BPF_REG_FP)
                                dst_reg = X86_REG_R9;
                }

                switch (insn->code) {
                        /* ALU */
                case BPF_ALU | BPF_ADD | BPF_X:
                case BPF_ALU | BPF_SUB | BPF_X:
                case BPF_ALU | BPF_AND | BPF_X:
                case BPF_ALU | BPF_OR | BPF_X:
                case BPF_ALU | BPF_XOR | BPF_X:
                case BPF_ALU64 | BPF_ADD | BPF_X:
                case BPF_ALU64 | BPF_SUB | BPF_X:
                case BPF_ALU64 | BPF_AND | BPF_X:
                case BPF_ALU64 | BPF_OR | BPF_X:
                case BPF_ALU64 | BPF_XOR | BPF_X:
                        maybe_emit_mod(&prog, dst_reg, src_reg,
                                       BPF_CLASS(insn->code) == BPF_ALU64);
                        b2 = simple_alu_opcodes[BPF_OP(insn->code)];
                        EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
                        break;

                case BPF_ALU64 | BPF_MOV | BPF_X:
                        if (insn_is_cast_user(insn)) {
                                if (dst_reg != src_reg)
                                        /* 32-bit mov */
                                        emit_mov_reg(&prog, false, dst_reg, src_reg);
                                /* shl dst_reg, 32 */
                                maybe_emit_1mod(&prog, dst_reg, true);
                                EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32);

                                /* or dst_reg, user_vm_start */
                                maybe_emit_1mod(&prog, dst_reg, true);
                                if (is_axreg(dst_reg))
                                        EMIT1_off32(0x0D,  user_vm_start >> 32);
                                else
                                        EMIT2_off32(0x81, add_1reg(0xC8, dst_reg),  user_vm_start >> 32);

                                /* rol dst_reg, 32 */
                                maybe_emit_1mod(&prog, dst_reg, true);
                                EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32);

                                /* xor r11, r11 */
                                EMIT3(0x4D, 0x31, 0xDB);

                                /* test dst_reg32, dst_reg32; check if lower 32-bit are zero */
                                maybe_emit_mod(&prog, dst_reg, dst_reg, false);
                                EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg));

                                /* cmove r11, dst_reg; if so, set dst_reg to zero */
                                /* WARNING: Intel swapped src/dst register encoding in CMOVcc !!! */
                                maybe_emit_mod(&prog, AUX_REG, dst_reg, true);
                                EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg));
                                break;
                        } else if (insn_is_mov_percpu_addr(insn)) {
                                /* mov <dst>, <src> (if necessary) */
                                EMIT_mov(dst_reg, src_reg);
#ifdef CONFIG_SMP
                                /* add <dst>, gs:[<off>] */
                                EMIT2(0x65, add_1mod(0x48, dst_reg));
                                EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25);
                                EMIT((u32)(unsigned long)&this_cpu_off, 4);
#endif
                                break;
                        }
                        fallthrough;
                case BPF_ALU | BPF_MOV | BPF_X:
                        if (insn->off == 0)
                                emit_mov_reg(&prog,
                                             BPF_CLASS(insn->code) == BPF_ALU64,
                                             dst_reg, src_reg);
                        else
                                emit_movsx_reg(&prog, insn->off,
                                               BPF_CLASS(insn->code) == BPF_ALU64,
                                               dst_reg, src_reg);
                        break;

                        /* neg dst */
                case BPF_ALU | BPF_NEG:
                case BPF_ALU64 | BPF_NEG:
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_ALU64);
                        EMIT2(0xF7, add_1reg(0xD8, dst_reg));
                        break;

                case BPF_ALU | BPF_ADD | BPF_K:
                case BPF_ALU | BPF_SUB | BPF_K:
                case BPF_ALU | BPF_AND | BPF_K:
                case BPF_ALU | BPF_OR | BPF_K:
                case BPF_ALU | BPF_XOR | BPF_K:
                case BPF_ALU64 | BPF_ADD | BPF_K:
                case BPF_ALU64 | BPF_SUB | BPF_K:
                case BPF_ALU64 | BPF_AND | BPF_K:
                case BPF_ALU64 | BPF_OR | BPF_K:
                case BPF_ALU64 | BPF_XOR | BPF_K:
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_ALU64);

                        /*
                         * b3 holds 'normal' opcode, b2 short form only valid
                         * in case dst is eax/rax.
                         */
                        switch (BPF_OP(insn->code)) {
                        case BPF_ADD:
                                b3 = 0xC0;
                                b2 = 0x05;
                                break;
                        case BPF_SUB:
                                b3 = 0xE8;
                                b2 = 0x2D;
                                break;
                        case BPF_AND:
                                b3 = 0xE0;
                                b2 = 0x25;
                                break;
                        case BPF_OR:
                                b3 = 0xC8;
                                b2 = 0x0D;
                                break;
                        case BPF_XOR:
                                b3 = 0xF0;
                                b2 = 0x35;
                                break;
                        }

                        if (is_imm8(imm32))
                                EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
                        else if (is_axreg(dst_reg))
                                EMIT1_off32(b2, imm32);
                        else
                                EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
                        break;

                case BPF_ALU64 | BPF_MOV | BPF_K:
                case BPF_ALU | BPF_MOV | BPF_K:
                        emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
                                       dst_reg, imm32);
                        break;

                case BPF_LD | BPF_IMM | BPF_DW:
                        emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
                        insn++;
                        i++;
                        break;

                        /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
                case BPF_ALU | BPF_MOD | BPF_X:
                case BPF_ALU | BPF_DIV | BPF_X:
                case BPF_ALU | BPF_MOD | BPF_K:
                case BPF_ALU | BPF_DIV | BPF_K:
                case BPF_ALU64 | BPF_MOD | BPF_X:
                case BPF_ALU64 | BPF_DIV | BPF_X:
                case BPF_ALU64 | BPF_MOD | BPF_K:
                case BPF_ALU64 | BPF_DIV | BPF_K: {
                        bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;

                        if (dst_reg != BPF_REG_0)
                                EMIT1(0x50); /* push rax */
                        if (dst_reg != BPF_REG_3)
                                EMIT1(0x52); /* push rdx */

                        if (BPF_SRC(insn->code) == BPF_X) {
                                if (src_reg == BPF_REG_0 ||
                                    src_reg == BPF_REG_3) {
                                        /* mov r11, src_reg */
                                        EMIT_mov(AUX_REG, src_reg);
                                        src_reg = AUX_REG;
                                }
                        } else {
                                /* mov r11, imm32 */
                                EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
                                src_reg = AUX_REG;
                        }

                        if (dst_reg != BPF_REG_0)
                                /* mov rax, dst_reg */
                                emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);

                        if (insn->off == 0) {
                                /*
                                 * xor edx, edx
                                 * equivalent to 'xor rdx, rdx', but one byte less
                                 */
                                EMIT2(0x31, 0xd2);

                                /* div src_reg */
                                maybe_emit_1mod(&prog, src_reg, is64);
                                EMIT2(0xF7, add_1reg(0xF0, src_reg));
                        } else {
                                if (BPF_CLASS(insn->code) == BPF_ALU)
                                        EMIT1(0x99); /* cdq */
                                else
                                        EMIT2(0x48, 0x99); /* cqo */

                                /* idiv src_reg */
                                maybe_emit_1mod(&prog, src_reg, is64);
                                EMIT2(0xF7, add_1reg(0xF8, src_reg));
                        }

                        if (BPF_OP(insn->code) == BPF_MOD &&
                            dst_reg != BPF_REG_3)
                                /* mov dst_reg, rdx */
                                emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3);
                        else if (BPF_OP(insn->code) == BPF_DIV &&
                                 dst_reg != BPF_REG_0)
                                /* mov dst_reg, rax */
                                emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0);

                        if (dst_reg != BPF_REG_3)
                                EMIT1(0x5A); /* pop rdx */
                        if (dst_reg != BPF_REG_0)
                                EMIT1(0x58); /* pop rax */
                        break;
                }

                case BPF_ALU | BPF_MUL | BPF_K:
                case BPF_ALU64 | BPF_MUL | BPF_K:
                        maybe_emit_mod(&prog, dst_reg, dst_reg,
                                       BPF_CLASS(insn->code) == BPF_ALU64);

                        if (is_imm8(imm32))
                                /* imul dst_reg, dst_reg, imm8 */
                                EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg),
                                      imm32);
                        else
                                /* imul dst_reg, dst_reg, imm32 */
                                EMIT2_off32(0x69,
                                            add_2reg(0xC0, dst_reg, dst_reg),
                                            imm32);
                        break;

                case BPF_ALU | BPF_MUL | BPF_X:
                case BPF_ALU64 | BPF_MUL | BPF_X:
                        maybe_emit_mod(&prog, src_reg, dst_reg,
                                       BPF_CLASS(insn->code) == BPF_ALU64);

                        /* imul dst_reg, src_reg */
                        EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));
                        break;

                        /* Shifts */
                case BPF_ALU | BPF_LSH | BPF_K:
                case BPF_ALU | BPF_RSH | BPF_K:
                case BPF_ALU | BPF_ARSH | BPF_K:
                case BPF_ALU64 | BPF_LSH | BPF_K:
                case BPF_ALU64 | BPF_RSH | BPF_K:
                case BPF_ALU64 | BPF_ARSH | BPF_K:
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_ALU64);

                        b3 = simple_alu_opcodes[BPF_OP(insn->code)];
                        if (imm32 == 1)
                                EMIT2(0xD1, add_1reg(b3, dst_reg));
                        else
                                EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
                        break;

                case BPF_ALU | BPF_LSH | BPF_X:
                case BPF_ALU | BPF_RSH | BPF_X:
                case BPF_ALU | BPF_ARSH | BPF_X:
                case BPF_ALU64 | BPF_LSH | BPF_X:
                case BPF_ALU64 | BPF_RSH | BPF_X:
                case BPF_ALU64 | BPF_ARSH | BPF_X:
                        /* BMI2 shifts aren't better when shift count is already in rcx */
                        if (boot_cpu_has(X86_FEATURE_BMI2) && src_reg != BPF_REG_4) {
                                /* shrx/sarx/shlx dst_reg, dst_reg, src_reg */
                                bool w = (BPF_CLASS(insn->code) == BPF_ALU64);
                                u8 op;

                                switch (BPF_OP(insn->code)) {
                                case BPF_LSH:
                                        op = 1; /* prefix 0x66 */
                                        break;
                                case BPF_RSH:
                                        op = 3; /* prefix 0xf2 */
                                        break;
                                case BPF_ARSH:
                                        op = 2; /* prefix 0xf3 */
                                        break;
                                }

                                emit_shiftx(&prog, dst_reg, src_reg, w, op);

                                break;
                        }

                        if (src_reg != BPF_REG_4) { /* common case */
                                /* Check for bad case when dst_reg == rcx */
                                if (dst_reg == BPF_REG_4) {
                                        /* mov r11, dst_reg */
                                        EMIT_mov(AUX_REG, dst_reg);
                                        dst_reg = AUX_REG;
                                } else {
                                        EMIT1(0x51); /* push rcx */
                                }
                                /* mov rcx, src_reg */
                                EMIT_mov(BPF_REG_4, src_reg);
                        }

                        /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_ALU64);

                        b3 = simple_alu_opcodes[BPF_OP(insn->code)];
                        EMIT2(0xD3, add_1reg(b3, dst_reg));

                        if (src_reg != BPF_REG_4) {
                                if (insn->dst_reg == BPF_REG_4)
                                        /* mov dst_reg, r11 */
                                        EMIT_mov(insn->dst_reg, AUX_REG);
                                else
                                        EMIT1(0x59); /* pop rcx */
                        }

                        break;

                case BPF_ALU | BPF_END | BPF_FROM_BE:
                case BPF_ALU64 | BPF_END | BPF_FROM_LE:
                        switch (imm32) {
                        case 16:
                                /* Emit 'ror %ax, 8' to swap lower 2 bytes */
                                EMIT1(0x66);
                                if (is_ereg(dst_reg))
                                        EMIT1(0x41);
                                EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);

                                /* Emit 'movzwl eax, ax' */
                                if (is_ereg(dst_reg))
                                        EMIT3(0x45, 0x0F, 0xB7);
                                else
                                        EMIT2(0x0F, 0xB7);
                                EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
                                break;
                        case 32:
                                /* Emit 'bswap eax' to swap lower 4 bytes */
                                if (is_ereg(dst_reg))
                                        EMIT2(0x41, 0x0F);
                                else
                                        EMIT1(0x0F);
                                EMIT1(add_1reg(0xC8, dst_reg));
                                break;
                        case 64:
                                /* Emit 'bswap rax' to swap 8 bytes */
                                EMIT3(add_1mod(0x48, dst_reg), 0x0F,
                                      add_1reg(0xC8, dst_reg));
                                break;
                        }
                        break;

                case BPF_ALU | BPF_END | BPF_FROM_LE:
                        switch (imm32) {
                        case 16:
                                /*
                                 * Emit 'movzwl eax, ax' to zero extend 16-bit
                                 * into 64 bit
                                 */
                                if (is_ereg(dst_reg))
                                        EMIT3(0x45, 0x0F, 0xB7);
                                else
                                        EMIT2(0x0F, 0xB7);
                                EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
                                break;
                        case 32:
                                /* Emit 'mov eax, eax' to clear upper 32-bits */
                                if (is_ereg(dst_reg))
                                        EMIT1(0x45);
                                EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
                                break;
                        case 64:
                                /* nop */
                                break;
                        }
                        break;

                        /* speculation barrier */
                case BPF_ST | BPF_NOSPEC:
                        EMIT_LFENCE();
                        break;

                        /* ST: *(u8*)(dst_reg + off) = imm */
                case BPF_ST | BPF_MEM | BPF_B:
                        if (is_ereg(dst_reg))
                                EMIT2(0x41, 0xC6);
                        else
                                EMIT1(0xC6);
                        goto st;
                case BPF_ST | BPF_MEM | BPF_H:
                        if (is_ereg(dst_reg))
                                EMIT3(0x66, 0x41, 0xC7);
                        else
                                EMIT2(0x66, 0xC7);
                        goto st;
                case BPF_ST | BPF_MEM | BPF_W:
                        if (is_ereg(dst_reg))
                                EMIT2(0x41, 0xC7);
                        else
                                EMIT1(0xC7);
                        goto st;
                case BPF_ST | BPF_MEM | BPF_DW:
                        EMIT2(add_1mod(0x48, dst_reg), 0xC7);

st:                        if (is_imm8(insn->off))
                                EMIT2(add_1reg(0x40, dst_reg), insn->off);
                        else
                                EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);

                        EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
                        break;

                        /* STX: *(u8*)(dst_reg + off) = src_reg */
                case BPF_STX | BPF_MEM | BPF_B:
                case BPF_STX | BPF_MEM | BPF_H:
                case BPF_STX | BPF_MEM | BPF_W:
                case BPF_STX | BPF_MEM | BPF_DW:
                        emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
                        break;

                case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
                case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
                case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
                case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:
                        start_of_ldx = prog;
                        emit_st_r12(&prog, BPF_SIZE(insn->code), dst_reg, insn->off, insn->imm);
                        goto populate_extable;

                        /* LDX: dst_reg = *(u8*)(src_reg + r12 + off) */
                case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
                case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
                case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
                case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
                case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
                case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
                case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
                case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
                        start_of_ldx = prog;
                        if (BPF_CLASS(insn->code) == BPF_LDX)
                                emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
                        else
                                emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
populate_extable:
                        {
                                struct exception_table_entry *ex;
                                u8 *_insn = image + proglen + (start_of_ldx - temp);
                                s64 delta;

                                if (!bpf_prog->aux->extable)
                                        break;

                                if (excnt >= bpf_prog->aux->num_exentries) {
                                        pr_err("mem32 extable bug\n");
                                        return -EFAULT;
                                }
                                ex = &bpf_prog->aux->extable[excnt++];

                                delta = _insn - (u8 *)&ex->insn;
                                /* switch ex to rw buffer for writes */
                                ex = (void *)rw_image + ((void *)ex - (void *)image);

                                ex->insn = delta;

                                ex->data = EX_TYPE_BPF;

                                ex->fixup = (prog - start_of_ldx) |
                                        ((BPF_CLASS(insn->code) == BPF_LDX ? reg2pt_regs[dst_reg] : DONT_CLEAR) << 8);
                        }
                        break;

                        /* LDX: dst_reg = *(u8*)(src_reg + off) */
                case BPF_LDX | BPF_MEM | BPF_B:
                case BPF_LDX | BPF_PROBE_MEM | BPF_B:
                case BPF_LDX | BPF_MEM | BPF_H:
                case BPF_LDX | BPF_PROBE_MEM | BPF_H:
                case BPF_LDX | BPF_MEM | BPF_W:
                case BPF_LDX | BPF_PROBE_MEM | BPF_W:
                case BPF_LDX | BPF_MEM | BPF_DW:
                case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
                        /* LDXS: dst_reg = *(s8*)(src_reg + off) */
                case BPF_LDX | BPF_MEMSX | BPF_B:
                case BPF_LDX | BPF_MEMSX | BPF_H:
                case BPF_LDX | BPF_MEMSX | BPF_W:
                case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
                case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
                case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
                        insn_off = insn->off;

                        if (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
                            BPF_MODE(insn->code) == BPF_PROBE_MEMSX) {
                                /* Conservatively check that src_reg + insn->off is a kernel address:
                                 *   src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE
                                 *   and
                                 *   src_reg + insn->off < VSYSCALL_ADDR
                                 */

                                u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR;
                                u8 *end_of_jmp;

                                /* movabsq r10, VSYSCALL_ADDR */
                                emit_mov_imm64(&prog, BPF_REG_AX, (long)VSYSCALL_ADDR >> 32,
                                               (u32)(long)VSYSCALL_ADDR);

                                /* mov src_reg, r11 */
                                EMIT_mov(AUX_REG, src_reg);

                                if (insn->off) {
                                        /* add r11, insn->off */
                                        maybe_emit_1mod(&prog, AUX_REG, true);
                                        EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off);
                                }

                                /* sub r11, r10 */
                                maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true);
                                EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX));

                                /* movabsq r10, limit */
                                emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32,
                                               (u32)(long)limit);

                                /* cmp r10, r11 */
                                maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true);
                                EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX));

                                /* if unsigned '>', goto load */
                                EMIT2(X86_JA, 0);
                                end_of_jmp = prog;

                                /* xor dst_reg, dst_reg */
                                emit_mov_imm32(&prog, false, dst_reg, 0);
                                /* jmp byte_after_ldx */
                                EMIT2(0xEB, 0);

                                /* populate jmp_offset for JAE above to jump to start_of_ldx */
                                start_of_ldx = prog;
                                end_of_jmp[-1] = start_of_ldx - end_of_jmp;
                        }
                        if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
                            BPF_MODE(insn->code) == BPF_MEMSX)
                                emit_ldsx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);
                        else
                                emit_ldx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);
                        if (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
                            BPF_MODE(insn->code) == BPF_PROBE_MEMSX) {
                                struct exception_table_entry *ex;
                                u8 *_insn = image + proglen + (start_of_ldx - temp);
                                s64 delta;

                                /* populate jmp_offset for JMP above */
                                start_of_ldx[-1] = prog - start_of_ldx;

                                if (!bpf_prog->aux->extable)
                                        break;

                                if (excnt >= bpf_prog->aux->num_exentries) {
                                        pr_err("ex gen bug\n");
                                        return -EFAULT;
                                }
                                ex = &bpf_prog->aux->extable[excnt++];

                                delta = _insn - (u8 *)&ex->insn;
                                if (!is_simm32(delta)) {
                                        pr_err("extable->insn doesn't fit into 32-bit\n");
                                        return -EFAULT;
                                }
                                /* switch ex to rw buffer for writes */
                                ex = (void *)rw_image + ((void *)ex - (void *)image);

                                ex->insn = delta;

                                ex->data = EX_TYPE_BPF;

                                if (dst_reg > BPF_REG_9) {
                                        pr_err("verifier error\n");
                                        return -EFAULT;
                                }
                                /*
                                 * Compute size of x86 insn and its target dest x86 register.
                                 * ex_handler_bpf() will use lower 8 bits to adjust
                                 * pt_regs->ip to jump over this x86 instruction
                                 * and upper bits to figure out which pt_regs to zero out.
                                 * End result: x86 insn "mov rbx, qword ptr [rax+0x14]"
                                 * of 4 bytes will be ignored and rbx will be zero inited.
                                 */
                                ex->fixup = (prog - start_of_ldx) | (reg2pt_regs[dst_reg] << 8);
                        }
                        break;

                case BPF_STX | BPF_ATOMIC | BPF_W:
                case BPF_STX | BPF_ATOMIC | BPF_DW:
                        if (insn->imm == (BPF_AND | BPF_FETCH) ||
                            insn->imm == (BPF_OR | BPF_FETCH) ||
                            insn->imm == (BPF_XOR | BPF_FETCH)) {
                                bool is64 = BPF_SIZE(insn->code) == BPF_DW;
                                u32 real_src_reg = src_reg;
                                u32 real_dst_reg = dst_reg;
                                u8 *branch_target;

                                /*
                                 * Can't be implemented with a single x86 insn.
                                 * Need to do a CMPXCHG loop.
                                 */

                                /* Will need RAX as a CMPXCHG operand so save R0 */
                                emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0);
                                if (src_reg == BPF_REG_0)
                                        real_src_reg = BPF_REG_AX;
                                if (dst_reg == BPF_REG_0)
                                        real_dst_reg = BPF_REG_AX;

                                branch_target = prog;
                                /* Load old value */
                                emit_ldx(&prog, BPF_SIZE(insn->code),
                                         BPF_REG_0, real_dst_reg, insn->off);
                                /*
                                 * Perform the (commutative) operation locally,
                                 * put the result in the AUX_REG.
                                 */
                                emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0);
                                maybe_emit_mod(&prog, AUX_REG, real_src_reg, is64);
                                EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
                                      add_2reg(0xC0, AUX_REG, real_src_reg));
                                /* Attempt to swap in new value */
                                err = emit_atomic(&prog, BPF_CMPXCHG,
                                                  real_dst_reg, AUX_REG,
                                                  insn->off,
                                                  BPF_SIZE(insn->code));
                                if (WARN_ON(err))
                                        return err;
                                /*
                                 * ZF tells us whether we won the race. If it's
                                 * cleared we need to try again.
                                 */
                                EMIT2(X86_JNE, -(prog - branch_target) - 2);
                                /* Return the pre-modification value */
                                emit_mov_reg(&prog, is64, real_src_reg, BPF_REG_0);
                                /* Restore R0 after clobbering RAX */
                                emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX);
                                break;
                        }

                        err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
                                          insn->off, BPF_SIZE(insn->code));
                        if (err)
                                return err;
                        break;

                case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
                case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
                        start_of_ldx = prog;
                        err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),
                                                dst_reg, src_reg, X86_REG_R12, insn->off);
                        if (err)
                                return err;
                        goto populate_extable;

                        /* call */
                case BPF_JMP | BPF_CALL: {
                        u8 *ip = image + addrs[i - 1];

                        func = (u8 *) __bpf_call_base + imm32;
                        if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
                                LOAD_TAIL_CALL_CNT_PTR(stack_depth);
                                ip += 7;
                        }
                        if (!imm32)
                                return -EINVAL;
                        if (priv_frame_ptr) {
                                push_r9(&prog);
                                ip += 2;
                        }
                        ip += x86_call_depth_emit_accounting(&prog, func, ip);
                        if (emit_call(&prog, func, ip))
                                return -EINVAL;
                        if (priv_frame_ptr)
                                pop_r9(&prog);
                        break;
                }

                case BPF_JMP | BPF_TAIL_CALL:
                        if (imm32)
                                emit_bpf_tail_call_direct(bpf_prog,
                                                          &bpf_prog->aux->poke_tab[imm32 - 1],
                                                          &prog, image + addrs[i - 1],
                                                          callee_regs_used,
                                                          stack_depth,
                                                          ctx);
                        else
                                emit_bpf_tail_call_indirect(bpf_prog,
                                                            &prog,
                                                            callee_regs_used,
                                                            stack_depth,
                                                            image + addrs[i - 1],
                                                            ctx);
                        break;

                        /* cond jump */
                case BPF_JMP | BPF_JEQ | BPF_X:
                case BPF_JMP | BPF_JNE | BPF_X:
                case BPF_JMP | BPF_JGT | BPF_X:
                case BPF_JMP | BPF_JLT | BPF_X:
                case BPF_JMP | BPF_JGE | BPF_X:
                case BPF_JMP | BPF_JLE | BPF_X:
                case BPF_JMP | BPF_JSGT | BPF_X:
                case BPF_JMP | BPF_JSLT | BPF_X:
                case BPF_JMP | BPF_JSGE | BPF_X:
                case BPF_JMP | BPF_JSLE | BPF_X:
                case BPF_JMP32 | BPF_JEQ | BPF_X:
                case BPF_JMP32 | BPF_JNE | BPF_X:
                case BPF_JMP32 | BPF_JGT | BPF_X:
                case BPF_JMP32 | BPF_JLT | BPF_X:
                case BPF_JMP32 | BPF_JGE | BPF_X:
                case BPF_JMP32 | BPF_JLE | BPF_X:
                case BPF_JMP32 | BPF_JSGT | BPF_X:
                case BPF_JMP32 | BPF_JSLT | BPF_X:
                case BPF_JMP32 | BPF_JSGE | BPF_X:
                case BPF_JMP32 | BPF_JSLE | BPF_X:
                        /* cmp dst_reg, src_reg */
                        maybe_emit_mod(&prog, dst_reg, src_reg,
                                       BPF_CLASS(insn->code) == BPF_JMP);
                        EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));
                        goto emit_cond_jmp;

                case BPF_JMP | BPF_JSET | BPF_X:
                case BPF_JMP32 | BPF_JSET | BPF_X:
                        /* test dst_reg, src_reg */
                        maybe_emit_mod(&prog, dst_reg, src_reg,
                                       BPF_CLASS(insn->code) == BPF_JMP);
                        EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));
                        goto emit_cond_jmp;

                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP32 | BPF_JSET | BPF_K:
                        /* test dst_reg, imm32 */
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_JMP);
                        EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
                        goto emit_cond_jmp;

                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JNE | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JLT | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JLE | BPF_K:
                case BPF_JMP | BPF_JSGT | BPF_K:
                case BPF_JMP | BPF_JSLT | BPF_K:
                case BPF_JMP | BPF_JSGE | BPF_K:
                case BPF_JMP | BPF_JSLE | BPF_K:
                case BPF_JMP32 | BPF_JEQ | BPF_K:
                case BPF_JMP32 | BPF_JNE | BPF_K:
                case BPF_JMP32 | BPF_JGT | BPF_K:
                case BPF_JMP32 | BPF_JLT | BPF_K:
                case BPF_JMP32 | BPF_JGE | BPF_K:
                case BPF_JMP32 | BPF_JLE | BPF_K:
                case BPF_JMP32 | BPF_JSGT | BPF_K:
                case BPF_JMP32 | BPF_JSLT | BPF_K:
                case BPF_JMP32 | BPF_JSGE | BPF_K:
                case BPF_JMP32 | BPF_JSLE | BPF_K:
                        /* test dst_reg, dst_reg to save one extra byte */
                        if (imm32 == 0) {
                                maybe_emit_mod(&prog, dst_reg, dst_reg,
                                               BPF_CLASS(insn->code) == BPF_JMP);
                                EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg));
                                goto emit_cond_jmp;
                        }

                        /* cmp dst_reg, imm8/32 */
                        maybe_emit_1mod(&prog, dst_reg,
                                        BPF_CLASS(insn->code) == BPF_JMP);

                        if (is_imm8(imm32))
                                EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
                        else
                                EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);

emit_cond_jmp:                /* Convert BPF opcode to x86 */
                        switch (BPF_OP(insn->code)) {
                        case BPF_JEQ:
                                jmp_cond = X86_JE;
                                break;
                        case BPF_JSET:
                        case BPF_JNE:
                                jmp_cond = X86_JNE;
                                break;
                        case BPF_JGT:
                                /* GT is unsigned '>', JA in x86 */
                                jmp_cond = X86_JA;
                                break;
                        case BPF_JLT:
                                /* LT is unsigned '<', JB in x86 */
                                jmp_cond = X86_JB;
                                break;
                        case BPF_JGE:
                                /* GE is unsigned '>=', JAE in x86 */
                                jmp_cond = X86_JAE;
                                break;
                        case BPF_JLE:
                                /* LE is unsigned '<=', JBE in x86 */
                                jmp_cond = X86_JBE;
                                break;
                        case BPF_JSGT:
                                /* Signed '>', GT in x86 */
                                jmp_cond = X86_JG;
                                break;
                        case BPF_JSLT:
                                /* Signed '<', LT in x86 */
                                jmp_cond = X86_JL;
                                break;
                        case BPF_JSGE:
                                /* Signed '>=', GE in x86 */
                                jmp_cond = X86_JGE;
                                break;
                        case BPF_JSLE:
                                /* Signed '<=', LE in x86 */
                                jmp_cond = X86_JLE;
                                break;
                        default: /* to silence GCC warning */
                                return -EFAULT;
                        }
                        jmp_offset = addrs[i + insn->off] - addrs[i];
                        if (is_imm8_jmp_offset(jmp_offset)) {
                                if (jmp_padding) {
                                        /* To keep the jmp_offset valid, the extra bytes are
                                         * padded before the jump insn, so we subtract the
                                         * 2 bytes of jmp_cond insn from INSN_SZ_DIFF.
                                         *
                                         * If the previous pass already emits an imm8
                                         * jmp_cond, then this BPF insn won't shrink, so
                                         * "nops" is 0.
                                         *
                                         * On the other hand, if the previous pass emits an
                                         * imm32 jmp_cond, the extra 4 bytes(*) is padded to
                                         * keep the image from shrinking further.
                                         *
                                         * (*) imm32 jmp_cond is 6 bytes, and imm8 jmp_cond
                                         *     is 2 bytes, so the size difference is 4 bytes.
                                         */
                                        nops = INSN_SZ_DIFF - 2;
                                        if (nops != 0 && nops != 4) {
                                                pr_err("unexpected jmp_cond padding: %d bytes\n",
                                                       nops);
                                                return -EFAULT;
                                        }
                                        emit_nops(&prog, nops);
                                }
                                EMIT2(jmp_cond, jmp_offset);
                        } else if (is_simm32(jmp_offset)) {
                                EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
                        } else {
                                pr_err("cond_jmp gen bug %llx\n", jmp_offset);
                                return -EFAULT;
                        }

                        break;

                case BPF_JMP | BPF_JA:
                case BPF_JMP32 | BPF_JA:
                        if (BPF_CLASS(insn->code) == BPF_JMP) {
                                if (insn->off == -1)
                                        /* -1 jmp instructions will always jump
                                         * backwards two bytes. Explicitly handling
                                         * this case avoids wasting too many passes
                                         * when there are long sequences of replaced
                                         * dead code.
                                         */
                                        jmp_offset = -2;
                                else
                                        jmp_offset = addrs[i + insn->off] - addrs[i];
                        } else {
                                if (insn->imm == -1)
                                        jmp_offset = -2;
                                else
                                        jmp_offset = addrs[i + insn->imm] - addrs[i];
                        }

                        if (!jmp_offset) {
                                /*
                                 * If jmp_padding is enabled, the extra nops will
                                 * be inserted. Otherwise, optimize out nop jumps.
                                 */
                                if (jmp_padding) {
                                        /* There are 3 possible conditions.
                                         * (1) This BPF_JA is already optimized out in
                                         *     the previous run, so there is no need
                                         *     to pad any extra byte (0 byte).
                                         * (2) The previous pass emits an imm8 jmp,
                                         *     so we pad 2 bytes to match the previous
                                         *     insn size.
                                         * (3) Similarly, the previous pass emits an
                                         *     imm32 jmp, and 5 bytes is padded.
                                         */
                                        nops = INSN_SZ_DIFF;
                                        if (nops != 0 && nops != 2 && nops != 5) {
                                                pr_err("unexpected nop jump padding: %d bytes\n",
                                                       nops);
                                                return -EFAULT;
                                        }
                                        emit_nops(&prog, nops);
                                }
                                break;
                        }
emit_jmp:
                        if (is_imm8_jmp_offset(jmp_offset)) {
                                if (jmp_padding) {
                                        /* To avoid breaking jmp_offset, the extra bytes
                                         * are padded before the actual jmp insn, so
                                         * 2 bytes is subtracted from INSN_SZ_DIFF.
                                         *
                                         * If the previous pass already emits an imm8
                                         * jmp, there is nothing to pad (0 byte).
                                         *
                                         * If it emits an imm32 jmp (5 bytes) previously
                                         * and now an imm8 jmp (2 bytes), then we pad
                                         * (5 - 2 = 3) bytes to stop the image from
                                         * shrinking further.
                                         */
                                        nops = INSN_SZ_DIFF - 2;
                                        if (nops != 0 && nops != 3) {
                                                pr_err("unexpected jump padding: %d bytes\n",
                                                       nops);
                                                return -EFAULT;
                                        }
                                        emit_nops(&prog, INSN_SZ_DIFF - 2);
                                }
                                EMIT2(0xEB, jmp_offset);
                        } else if (is_simm32(jmp_offset)) {
                                EMIT1_off32(0xE9, jmp_offset);
                        } else {
                                pr_err("jmp gen bug %llx\n", jmp_offset);
                                return -EFAULT;
                        }
                        break;

                case BPF_JMP | BPF_EXIT:
                        if (seen_exit) {
                                jmp_offset = ctx->cleanup_addr - addrs[i];
                                goto emit_jmp;
                        }
                        seen_exit = true;
                        /* Update cleanup_addr */
                        ctx->cleanup_addr = proglen;
                        if (bpf_prog->aux->exception_boundary) {
                                pop_callee_regs(&prog, all_callee_regs_used);
                                pop_r12(&prog);
                        } else {
                                pop_callee_regs(&prog, callee_regs_used);
                                if (arena_vm_start)
                                        pop_r12(&prog);
                        }
                        EMIT1(0xC9);         /* leave */
                        emit_return(&prog, image + addrs[i - 1] + (prog - temp));
                        break;

                default:
                        /*
                         * By design x86-64 JIT should support all BPF instructions.
                         * This error will be seen if new instruction was added
                         * to the interpreter, but not to the JIT, or if there is
                         * junk in bpf_prog.
                         */
                        pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
                        return -EINVAL;
                }

                ilen = prog - temp;
                if (ilen > BPF_MAX_INSN_SIZE) {
                        pr_err("bpf_jit: fatal insn size error\n");
                        return -EFAULT;
                }

                if (image) {
                        /*
                         * When populating the image, assert that:
                         *
                         *  i) We do not write beyond the allocated space, and
                         * ii) addrs[i] did not change from the prior run, in order
                         *     to validate assumptions made for computing branch
                         *     displacements.
                         */
                        if (unlikely(proglen + ilen > oldproglen ||
                                     proglen + ilen != addrs[i])) {
                                pr_err("bpf_jit: fatal error\n");
                                return -EFAULT;
                        }
                        memcpy(rw_image + proglen, temp, ilen);
                }
                proglen += ilen;
                addrs[i] = proglen;
                prog = temp;
        }

        if (image && excnt != bpf_prog->aux->num_exentries) {
                pr_err("extable is not populated\n");
                return -EFAULT;
        }
        return proglen;
}

static void clean_stack_garbage(const struct btf_func_model *m,
                                u8 **pprog, int nr_stack_slots,
                                int stack_size)
{
        int arg_size, off;
        u8 *prog;

        /* Generally speaking, the compiler will pass the arguments
         * on-stack with "push" instruction, which will take 8-byte
         * on the stack. In this case, there won't be garbage values
         * while we copy the arguments from origin stack frame to current
         * in BPF_DW.
         *
         * However, sometimes the compiler will only allocate 4-byte on
         * the stack for the arguments. For now, this case will only
         * happen if there is only one argument on-stack and its size
         * not more than 4 byte. In this case, there will be garbage
         * values on the upper 4-byte where we store the argument on
         * current stack frame.
         *
         * arguments on origin stack:
         *
         * stack_arg_1(4-byte) xxx(4-byte)
         *
         * what we copy:
         *
         * stack_arg_1(8-byte): stack_arg_1(origin) xxx
         *
         * and the xxx is the garbage values which we should clean here.
         */
        if (nr_stack_slots != 1)
                return;

        /* the size of the last argument */
        arg_size = m->arg_size[m->nr_args - 1];
        if (arg_size <= 4) {
                off = -(stack_size - 4);
                prog = *pprog;
                /* mov DWORD PTR [rbp + off], 0 */
                if (!is_imm8(off))
                        EMIT2_off32(0xC7, 0x85, off);
                else
                        EMIT3(0xC7, 0x45, off);
                EMIT(0, 4);
                *pprog = prog;
        }
}

/* get the count of the regs that are used to pass arguments */
static int get_nr_used_regs(const struct btf_func_model *m)
{
        int i, arg_regs, nr_used_regs = 0;

        for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
                arg_regs = (m->arg_size[i] + 7) / 8;
                if (nr_used_regs + arg_regs <= 6)
                        nr_used_regs += arg_regs;

                if (nr_used_regs >= 6)
                        break;
        }

        return nr_used_regs;
}

static void save_args(const struct btf_func_model *m, u8 **prog,
                      int stack_size, bool for_call_origin)
{
        int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0;
        int i, j;

        /* Store function arguments to stack.
         * For a function that accepts two pointers the sequence will be:
         * mov QWORD PTR [rbp-0x10],rdi
         * mov QWORD PTR [rbp-0x8],rsi
         */
        for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
                arg_regs = (m->arg_size[i] + 7) / 8;

                /* According to the research of Yonghong, struct members
                 * should be all in register or all on the stack.
                 * Meanwhile, the compiler will pass the argument on regs
                 * if the remaining regs can hold the argument.
                 *
                 * Disorder of the args can happen. For example:
                 *
                 * struct foo_struct {
                 *     long a;
                 *     int b;
                 * };
                 * int foo(char, char, char, char, char, struct foo_struct,
                 *         char);
                 *
                 * the arg1-5,arg7 will be passed by regs, and arg6 will
                 * by stack.
                 */
                if (nr_regs + arg_regs > 6) {
                        /* copy function arguments from origin stack frame
                         * into current stack frame.
                         *
                         * The starting address of the arguments on-stack
                         * is:
                         *   rbp + 8(push rbp) +
                         *   8(return addr of origin call) +
                         *   8(return addr of the caller)
                         * which means: rbp + 24
                         */
                        for (j = 0; j < arg_regs; j++) {
                                emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
                                         nr_stack_slots * 8 + 0x18);
                                emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0,
                                         -stack_size);

                                if (!nr_stack_slots)
                                        first_off = stack_size;
                                stack_size -= 8;
                                nr_stack_slots++;
                        }
                } else {
                        /* Only copy the arguments on-stack to current
                         * 'stack_size' and ignore the regs, used to
                         * prepare the arguments on-stack for origin call.
                         */
                        if (for_call_origin) {
                                nr_regs += arg_regs;
                                continue;
                        }

                        /* copy the arguments from regs into stack */
                        for (j = 0; j < arg_regs; j++) {
                                emit_stx(prog, BPF_DW, BPF_REG_FP,
                                         nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs,
                                         -stack_size);
                                stack_size -= 8;
                                nr_regs++;
                        }
                }
        }

        clean_stack_garbage(m, prog, nr_stack_slots, first_off);
}

static void restore_regs(const struct btf_func_model *m, u8 **prog,
                         int stack_size)
{
        int i, j, arg_regs, nr_regs = 0;

        /* Restore function arguments from stack.
         * For a function that accepts two pointers the sequence will be:
         * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10]
         * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8]
         *
         * The logic here is similar to what we do in save_args()
         */
        for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
                arg_regs = (m->arg_size[i] + 7) / 8;
                if (nr_regs + arg_regs <= 6) {
                        for (j = 0; j < arg_regs; j++) {
                                emit_ldx(prog, BPF_DW,
                                         nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs,
                                         BPF_REG_FP,
                                         -stack_size);
                                stack_size -= 8;
                                nr_regs++;
                        }
                } else {
                        stack_size -= 8 * arg_regs;
                }

                if (nr_regs >= 6)
                        break;
        }
}

static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
                           struct bpf_tramp_link *l, int stack_size,
                           int run_ctx_off, bool save_ret,
                           void *image, void *rw_image)
{
        u8 *prog = *pprog;
        u8 *jmp_insn;
        int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
        struct bpf_prog *p = l->link.prog;
        u64 cookie = l->cookie;

        /* mov rdi, cookie */
        emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);

        /* Prepare struct bpf_tramp_run_ctx.
         *
         * bpf_tramp_run_ctx is already preserved by
         * arch_prepare_bpf_trampoline().
         *
         * mov QWORD PTR [rbp - run_ctx_off + ctx_cookie_off], rdi
         */
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);

        /* arg1: mov rdi, progs[i] */
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
        /* arg2: lea rsi, [rbp - ctx_cookie_off] */
        if (!is_imm8(-run_ctx_off))
                EMIT3_off32(0x48, 0x8D, 0xB5, -run_ctx_off);
        else
                EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);

        if (emit_rsb_call(&prog, bpf_trampoline_enter(p), image + (prog - (u8 *)rw_image)))
                return -EINVAL;
        /* remember prog start time returned by __bpf_prog_enter */
        emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);

        /* if (__bpf_prog_enter*(prog) == 0)
         *        goto skip_exec_of_prog;
         */
        EMIT3(0x48, 0x85, 0xC0);  /* test rax,rax */
        /* emit 2 nops that will be replaced with JE insn */
        jmp_insn = prog;
        emit_nops(&prog, 2);

        /* arg1: lea rdi, [rbp - stack_size] */
        if (!is_imm8(-stack_size))
                EMIT3_off32(0x48, 0x8D, 0xBD, -stack_size);
        else
                EMIT4(0x48, 0x8D, 0x7D, -stack_size);
        /* arg2: progs[i]->insnsi for interpreter */
        if (!p->jited)
                emit_mov_imm64(&prog, BPF_REG_2,
                               (long) p->insnsi >> 32,
                               (u32) (long) p->insnsi);
        /* call JITed bpf program or interpreter */
        if (emit_rsb_call(&prog, p->bpf_func, image + (prog - (u8 *)rw_image)))
                return -EINVAL;

        /*
         * BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
         * of the previous call which is then passed on the stack to
         * the next BPF program.
         *
         * BPF_TRAMP_FENTRY trampoline may need to return the return
         * value of BPF_PROG_TYPE_STRUCT_OPS prog.
         */
        if (save_ret)
                emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);

        /* replace 2 nops with JE insn, since jmp target is known */
        jmp_insn[0] = X86_JE;
        jmp_insn[1] = prog - jmp_insn - 2;

        /* arg1: mov rdi, progs[i] */
        emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
        /* arg2: mov rsi, rbx <- start time in nsec */
        emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
        /* arg3: lea rdx, [rbp - run_ctx_off] */
        if (!is_imm8(-run_ctx_off))
                EMIT3_off32(0x48, 0x8D, 0x95, -run_ctx_off);
        else
                EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
        if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image)))
                return -EINVAL;

        *pprog = prog;
        return 0;
}

static void emit_align(u8 **pprog, u32 align)
{
        u8 *target, *prog = *pprog;

        target = PTR_ALIGN(prog, align);
        if (target != prog)
                emit_nops(&prog, target - prog);

        *pprog = prog;
}

static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
{
        u8 *prog = *pprog;
        s64 offset;

        offset = func - (ip + 2 + 4);
        if (!is_simm32(offset)) {
                pr_err("Target %p is out of range\n", func);
                return -EINVAL;
        }
        EMIT2_off32(0x0F, jmp_cond + 0x10, offset);
        *pprog = prog;
        return 0;
}

static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
                      struct bpf_tramp_links *tl, int stack_size,
                      int run_ctx_off, bool save_ret,
                      void *image, void *rw_image)
{
        int i;
        u8 *prog = *pprog;

        for (i = 0; i < tl->nr_links; i++) {
                if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
                                    run_ctx_off, save_ret, image, rw_image))
                        return -EINVAL;
        }
        *pprog = prog;
        return 0;
}

static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
                              struct bpf_tramp_links *tl, int stack_size,
                              int run_ctx_off, u8 **branches,
                              void *image, void *rw_image)
{
        u8 *prog = *pprog;
        int i;

        /* The first fmod_ret program will receive a garbage return value.
         * Set this to 0 to avoid confusing the program.
         */
        emit_mov_imm32(&prog, false, BPF_REG_0, 0);
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
        for (i = 0; i < tl->nr_links; i++) {
                if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true,
                                    image, rw_image))
                        return -EINVAL;

                /* mod_ret prog stored return value into [rbp - 8]. Emit:
                 * if (*(u64 *)(rbp - 8) !=  0)
                 *        goto do_fexit;
                 */
                /* cmp QWORD PTR [rbp - 0x8], 0x0 */
                EMIT4(0x48, 0x83, 0x7d, 0xf8); EMIT1(0x00);

                /* Save the location of the branch and Generate 6 nops
                 * (4 bytes for an offset and 2 bytes for the jump) These nops
                 * are replaced with a conditional jump once do_fexit (i.e. the
                 * start of the fexit invocation) is finalized.
                 */
                branches[i] = prog;
                emit_nops(&prog, 4 + 2);
        }

        *pprog = prog;
        return 0;
}

/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack)        \
        __LOAD_TCC_PTR(-round_up(stack, 8) - 8)

/* Example:
 * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev);
 * its 'struct btf_func_model' will be nr_args=2
 * The assembly code when eth_type_trans is executing after trampoline:
 *
 * push rbp
 * mov rbp, rsp
 * sub rsp, 16                     // space for skb and dev
 * push rbx                        // temp regs to pass start time
 * mov qword ptr [rbp - 16], rdi   // save skb pointer to stack
 * mov qword ptr [rbp - 8], rsi    // save dev pointer to stack
 * call __bpf_prog_enter           // rcu_read_lock and preempt_disable
 * mov rbx, rax                    // remember start time in bpf stats are enabled
 * lea rdi, [rbp - 16]             // R1==ctx of bpf prog
 * call addr_of_jited_FENTRY_prog
 * movabsq rdi, 64bit_addr_of_struct_bpf_prog  // unused if bpf stats are off
 * mov rsi, rbx                    // prog start time
 * call __bpf_prog_exit            // rcu_read_unlock, preempt_enable and stats math
 * mov rdi, qword ptr [rbp - 16]   // restore skb pointer from stack
 * mov rsi, qword ptr [rbp - 8]    // restore dev pointer from stack
 * pop rbx
 * leave
 * ret
 *
 * eth_type_trans has 5 byte nop at the beginning. These 5 bytes will be
 * replaced with 'call generated_bpf_trampoline'. When it returns
 * eth_type_trans will continue executing with original skb and dev pointers.
 *
 * The assembly code when eth_type_trans is called from trampoline:
 *
 * push rbp
 * mov rbp, rsp
 * sub rsp, 24                     // space for skb, dev, return value
 * push rbx                        // temp regs to pass start time
 * mov qword ptr [rbp - 24], rdi   // save skb pointer to stack
 * mov qword ptr [rbp - 16], rsi   // save dev pointer to stack
 * call __bpf_prog_enter           // rcu_read_lock and preempt_disable
 * mov rbx, rax                    // remember start time if bpf stats are enabled
 * lea rdi, [rbp - 24]             // R1==ctx of bpf prog
 * call addr_of_jited_FENTRY_prog  // bpf prog can access skb and dev
 * movabsq rdi, 64bit_addr_of_struct_bpf_prog  // unused if bpf stats are off
 * mov rsi, rbx                    // prog start time
 * call __bpf_prog_exit            // rcu_read_unlock, preempt_enable and stats math
 * mov rdi, qword ptr [rbp - 24]   // restore skb pointer from stack
 * mov rsi, qword ptr [rbp - 16]   // restore dev pointer from stack
 * call eth_type_trans+5           // execute body of eth_type_trans
 * mov qword ptr [rbp - 8], rax    // save return value
 * call __bpf_prog_enter           // rcu_read_lock and preempt_disable
 * mov rbx, rax                    // remember start time in bpf stats are enabled
 * lea rdi, [rbp - 24]             // R1==ctx of bpf prog
 * call addr_of_jited_FEXIT_prog   // bpf prog can access skb, dev, return value
 * movabsq rdi, 64bit_addr_of_struct_bpf_prog  // unused if bpf stats are off
 * mov rsi, rbx                    // prog start time
 * call __bpf_prog_exit            // rcu_read_unlock, preempt_enable and stats math
 * mov rax, qword ptr [rbp - 8]    // restore eth_type_trans's return value
 * pop rbx
 * leave
 * add rsp, 8                      // skip eth_type_trans's frame
 * ret                             // return to its caller
 */
static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_image,
                                         void *rw_image_end, void *image,
                                         const struct btf_func_model *m, u32 flags,
                                         struct bpf_tramp_links *tlinks,
                                         void *func_addr)
{
        int i, ret, nr_regs = m->nr_args, stack_size = 0;
        int regs_off, nregs_off, ip_off, run_ctx_off, arg_stack_off, rbx_off;
        struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
        struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
        struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
        void *orig_call = func_addr;
        u8 **branches = NULL;
        u8 *prog;
        bool save_ret;

        /*
         * F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is
         * explicitly incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG
         * because @func_addr.
         */
        WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
                     (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));

        /* extra registers for struct arguments */
        for (i = 0; i < m->nr_args; i++) {
                if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
                        nr_regs += (m->arg_size[i] + 7) / 8 - 1;
        }

        /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6
         * are passed through regs, the remains are through stack.
         */
        if (nr_regs > MAX_BPF_FUNC_ARGS)
                return -ENOTSUPP;

        /* Generated trampoline stack layout:
         *
         * RBP + 8         [ return address  ]
         * RBP + 0         [ RBP             ]
         *
         * RBP - 8         [ return value    ]  BPF_TRAMP_F_CALL_ORIG or
         *                                      BPF_TRAMP_F_RET_FENTRY_RET flags
         *
         *                 [ reg_argN        ]  always
         *                 [ ...             ]
         * RBP - regs_off  [ reg_arg1        ]  program's ctx pointer
         *
         * RBP - nregs_off [ regs count             ]  always
         *
         * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag
         *
         * RBP - rbx_off   [ rbx value       ]  always
         *
         * RBP - run_ctx_off [ bpf_tramp_run_ctx ]
         *
         *                     [ stack_argN ]  BPF_TRAMP_F_CALL_ORIG
         *                     [ ...        ]
         *                     [ stack_arg2 ]
         * RBP - arg_stack_off [ stack_arg1 ]
         * RSP                 [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX
         */

        /* room for return value of orig_call or fentry prog */
        save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
        if (save_ret)
                stack_size += 8;

        stack_size += nr_regs * 8;
        regs_off = stack_size;

        /* regs count  */
        stack_size += 8;
        nregs_off = stack_size;

        if (flags & BPF_TRAMP_F_IP_ARG)
                stack_size += 8; /* room for IP address argument */

        ip_off = stack_size;

        stack_size += 8;
        rbx_off = stack_size;

        stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7;
        run_ctx_off = stack_size;

        if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) {
                /* the space that used to pass arguments on-stack */
                stack_size += (nr_regs - get_nr_used_regs(m)) * 8;
                /* make sure the stack pointer is 16-byte aligned if we
                 * need pass arguments on stack, which means
                 *  [stack_size + 8(rbp) + 8(rip) + 8(origin rip)]
                 * should be 16-byte aligned. Following code depend on
                 * that stack_size is already 8-byte aligned.
                 */
                stack_size += (stack_size % 16) ? 0 : 8;
        }

        arg_stack_off = stack_size;

        if (flags & BPF_TRAMP_F_SKIP_FRAME) {
                /* skip patched call instruction and point orig_call to actual
                 * body of the kernel function.
                 */
                if (is_endbr(*(u32 *)orig_call))
                        orig_call += ENDBR_INSN_SIZE;
                orig_call += X86_PATCH_SIZE;
        }

        prog = rw_image;

        if (flags & BPF_TRAMP_F_INDIRECT) {
                /*
                 * Indirect call for bpf_struct_ops
                 */
                emit_cfi(&prog, cfi_get_func_hash(func_addr));
        } else {
                /*
                 * Direct-call fentry stub, as such it needs accounting for the
                 * __fentry__ call.
                 */
                x86_call_depth_emit_accounting(&prog, NULL, image);
        }
        EMIT1(0x55);                 /* push rbp */
        EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
        if (!is_imm8(stack_size)) {
                /* sub rsp, stack_size */
                EMIT3_off32(0x48, 0x81, 0xEC, stack_size);
        } else {
                /* sub rsp, stack_size */
                EMIT4(0x48, 0x83, 0xEC, stack_size);
        }
        if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
                EMIT1(0x50);                /* push rax */
        /* mov QWORD PTR [rbp - rbx_off], rbx */
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);

        /* Store number of argument registers of the traced function:
         *   mov rax, nr_regs
         *   mov QWORD PTR [rbp - nregs_off], rax
         */
        emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_regs);
        emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -nregs_off);

        if (flags & BPF_TRAMP_F_IP_ARG) {
                /* Store IP address of the traced function:
                 * movabsq rax, func_addr
                 * mov QWORD PTR [rbp - ip_off], rax
                 */
                emit_mov_imm64(&prog, BPF_REG_0, (long) func_addr >> 32, (u32) (long) func_addr);
                emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off);
        }

        save_args(m, &prog, regs_off, false);

        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                /* arg1: mov rdi, im */
                emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
                if (emit_rsb_call(&prog, __bpf_tramp_enter,
                                  image + (prog - (u8 *)rw_image))) {
                        ret = -EINVAL;
                        goto cleanup;
                }
        }

        if (fentry->nr_links) {
                if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off,
                               flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image))
                        return -EINVAL;
        }

        if (fmod_ret->nr_links) {
                branches = kcalloc(fmod_ret->nr_links, sizeof(u8 *),
                                   GFP_KERNEL);
                if (!branches)
                        return -ENOMEM;

                if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,
                                       run_ctx_off, branches, image, rw_image)) {
                        ret = -EINVAL;
                        goto cleanup;
                }
        }

        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                restore_regs(m, &prog, regs_off);
                save_args(m, &prog, arg_stack_off, true);

                if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
                        /* Before calling the original function, load the
                         * tail_call_cnt_ptr from stack to rax.
                         */
                        LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
                }

                if (flags & BPF_TRAMP_F_ORIG_STACK) {
                        emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, 8);
                        EMIT2(0xff, 0xd3); /* call *rbx */
                } else {
                        /* call original function */
                        if (emit_rsb_call(&prog, orig_call, image + (prog - (u8 *)rw_image))) {
                                ret = -EINVAL;
                                goto cleanup;
                        }
                }
                /* remember return value in a stack for bpf prog to access */
                emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
                im->ip_after_call = image + (prog - (u8 *)rw_image);
                emit_nops(&prog, X86_PATCH_SIZE);
        }

        if (fmod_ret->nr_links) {
                /* From Intel 64 and IA-32 Architectures Optimization
                 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
                 * Coding Rule 11: All branch targets should be 16-byte
                 * aligned.
                 */
                emit_align(&prog, 16);
                /* Update the branches saved in invoke_bpf_mod_ret with the
                 * aligned address of do_fexit.
                 */
                for (i = 0; i < fmod_ret->nr_links; i++) {
                        emit_cond_near_jump(&branches[i], image + (prog - (u8 *)rw_image),
                                            image + (branches[i] - (u8 *)rw_image), X86_JNE);
                }
        }

        if (fexit->nr_links) {
                if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off,
                               false, image, rw_image)) {
                        ret = -EINVAL;
                        goto cleanup;
                }
        }

        if (flags & BPF_TRAMP_F_RESTORE_REGS)
                restore_regs(m, &prog, regs_off);

        /* This needs to be done regardless. If there were fmod_ret programs,
         * the return value is only updated on the stack and still needs to be
         * restored to R0.
         */
        if (flags & BPF_TRAMP_F_CALL_ORIG) {
                im->ip_epilogue = image + (prog - (u8 *)rw_image);
                /* arg1: mov rdi, im */
                emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
                if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) {
                        ret = -EINVAL;
                        goto cleanup;
                }
        } else if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
                /* Before running the original function, load the
                 * tail_call_cnt_ptr from stack to rax.
                 */
                LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
        }

        /* restore return value of orig_call or fentry prog back into RAX */
        if (save_ret)
                emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);

        emit_ldx(&prog, BPF_DW, BPF_REG_6, BPF_REG_FP, -rbx_off);
        EMIT1(0xC9); /* leave */
        if (flags & BPF_TRAMP_F_SKIP_FRAME) {
                /* skip our return address and return to parent */
                EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
        }
        emit_return(&prog, image + (prog - (u8 *)rw_image));
        /* Make sure the trampoline generation logic doesn't overflow */
        if (WARN_ON_ONCE(prog > (u8 *)rw_image_end - BPF_INSN_SAFETY)) {
                ret = -EFAULT;
                goto cleanup;
        }
        ret = prog - (u8 *)rw_image + BPF_INSN_SAFETY;

cleanup:
        kfree(branches);
        return ret;
}

void *arch_alloc_bpf_trampoline(unsigned int size)
{
        return bpf_prog_pack_alloc(size, jit_fill_hole);
}

void arch_free_bpf_trampoline(void *image, unsigned int size)
{
        bpf_prog_pack_free(image, size);
}

int arch_protect_bpf_trampoline(void *image, unsigned int size)
{
        return 0;
}

int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
                                const struct btf_func_model *m, u32 flags,
                                struct bpf_tramp_links *tlinks,
                                void *func_addr)
{
        void *rw_image, *tmp;
        int ret;
        u32 size = image_end - image;

        /* rw_image doesn't need to be in module memory range, so we can
         * use kvmalloc.
         */
        rw_image = kvmalloc(size, GFP_KERNEL);
        if (!rw_image)
                return -ENOMEM;

        ret = __arch_prepare_bpf_trampoline(im, rw_image, rw_image + size, image, m,
                                            flags, tlinks, func_addr);
        if (ret < 0)
                goto out;

        tmp = bpf_arch_text_copy(image, rw_image, size);
        if (IS_ERR(tmp))
                ret = PTR_ERR(tmp);
out:
        kvfree(rw_image);
        return ret;
}

int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
                             struct bpf_tramp_links *tlinks, void *func_addr)
{
        struct bpf_tramp_image im;
        void *image;
        int ret;

        /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline().
         * This will NOT cause fragmentation in direct map, as we do not
         * call set_memory_*() on this buffer.
         *
         * We cannot use kvmalloc here, because we need image to be in
         * module memory range.
         */
        image = bpf_jit_alloc_exec(PAGE_SIZE);
        if (!image)
                return -ENOMEM;

        ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image,
                                            m, flags, tlinks, func_addr);
        bpf_jit_free_exec(image);
        return ret;
}

static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)
{
        u8 *jg_reloc, *prog = *pprog;
        int pivot, err, jg_bytes = 1;
        s64 jg_offset;

        if (a == b) {
                /* Leaf node of recursion, i.e. not a range of indices
                 * anymore.
                 */
                EMIT1(add_1mod(0x48, BPF_REG_3));        /* cmp rdx,func */
                if (!is_simm32(progs[a]))
                        return -1;
                EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3),
                            progs[a]);
                err = emit_cond_near_jump(&prog,        /* je func */
                                          (void *)progs[a], image + (prog - buf),
                                          X86_JE);
                if (err)
                        return err;

                emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf));

                *pprog = prog;
                return 0;
        }

        /* Not a leaf node, so we pivot, and recursively descend into
         * the lower and upper ranges.
         */
        pivot = (b - a) / 2;
        EMIT1(add_1mod(0x48, BPF_REG_3));                /* cmp rdx,func */
        if (!is_simm32(progs[a + pivot]))
                return -1;
        EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a + pivot]);

        if (pivot > 2) {                                /* jg upper_part */
                /* Require near jump. */
                jg_bytes = 4;
                EMIT2_off32(0x0F, X86_JG + 0x10, 0);
        } else {
                EMIT2(X86_JG, 0);
        }
        jg_reloc = prog;

        err = emit_bpf_dispatcher(&prog, a, a + pivot,        /* emit lower_part */
                                  progs, image, buf);
        if (err)
                return err;

        /* From Intel 64 and IA-32 Architectures Optimization
         * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
         * Coding Rule 11: All branch targets should be 16-byte
         * aligned.
         */
        emit_align(&prog, 16);
        jg_offset = prog - jg_reloc;
        emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes);

        err = emit_bpf_dispatcher(&prog, a + pivot + 1,        /* emit upper_part */
                                  b, progs, image, buf);
        if (err)
                return err;

        *pprog = prog;
        return 0;
}

static int cmp_ips(const void *a, const void *b)
{
        const s64 *ipa = a;
        const s64 *ipb = b;

        if (*ipa > *ipb)
                return 1;
        if (*ipa < *ipb)
                return -1;
        return 0;
}

int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
{
        u8 *prog = buf;

        sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL);
        return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);
}

static const char *bpf_get_prog_name(struct bpf_prog *prog)
{
        if (prog->aux->ksym.prog)
                return prog->aux->ksym.name;
        return prog->aux->name;
}

static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
{
        int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
        u64 *stack_ptr;

        for_each_possible_cpu(cpu) {
                stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
                stack_ptr[0] = PRIV_STACK_GUARD_VAL;
                stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
        }
}

static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
                                   struct bpf_prog *prog)
{
        int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
        u64 *stack_ptr;

        for_each_possible_cpu(cpu) {
                stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
                if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
                    stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL) {
                        pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
                               bpf_get_prog_name(prog));
                        break;
                }
        }
}

struct x64_jit_data {
        struct bpf_binary_header *rw_header;
        struct bpf_binary_header *header;
        int *addrs;
        u8 *image;
        int proglen;
        struct jit_context ctx;
};

#define MAX_PASSES 20
#define PADDING_PASSES (MAX_PASSES - 5)

struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{
        struct bpf_binary_header *rw_header = NULL;
        struct bpf_binary_header *header = NULL;
        struct bpf_prog *tmp, *orig_prog = prog;
        void __percpu *priv_stack_ptr = NULL;
        struct x64_jit_data *jit_data;
        int priv_stack_alloc_sz;
        int proglen, oldproglen = 0;
        struct jit_context ctx = {};
        bool tmp_blinded = false;
        bool extra_pass = false;
        bool padding = false;
        u8 *rw_image = NULL;
        u8 *image = NULL;
        int *addrs;
        int pass;
        int i;

        if (!prog->jit_requested)
                return orig_prog;

        tmp = bpf_jit_blind_constants(prog);
        /*
         * If blinding was requested and we failed during blinding,
         * we must fall back to the interpreter.
         */
        if (IS_ERR(tmp))
                return orig_prog;
        if (tmp != prog) {
                tmp_blinded = true;
                prog = tmp;
        }

        jit_data = prog->aux->jit_data;
        if (!jit_data) {
                jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
                if (!jit_data) {
                        prog = orig_prog;
                        goto out;
                }
                prog->aux->jit_data = jit_data;
        }
        priv_stack_ptr = prog->aux->priv_stack_ptr;
        if (!priv_stack_ptr && prog->aux->jits_use_priv_stack) {
                /* Allocate actual private stack size with verifier-calculated
                 * stack size plus two memory guards to protect overflow and
                 * underflow.
                 */
                priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
                                      2 * PRIV_STACK_GUARD_SZ;
                priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);
                if (!priv_stack_ptr) {
                        prog = orig_prog;
                        goto out_priv_stack;
                }

                priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
                prog->aux->priv_stack_ptr = priv_stack_ptr;
        }
        addrs = jit_data->addrs;
        if (addrs) {
                ctx = jit_data->ctx;
                oldproglen = jit_data->proglen;
                image = jit_data->image;
                header = jit_data->header;
                rw_header = jit_data->rw_header;
                rw_image = (void *)rw_header + ((void *)image - (void *)header);
                extra_pass = true;
                padding = true;
                goto skip_init_addrs;
        }
        addrs = kvmalloc_array(prog->len + 1, sizeof(*addrs), GFP_KERNEL);
        if (!addrs) {
                prog = orig_prog;
                goto out_addrs;
        }

        /*
         * Before first pass, make a rough estimation of addrs[]
         * each BPF instruction is translated to less than 64 bytes
         */
        for (proglen = 0, i = 0; i <= prog->len; i++) {
                proglen += 64;
                addrs[i] = proglen;
        }
        ctx.cleanup_addr = proglen;
skip_init_addrs:

        /*
         * JITed image shrinks with every pass and the loop iterates
         * until the image stops shrinking. Very large BPF programs
         * may converge on the last pass. In such case do one more
         * pass to emit the final image.
         */
        for (pass = 0; pass < MAX_PASSES || image; pass++) {
                if (!padding && pass >= PADDING_PASSES)
                        padding = true;
                proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);
                if (proglen <= 0) {
out_image:
                        image = NULL;
                        if (header) {
                                bpf_arch_text_copy(&header->size, &rw_header->size,
                                                   sizeof(rw_header->size));
                                bpf_jit_binary_pack_free(header, rw_header);
                        }
                        /* Fall back to interpreter mode */
                        prog = orig_prog;
                        if (extra_pass) {
                                prog->bpf_func = NULL;
                                prog->jited = 0;
                                prog->jited_len = 0;
                        }
                        goto out_addrs;
                }
                if (image) {
                        if (proglen != oldproglen) {
                                pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
                                       proglen, oldproglen);
                                goto out_image;
                        }
                        break;
                }
                if (proglen == oldproglen) {
                        /*
                         * The number of entries in extable is the number of BPF_LDX
                         * insns that access kernel memory via "pointer to BTF type".
                         * The verifier changed their opcode from LDX|MEM|size
                         * to LDX|PROBE_MEM|size to make JITing easier.
                         */
                        u32 align = __alignof__(struct exception_table_entry);
                        u32 extable_size = prog->aux->num_exentries *
                                sizeof(struct exception_table_entry);

                        /* allocate module memory for x86 insns and extable */
                        header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
                                                           &image, align, &rw_header, &rw_image,
                                                           jit_fill_hole);
                        if (!header) {
                                prog = orig_prog;
                                goto out_addrs;
                        }
                        prog->aux->extable = (void *) image + roundup(proglen, align);
                }
                oldproglen = proglen;
                cond_resched();
        }

        if (bpf_jit_enable > 1)
                bpf_jit_dump(prog->len, proglen, pass + 1, rw_image);

        if (image) {
                if (!prog->is_func || extra_pass) {
                        /*
                         * bpf_jit_binary_pack_finalize fails in two scenarios:
                         *   1) header is not pointing to proper module memory;
                         *   2) the arch doesn't support bpf_arch_text_copy().
                         *
                         * Both cases are serious bugs and justify WARN_ON.
                         */
                        if (WARN_ON(bpf_jit_binary_pack_finalize(header, rw_header))) {
                                /* header has been freed */
                                header = NULL;
                                goto out_image;
                        }

                        bpf_tail_call_direct_fixup(prog);
                } else {
                        jit_data->addrs = addrs;
                        jit_data->ctx = ctx;
                        jit_data->proglen = proglen;
                        jit_data->image = image;
                        jit_data->header = header;
                        jit_data->rw_header = rw_header;
                }
                /*
                 * ctx.prog_offset is used when CFI preambles put code *before*
                 * the function. See emit_cfi(). For FineIBT specifically this code
                 * can also be executed and bpf_prog_kallsyms_add() will
                 * generate an additional symbol to cover this, hence also
                 * decrement proglen.
                 */
                prog->bpf_func = (void *)image + cfi_get_offset();
                prog->jited = 1;
                prog->jited_len = proglen - cfi_get_offset();
        } else {
                prog = orig_prog;
        }

        if (!image || !prog->is_func || extra_pass) {
                if (image)
                        bpf_prog_fill_jited_linfo(prog, addrs + 1);
out_addrs:
                kvfree(addrs);
                if (!image && priv_stack_ptr) {
                        free_percpu(priv_stack_ptr);
                        prog->aux->priv_stack_ptr = NULL;
                }
out_priv_stack:
                kfree(jit_data);
                prog->aux->jit_data = NULL;
        }
out:
        if (tmp_blinded)
                bpf_jit_prog_release_other(prog, prog == orig_prog ?
                                           tmp : orig_prog);
        return prog;
}

bool bpf_jit_supports_kfunc_call(void)
{
        return true;
}

void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
        if (text_poke_copy(dst, src, len) == NULL)
                return ERR_PTR(-EINVAL);
        return dst;
}

/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bool bpf_jit_supports_subprog_tailcalls(void)
{
        return true;
}

bool bpf_jit_supports_percpu_insn(void)
{
        return true;
}

void bpf_jit_free(struct bpf_prog *prog)
{
        if (prog->jited) {
                struct x64_jit_data *jit_data = prog->aux->jit_data;
                struct bpf_binary_header *hdr;
                void __percpu *priv_stack_ptr;
                int priv_stack_alloc_sz;

                /*
                 * If we fail the final pass of JIT (from jit_subprogs),
                 * the program may not be finalized yet. Call finalize here
                 * before freeing it.
                 */
                if (jit_data) {
                        bpf_jit_binary_pack_finalize(jit_data->header,
                                                     jit_data->rw_header);
                        kvfree(jit_data->addrs);
                        kfree(jit_data);
                }
                prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset();
                hdr = bpf_jit_binary_pack_hdr(prog);
                bpf_jit_binary_pack_free(hdr, NULL);
                priv_stack_ptr = prog->aux->priv_stack_ptr;
                if (priv_stack_ptr) {
                        priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
                                              2 * PRIV_STACK_GUARD_SZ;
                        priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_sz, prog);
                        free_percpu(prog->aux->priv_stack_ptr);
                }
                WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
        }

        bpf_prog_unlock_free(prog);
}

bool bpf_jit_supports_exceptions(void)
{
        /* We unwind through both kernel frames (starting from within bpf_throw
         * call) and BPF frames. Therefore we require ORC unwinder to be enabled
         * to walk kernel frames and reach BPF frames in the stack trace.
         */
        return IS_ENABLED(CONFIG_UNWINDER_ORC);
}

bool bpf_jit_supports_private_stack(void)
{
        return true;
}

void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
{
#if defined(CONFIG_UNWINDER_ORC)
        struct unwind_state state;
        unsigned long addr;

        for (unwind_start(&state, current, NULL, NULL); !unwind_done(&state);
             unwind_next_frame(&state)) {
                addr = unwind_get_return_address(&state);
                if (!addr || !consume_fn(cookie, (u64)addr, (u64)state.sp, (u64)state.bp))
                        break;
        }
        return;
#endif
        WARN(1, "verification of programs using bpf_throw should have failed\n");
}

void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
                               struct bpf_prog *new, struct bpf_prog *old)
{
        u8 *old_addr, *new_addr, *old_bypass_addr;
        int ret;

        old_bypass_addr = old ? NULL : poke->bypass_addr;
        old_addr = old ? (u8 *)old->bpf_func + poke->adj_off : NULL;
        new_addr = new ? (u8 *)new->bpf_func + poke->adj_off : NULL;

        /*
         * On program loading or teardown, the program's kallsym entry
         * might not be in place, so we use __bpf_arch_text_poke to skip
         * the kallsyms check.
         */
        if (new) {
                ret = __bpf_arch_text_poke(poke->tailcall_target,
                                           BPF_MOD_JUMP,
                                           old_addr, new_addr);
                BUG_ON(ret < 0);
                if (!old) {
                        ret = __bpf_arch_text_poke(poke->tailcall_bypass,
                                                   BPF_MOD_JUMP,
                                                   poke->bypass_addr,
                                                   NULL);
                        BUG_ON(ret < 0);
                }
        } else {
                ret = __bpf_arch_text_poke(poke->tailcall_bypass,
                                           BPF_MOD_JUMP,
                                           old_bypass_addr,
                                           poke->bypass_addr);
                BUG_ON(ret < 0);
                /* let other CPUs finish the execution of program
                 * so that it will not possible to expose them
                 * to invalid nop, stack unwind, nop state
                 */
                if (!ret)
                        synchronize_rcu();
                ret = __bpf_arch_text_poke(poke->tailcall_target,
                                           BPF_MOD_JUMP,
                                           old_addr, NULL);
                BUG_ON(ret < 0);
        }
}

bool bpf_jit_supports_arena(void)
{
        return true;
}

bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
{
        if (!in_arena)
                return true;
        switch (insn->code) {
        case BPF_STX | BPF_ATOMIC | BPF_W:
        case BPF_STX | BPF_ATOMIC | BPF_DW:
                if (insn->imm == (BPF_AND | BPF_FETCH) ||
                    insn->imm == (BPF_OR | BPF_FETCH) ||
                    insn->imm == (BPF_XOR | BPF_FETCH))
                        return false;
        }
        return true;
}

bool bpf_jit_supports_ptr_xchg(void)
{
        return true;
}

/* x86-64 JIT emits its own code to filter user addresses so return 0 here */
u64 bpf_arch_uaddress_limit(void)
{
        return 0;
}


















































































































































   14 
   14 
    5 


   13 















   13 
   13 








































   37 

   37 
   37 
   37 

   37 








   36 






   35 
   35 
   35 

   35 















  139 

  139 








  138 
   18 




  132 
   16 

   16 



  134 
   37 


  112 
  112 
  112 

  112 
  112 
    4 
  112 
  106 
  112 
  108 

   22 

  112 
  102 
   27 


   27 


  112 














  112 
   19 




   19 
   12 


    7 
   19 




  106 
   98 


   26 
  106 

  112 
   17 

  112 
  112 





  138 








  138 









   37 

  139 
   37 

  138 










   14 








   14 
   14 

   14 


   14 
   13 


   14 


    3 











    1 


    1 













    1 
    1 









    2 
    2 

    2 










  106 


   35 
   35 
   34 















  105 


   31 


  105 

   96 
   25 
   13 

   13 
   13 



   96 

   95 
   25 
   96 
   90 

  105 


   30 
   30 



    1 




   30 





   30 

    1 

   30 

   27 
   27 
    1 




    4 




   30 

   19 




   30 
   30 
   30 







   30 


















    3 
    3 
    3 

    3 

    3 
    2 


    1 
    1 




    3 
    1 



    3 






   30 























    7 
    7 

    7 


    6 



    6 





    6 


    6 



    6 



    7 









  107 
   70 
  107 

  106 
   91 
   91 
   32 
   11 
  106 
   11 
  107 
    1 






  107 
    5 


  105 
   93 

  107 

  106 






  107 












  107 







   99 
    1 
  106 



  100 





  107 







  107 


  107 


  105 











  104 


  105 


  105 



  105 



  105 


  105 
  105 

















  103 
  105 













































    4 

    4 





















































    4 






















    4 



















































































































  108 


  108 

  108 
  107 


   96 
   36 

   36 







   36 



























  108 
  107 














  106 
   33 





  107 

















  103 






    5 


    5 










   15 
   12 

   15 
   10 
   15 
   13 
   14 
    6 








   10 


   15 











  101 
  101 
  101 

  101 

  101 












  103 





   10 


   10 




  102 




  103 


    4 















    1 







    1 


    1 


  102 

































  103 
   93 









  103 




















  102 
    1 



  103 

  102 

  109 



  103 
  103 











   10 

   10 













  101 


    5 

  102 

















  100 

   12 

  101 
  100 


  101 
  101 





  101 





  101 


   92 
  100 








    6 
    5 




    1 

    1 

































   20 





   20 



   20 
















   20 




   20 
   13 

   20 


   20 


   20 






    7 






   19 






   20 
   19 

    1 
   20 



   20 



   13 

   20 
   18 




    4 











    4 


    4 









    8 
    8 


    2 

    8 


    5 













































  109 



  109 
  109 


   20 

   20 








   20 






   20 



















    4 



   17 



















    4 
  109 




























  109 



   12 


  103 




  108 
  102 
   34 
  103 

   95 

























   34 
   28 
    8 
    5 






























  109 











   10 








































   10 








  103 
  103 















   11 
  109 






   71 




  108 


  102 
  102 

  101 

  102 



















    1 





  102 


    7 
   78 





   69 
   78 




    5 


    5 
    5 

    5 


   78 









  108 

















   78 










    5 
   78 
   12 
   12 


    6 
   78 

   12 






































  109 




  109 

  109 

  109 






  108 

  103 
    4 



  103 



  103 
   67 

   70 
  108 





  109 


   78 
   78 




   78 





    5 




   77 







   77 





   75 


   77 




   77 




   77 


   77 










   11 





   11 
    1 

    1 





   10 




    1 

    1 

   10 



   11 







   16 

    8 








   10 








    1 
    9 





   14 


   14 


   14 


   10 


    9 
    1 

    9 
    9 
    9 
    9 



















   16 




   16 
   16 


   15 
   15 

   14 

   14 




   16 



    9 





   20 


   10 










   20 


    9 




















   16 




   19 


   18 




   19 


   18 



   18 

   17 



   16 


   16 


   10 
    2 











   16 


















   16 



   15 



    1 
    1 






   18 




   19 











   57 


   57 





   57 



   26 

   25 

   25 


   25 
   25 




   57 





   57 




























   61 



   60 

   60 




   60 




   57 




   57 



   52 











   82 
   42 




   45 








   44 




   43 


   43 


   44 
   44 















   84 





   81 
   73 

   82 
   81 
    1 

   80 
   20 


   61 

   71 



   83 























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
// SPDX-License-Identifier: GPL-2.0
/*
 * Memory Migration functionality - linux/mm/migrate.c
 *
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 *
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 *
 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
 * Hirokazu Takahashi <taka@valinux.co.jp>
 * Dave Hansen <haveblue@us.ibm.com>
 * Christoph Lameter
 */

#include <linux/migrate.h>
#include <linux/export.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
#include <linux/buffer_head.h>
#include <linux/mm_inline.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/writeback.h>
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
#include <linux/security.h>
#include <linux/backing-dev.h>
#include <linux/compaction.h>
#include <linux/syscalls.h>
#include <linux/compat.h>
#include <linux/hugetlb.h>
#include <linux/gfp.h>
#include <linux/pfn_t.h>
#include <linux/page_idle.h>
#include <linux/page_owner.h>
#include <linux/sched/mm.h>
#include <linux/ptrace.h>
#include <linux/memory.h>
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
#include <linux/pagewalk.h>

#include <asm/tlbflush.h>

#include <trace/events/migrate.h>

#include "internal.h"

bool isolate_movable_page(struct page *page, isolate_mode_t mode)
{
        struct folio *folio = folio_get_nontail_page(page);
        const struct movable_operations *mops;

        /*
         * Avoid burning cycles with pages that are yet under __free_pages(),
         * or just got freed under us.
         *
         * In case we 'win' a race for a movable page being freed under us and
         * raise its refcount preventing __free_pages() from doing its job
         * the put_page() at the end of this block will take care of
         * release this page, thus avoiding a nasty leakage.
         */
        if (!folio)
                goto out;

        if (unlikely(folio_test_slab(folio)))
                goto out_putfolio;
        /* Pairs with smp_wmb() in slab freeing, e.g. SLUB's __free_slab() */
        smp_rmb();
        /*
         * Check movable flag before taking the page lock because
         * we use non-atomic bitops on newly allocated page flags so
         * unconditionally grabbing the lock ruins page's owner side.
         */
        if (unlikely(!__folio_test_movable(folio)))
                goto out_putfolio;
        /* Pairs with smp_wmb() in slab allocation, e.g. SLUB's alloc_slab_page() */
        smp_rmb();
        if (unlikely(folio_test_slab(folio)))
                goto out_putfolio;

        /*
         * As movable pages are not isolated from LRU lists, concurrent
         * compaction threads can race against page migration functions
         * as well as race against the releasing a page.
         *
         * In order to avoid having an already isolated movable page
         * being (wrongly) re-isolated while it is under migration,
         * or to avoid attempting to isolate pages being released,
         * lets be sure we have the page lock
         * before proceeding with the movable page isolation steps.
         */
        if (unlikely(!folio_trylock(folio)))
                goto out_putfolio;

        if (!folio_test_movable(folio) || folio_test_isolated(folio))
                goto out_no_isolated;

        mops = folio_movable_ops(folio);
        VM_BUG_ON_FOLIO(!mops, folio);

        if (!mops->isolate_page(&folio->page, mode))
                goto out_no_isolated;

        /* Driver shouldn't use the isolated flag */
        WARN_ON_ONCE(folio_test_isolated(folio));
        folio_set_isolated(folio);
        folio_unlock(folio);

        return true;

out_no_isolated:
        folio_unlock(folio);
out_putfolio:
        folio_put(folio);
out:
        return false;
}

static void putback_movable_folio(struct folio *folio)
{
        const struct movable_operations *mops = folio_movable_ops(folio);

        mops->putback_page(&folio->page);
        folio_clear_isolated(folio);
}

/*
 * Put previously isolated pages back onto the appropriate lists
 * from where they were once taken off for compaction/migration.
 *
 * This function shall be used whenever the isolated pageset has been
 * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
 * and isolate_hugetlb().
 */
void putback_movable_pages(struct list_head *l)
{
        struct folio *folio;
        struct folio *folio2;

        list_for_each_entry_safe(folio, folio2, l, lru) {
                if (unlikely(folio_test_hugetlb(folio))) {
                        folio_putback_active_hugetlb(folio);
                        continue;
                }
                list_del(&folio->lru);
                /*
                 * We isolated non-lru movable folio so here we can use
                 * __folio_test_movable because LRU folio's mapping cannot
                 * have PAGE_MAPPING_MOVABLE.
                 */
                if (unlikely(__folio_test_movable(folio))) {
                        VM_BUG_ON_FOLIO(!folio_test_isolated(folio), folio);
                        folio_lock(folio);
                        if (folio_test_movable(folio))
                                putback_movable_folio(folio);
                        else
                                folio_clear_isolated(folio);
                        folio_unlock(folio);
                        folio_put(folio);
                } else {
                        node_stat_mod_folio(folio, NR_ISOLATED_ANON +
                                        folio_is_file_lru(folio), -folio_nr_pages(folio));
                        folio_putback_lru(folio);
                }
        }
}

/* Must be called with an elevated refcount on the non-hugetlb folio */
bool isolate_folio_to_list(struct folio *folio, struct list_head *list)
{
        bool isolated, lru;

        if (folio_test_hugetlb(folio))
                return isolate_hugetlb(folio, list);

        lru = !__folio_test_movable(folio);
        if (lru)
                isolated = folio_isolate_lru(folio);
        else
                isolated = isolate_movable_page(&folio->page,
                                                ISOLATE_UNEVICTABLE);

        if (!isolated)
                return false;

        list_add(&folio->lru, list);
        if (lru)
                node_stat_add_folio(folio, NR_ISOLATED_ANON +
                                    folio_is_file_lru(folio));

        return true;
}

static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw,
                                          struct folio *folio,
                                          unsigned long idx)
{
        struct page *page = folio_page(folio, idx);
        bool contains_data;
        pte_t newpte;
        void *addr;

        if (PageCompound(page))
                return false;
        VM_BUG_ON_PAGE(!PageAnon(page), page);
        VM_BUG_ON_PAGE(!PageLocked(page), page);
        VM_BUG_ON_PAGE(pte_present(*pvmw->pte), page);

        if (folio_test_mlocked(folio) || (pvmw->vma->vm_flags & VM_LOCKED) ||
            mm_forbids_zeropage(pvmw->vma->vm_mm))
                return false;

        /*
         * The pmd entry mapping the old thp was flushed and the pte mapping
         * this subpage has been non present. If the subpage is only zero-filled
         * then map it to the shared zeropage.
         */
        addr = kmap_local_page(page);
        contains_data = memchr_inv(addr, 0, PAGE_SIZE);
        kunmap_local(addr);

        if (contains_data)
                return false;

        newpte = pte_mkspecial(pfn_pte(my_zero_pfn(pvmw->address),
                                        pvmw->vma->vm_page_prot));
        set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte);

        dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio));
        return true;
}

struct rmap_walk_arg {
        struct folio *folio;
        bool map_unused_to_zeropage;
};

/*
 * Restore a potential migration pte to a working pte entry
 */
static bool remove_migration_pte(struct folio *folio,
                struct vm_area_struct *vma, unsigned long addr, void *arg)
{
        struct rmap_walk_arg *rmap_walk_arg = arg;
        DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION);

        while (page_vma_mapped_walk(&pvmw)) {
                rmap_t rmap_flags = RMAP_NONE;
                pte_t old_pte;
                pte_t pte;
                swp_entry_t entry;
                struct page *new;
                unsigned long idx = 0;

                /* pgoff is invalid for ksm pages, but they are never large */
                if (folio_test_large(folio) && !folio_test_hugetlb(folio))
                        idx = linear_page_index(vma, pvmw.address) - pvmw.pgoff;
                new = folio_page(folio, idx);

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
                /* PMD-mapped THP migration entry */
                if (!pvmw.pte) {
                        VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
                                        !folio_test_pmd_mappable(folio), folio);
                        remove_migration_pmd(&pvmw, new);
                        continue;
                }
#endif
                if (rmap_walk_arg->map_unused_to_zeropage &&
                    try_to_map_unused_to_zeropage(&pvmw, folio, idx))
                        continue;

                folio_get(folio);
                pte = mk_pte(new, READ_ONCE(vma->vm_page_prot));
                old_pte = ptep_get(pvmw.pte);

                entry = pte_to_swp_entry(old_pte);
                if (!is_migration_entry_young(entry))
                        pte = pte_mkold(pte);
                if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
                        pte = pte_mkdirty(pte);
                if (pte_swp_soft_dirty(old_pte))
                        pte = pte_mksoft_dirty(pte);
                else
                        pte = pte_clear_soft_dirty(pte);

                if (is_writable_migration_entry(entry))
                        pte = pte_mkwrite(pte, vma);
                else if (pte_swp_uffd_wp(old_pte))
                        pte = pte_mkuffd_wp(pte);

                if (folio_test_anon(folio) && !is_readable_migration_entry(entry))
                        rmap_flags |= RMAP_EXCLUSIVE;

                if (unlikely(is_device_private_page(new))) {
                        if (pte_write(pte))
                                entry = make_writable_device_private_entry(
                                                        page_to_pfn(new));
                        else
                                entry = make_readable_device_private_entry(
                                                        page_to_pfn(new));
                        pte = swp_entry_to_pte(entry);
                        if (pte_swp_soft_dirty(old_pte))
                                pte = pte_swp_mksoft_dirty(pte);
                        if (pte_swp_uffd_wp(old_pte))
                                pte = pte_swp_mkuffd_wp(pte);
                }

#ifdef CONFIG_HUGETLB_PAGE
                if (folio_test_hugetlb(folio)) {
                        struct hstate *h = hstate_vma(vma);
                        unsigned int shift = huge_page_shift(h);
                        unsigned long psize = huge_page_size(h);

                        pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
                        if (folio_test_anon(folio))
                                hugetlb_add_anon_rmap(folio, vma, pvmw.address,
                                                      rmap_flags);
                        else
                                hugetlb_add_file_rmap(folio);
                        set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte,
                                        psize);
                } else
#endif
                {
                        if (folio_test_anon(folio))
                                folio_add_anon_rmap_pte(folio, new, vma,
                                                        pvmw.address, rmap_flags);
                        else
                                folio_add_file_rmap_pte(folio, new, vma);
                        set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
                }
                if (vma->vm_flags & VM_LOCKED)
                        mlock_drain_local();

                trace_remove_migration_pte(pvmw.address, pte_val(pte),
                                           compound_order(new));

                /* No need to invalidate - it was non-present before */
                update_mmu_cache(vma, pvmw.address, pvmw.pte);
        }

        return true;
}

/*
 * Get rid of all migration entries and replace them by
 * references to the indicated page.
 */
void remove_migration_ptes(struct folio *src, struct folio *dst, int flags)
{
        struct rmap_walk_arg rmap_walk_arg = {
                .folio = src,
                .map_unused_to_zeropage = flags & RMP_USE_SHARED_ZEROPAGE,
        };

        struct rmap_walk_control rwc = {
                .rmap_one = remove_migration_pte,
                .arg = &rmap_walk_arg,
        };

        VM_BUG_ON_FOLIO((flags & RMP_USE_SHARED_ZEROPAGE) && (src != dst), src);

        if (flags & RMP_LOCKED)
                rmap_walk_locked(dst, &rwc);
        else
                rmap_walk(dst, &rwc);
}

/*
 * Something used the pte of a page under migration. We need to
 * get to the page and wait until migration is finished.
 * When we return from this function the fault will be retried.
 */
void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
                          unsigned long address)
{
        spinlock_t *ptl;
        pte_t *ptep;
        pte_t pte;
        swp_entry_t entry;

        ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
        if (!ptep)
                return;

        pte = ptep_get(ptep);
        pte_unmap(ptep);

        if (!is_swap_pte(pte))
                goto out;

        entry = pte_to_swp_entry(pte);
        if (!is_migration_entry(entry))
                goto out;

        migration_entry_wait_on_locked(entry, ptl);
        return;
out:
        spin_unlock(ptl);
}

#ifdef CONFIG_HUGETLB_PAGE
/*
 * The vma read lock must be held upon entry. Holding that lock prevents either
 * the pte or the ptl from being freed.
 *
 * This function will release the vma lock before returning.
 */
void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
{
        spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), vma->vm_mm, ptep);
        pte_t pte;

        hugetlb_vma_assert_locked(vma);
        spin_lock(ptl);
        pte = huge_ptep_get(vma->vm_mm, addr, ptep);

        if (unlikely(!is_hugetlb_entry_migration(pte))) {
                spin_unlock(ptl);
                hugetlb_vma_unlock_read(vma);
        } else {
                /*
                 * If migration entry existed, safe to release vma lock
                 * here because the pgtable page won't be freed without the
                 * pgtable lock released.  See comment right above pgtable
                 * lock release in migration_entry_wait_on_locked().
                 */
                hugetlb_vma_unlock_read(vma);
                migration_entry_wait_on_locked(pte_to_swp_entry(pte), ptl);
        }
}
#endif

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
{
        spinlock_t *ptl;

        ptl = pmd_lock(mm, pmd);
        if (!is_pmd_migration_entry(*pmd))
                goto unlock;
        migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), ptl);
        return;
unlock:
        spin_unlock(ptl);
}
#endif

static int folio_expected_refs(struct address_space *mapping,
                struct folio *folio)
{
        int refs = 1;
        if (!mapping)
                return refs;

        refs += folio_nr_pages(folio);
        if (folio_test_private(folio))
                refs++;

        return refs;
}

/*
 * Replace the folio in the mapping.
 *
 * The number of remaining references must be:
 * 1 for anonymous folios without a mapping
 * 2 for folios with a mapping
 * 3 for folios with a mapping and the private flag set.
 */
static int __folio_migrate_mapping(struct address_space *mapping,
                struct folio *newfolio, struct folio *folio, int expected_count)
{
        XA_STATE(xas, &mapping->i_pages, folio_index(folio));
        struct zone *oldzone, *newzone;
        int dirty;
        long nr = folio_nr_pages(folio);
        long entries, i;

        if (!mapping) {
                /* Take off deferred split queue while frozen and memcg set */
                if (folio_test_large(folio) &&
                    folio_test_large_rmappable(folio)) {
                        if (!folio_ref_freeze(folio, expected_count))
                                return -EAGAIN;
                        folio_unqueue_deferred_split(folio);
                        folio_ref_unfreeze(folio, expected_count);
                }

                /* No turning back from here */
                newfolio->index = folio->index;
                newfolio->mapping = folio->mapping;
                if (folio_test_anon(folio) && folio_test_large(folio))
                        mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
                if (folio_test_swapbacked(folio))
                        __folio_set_swapbacked(newfolio);

                return MIGRATEPAGE_SUCCESS;
        }

        oldzone = folio_zone(folio);
        newzone = folio_zone(newfolio);

        xas_lock_irq(&xas);
        if (!folio_ref_freeze(folio, expected_count)) {
                xas_unlock_irq(&xas);
                return -EAGAIN;
        }

        /* Take off deferred split queue while frozen and memcg set */
        folio_unqueue_deferred_split(folio);

        /*
         * Now we know that no one else is looking at the folio:
         * no turning back from here.
         */
        newfolio->index = folio->index;
        newfolio->mapping = folio->mapping;
        if (folio_test_anon(folio) && folio_test_large(folio))
                mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1);
        folio_ref_add(newfolio, nr); /* add cache reference */
        if (folio_test_swapbacked(folio)) {
                __folio_set_swapbacked(newfolio);
                if (folio_test_swapcache(folio)) {
                        folio_set_swapcache(newfolio);
                        newfolio->private = folio_get_private(folio);
                }
                entries = nr;
        } else {
                VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
                entries = 1;
        }

        /* Move dirty while folio refs frozen and newfolio not yet exposed */
        dirty = folio_test_dirty(folio);
        if (dirty) {
                folio_clear_dirty(folio);
                folio_set_dirty(newfolio);
        }

        /* Swap cache still stores N entries instead of a high-order entry */
        for (i = 0; i < entries; i++) {
                xas_store(&xas, newfolio);
                xas_next(&xas);
        }

        /*
         * Drop cache reference from old folio by unfreezing
         * to one less reference.
         * We know this isn't the last reference.
         */
        folio_ref_unfreeze(folio, expected_count - nr);

        xas_unlock(&xas);
        /* Leave irq disabled to prevent preemption while updating stats */

        /*
         * If moved to a different zone then also account
         * the folio for that zone. Other VM counters will be
         * taken care of when we establish references to the
         * new folio and drop references to the old folio.
         *
         * Note that anonymous folios are accounted for
         * via NR_FILE_PAGES and NR_ANON_MAPPED if they
         * are mapped to swap space.
         */
        if (newzone != oldzone) {
                struct lruvec *old_lruvec, *new_lruvec;
                struct mem_cgroup *memcg;

                memcg = folio_memcg(folio);
                old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
                new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);

                __mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
                __mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
                if (folio_test_swapbacked(folio) && !folio_test_swapcache(folio)) {
                        __mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
                        __mod_lruvec_state(new_lruvec, NR_SHMEM, nr);

                        if (folio_test_pmd_mappable(folio)) {
                                __mod_lruvec_state(old_lruvec, NR_SHMEM_THPS, -nr);
                                __mod_lruvec_state(new_lruvec, NR_SHMEM_THPS, nr);
                        }
                }
#ifdef CONFIG_SWAP
                if (folio_test_swapcache(folio)) {
                        __mod_lruvec_state(old_lruvec, NR_SWAPCACHE, -nr);
                        __mod_lruvec_state(new_lruvec, NR_SWAPCACHE, nr);
                }
#endif
                if (dirty && mapping_can_writeback(mapping)) {
                        __mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
                        __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
                        __mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
                        __mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
                }
        }
        local_irq_enable();

        return MIGRATEPAGE_SUCCESS;
}

int folio_migrate_mapping(struct address_space *mapping,
                struct folio *newfolio, struct folio *folio, int extra_count)
{
        int expected_count = folio_expected_refs(mapping, folio) + extra_count;

        if (folio_ref_count(folio) != expected_count)
                return -EAGAIN;

        return __folio_migrate_mapping(mapping, newfolio, folio, expected_count);
}
EXPORT_SYMBOL(folio_migrate_mapping);

/*
 * The expected number of remaining references is the same as that
 * of folio_migrate_mapping().
 */
int migrate_huge_page_move_mapping(struct address_space *mapping,
                                   struct folio *dst, struct folio *src)
{
        XA_STATE(xas, &mapping->i_pages, folio_index(src));
        int rc, expected_count = folio_expected_refs(mapping, src);

        if (folio_ref_count(src) != expected_count)
                return -EAGAIN;

        rc = folio_mc_copy(dst, src);
        if (unlikely(rc))
                return rc;

        xas_lock_irq(&xas);
        if (!folio_ref_freeze(src, expected_count)) {
                xas_unlock_irq(&xas);
                return -EAGAIN;
        }

        dst->index = src->index;
        dst->mapping = src->mapping;

        folio_ref_add(dst, folio_nr_pages(dst));

        xas_store(&xas, dst);

        folio_ref_unfreeze(src, expected_count - folio_nr_pages(src));

        xas_unlock_irq(&xas);

        return MIGRATEPAGE_SUCCESS;
}

/*
 * Copy the flags and some other ancillary information
 */
void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
{
        int cpupid;

        if (folio_test_referenced(folio))
                folio_set_referenced(newfolio);
        if (folio_test_uptodate(folio))
                folio_mark_uptodate(newfolio);
        if (folio_test_clear_active(folio)) {
                VM_BUG_ON_FOLIO(folio_test_unevictable(folio), folio);
                folio_set_active(newfolio);
        } else if (folio_test_clear_unevictable(folio))
                folio_set_unevictable(newfolio);
        if (folio_test_workingset(folio))
                folio_set_workingset(newfolio);
        if (folio_test_checked(folio))
                folio_set_checked(newfolio);
        /*
         * PG_anon_exclusive (-> PG_mappedtodisk) is always migrated via
         * migration entries. We can still have PG_anon_exclusive set on an
         * effectively unmapped and unreferenced first sub-pages of an
         * anonymous THP: we can simply copy it here via PG_mappedtodisk.
         */
        if (folio_test_mappedtodisk(folio))
                folio_set_mappedtodisk(newfolio);

        /* Move dirty on pages not done by folio_migrate_mapping() */
        if (folio_test_dirty(folio))
                folio_set_dirty(newfolio);

        if (folio_test_young(folio))
                folio_set_young(newfolio);
        if (folio_test_idle(folio))
                folio_set_idle(newfolio);

        /*
         * Copy NUMA information to the new page, to prevent over-eager
         * future migrations of this same page.
         */
        cpupid = folio_xchg_last_cpupid(folio, -1);
        /*
         * For memory tiering mode, when migrate between slow and fast
         * memory node, reset cpupid, because that is used to record
         * page access time in slow memory node.
         */
        if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) {
                bool f_toptier = node_is_toptier(folio_nid(folio));
                bool t_toptier = node_is_toptier(folio_nid(newfolio));

                if (f_toptier != t_toptier)
                        cpupid = -1;
        }
        folio_xchg_last_cpupid(newfolio, cpupid);

        folio_migrate_ksm(newfolio, folio);
        /*
         * Please do not reorder this without considering how mm/ksm.c's
         * ksm_get_folio() depends upon ksm_migrate_page() and the
         * swapcache flag.
         */
        if (folio_test_swapcache(folio))
                folio_clear_swapcache(folio);
        folio_clear_private(folio);

        /* page->private contains hugetlb specific flags */
        if (!folio_test_hugetlb(folio))
                folio->private = NULL;

        /*
         * If any waiters have accumulated on the new page then
         * wake them up.
         */
        if (folio_test_writeback(newfolio))
                folio_end_writeback(newfolio);

        /*
         * PG_readahead shares the same bit with PG_reclaim.  The above
         * end_page_writeback() may clear PG_readahead mistakenly, so set the
         * bit after that.
         */
        if (folio_test_readahead(folio))
                folio_set_readahead(newfolio);

        folio_copy_owner(newfolio, folio);
        pgalloc_tag_copy(newfolio, folio);

        mem_cgroup_migrate(folio, newfolio);
}
EXPORT_SYMBOL(folio_migrate_flags);

/************************************************************
 *                    Migration functions
 ***********************************************************/

static int __migrate_folio(struct address_space *mapping, struct folio *dst,
                           struct folio *src, void *src_private,
                           enum migrate_mode mode)
{
        int rc, expected_count = folio_expected_refs(mapping, src);

        /* Check whether src does not have extra refs before we do more work */
        if (folio_ref_count(src) != expected_count)
                return -EAGAIN;

        rc = folio_mc_copy(dst, src);
        if (unlikely(rc))
                return rc;

        rc = __folio_migrate_mapping(mapping, dst, src, expected_count);
        if (rc != MIGRATEPAGE_SUCCESS)
                return rc;

        if (src_private)
                folio_attach_private(dst, folio_detach_private(src));

        folio_migrate_flags(dst, src);
        return MIGRATEPAGE_SUCCESS;
}

/**
 * migrate_folio() - Simple folio migration.
 * @mapping: The address_space containing the folio.
 * @dst: The folio to migrate the data to.
 * @src: The folio containing the current data.
 * @mode: How to migrate the page.
 *
 * Common logic to directly migrate a single LRU folio suitable for
 * folios that do not have private data.
 *
 * Folios are locked upon entry and exit.
 */
int migrate_folio(struct address_space *mapping, struct folio *dst,
                  struct folio *src, enum migrate_mode mode)
{
        BUG_ON(folio_test_writeback(src));        /* Writeback must be complete */
        return __migrate_folio(mapping, dst, src, NULL, mode);
}
EXPORT_SYMBOL(migrate_folio);

#ifdef CONFIG_BUFFER_HEAD
/* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head,
                                                        enum migrate_mode mode)
{
        struct buffer_head *bh = head;
        struct buffer_head *failed_bh;

        do {
                if (!trylock_buffer(bh)) {
                        if (mode == MIGRATE_ASYNC)
                                goto unlock;
                        if (mode == MIGRATE_SYNC_LIGHT && !buffer_uptodate(bh))
                                goto unlock;
                        lock_buffer(bh);
                }

                bh = bh->b_this_page;
        } while (bh != head);

        return true;

unlock:
        /* We failed to lock the buffer and cannot stall. */
        failed_bh = bh;
        bh = head;
        while (bh != failed_bh) {
                unlock_buffer(bh);
                bh = bh->b_this_page;
        }

        return false;
}

static int __buffer_migrate_folio(struct address_space *mapping,
                struct folio *dst, struct folio *src, enum migrate_mode mode,
                bool check_refs)
{
        struct buffer_head *bh, *head;
        int rc;
        int expected_count;

        head = folio_buffers(src);
        if (!head)
                return migrate_folio(mapping, dst, src, mode);

        /* Check whether page does not have extra refs before we do more work */
        expected_count = folio_expected_refs(mapping, src);
        if (folio_ref_count(src) != expected_count)
                return -EAGAIN;

        if (!buffer_migrate_lock_buffers(head, mode))
                return -EAGAIN;

        if (check_refs) {
                bool busy;
                bool invalidated = false;

recheck_buffers:
                busy = false;
                spin_lock(&mapping->i_private_lock);
                bh = head;
                do {
                        if (atomic_read(&bh->b_count)) {
                                busy = true;
                                break;
                        }
                        bh = bh->b_this_page;
                } while (bh != head);
                if (busy) {
                        if (invalidated) {
                                rc = -EAGAIN;
                                goto unlock_buffers;
                        }
                        spin_unlock(&mapping->i_private_lock);
                        invalidate_bh_lrus();
                        invalidated = true;
                        goto recheck_buffers;
                }
        }

        rc = filemap_migrate_folio(mapping, dst, src, mode);
        if (rc != MIGRATEPAGE_SUCCESS)
                goto unlock_buffers;

        bh = head;
        do {
                folio_set_bh(bh, dst, bh_offset(bh));
                bh = bh->b_this_page;
        } while (bh != head);

unlock_buffers:
        if (check_refs)
                spin_unlock(&mapping->i_private_lock);
        bh = head;
        do {
                unlock_buffer(bh);
                bh = bh->b_this_page;
        } while (bh != head);

        return rc;
}

/**
 * buffer_migrate_folio() - Migration function for folios with buffers.
 * @mapping: The address space containing @src.
 * @dst: The folio to migrate to.
 * @src: The folio to migrate from.
 * @mode: How to migrate the folio.
 *
 * This function can only be used if the underlying filesystem guarantees
 * that no other references to @src exist. For example attached buffer
 * heads are accessed only under the folio lock.  If your filesystem cannot
 * provide this guarantee, buffer_migrate_folio_norefs() may be more
 * appropriate.
 *
 * Return: 0 on success or a negative errno on failure.
 */
int buffer_migrate_folio(struct address_space *mapping,
                struct folio *dst, struct folio *src, enum migrate_mode mode)
{
        return __buffer_migrate_folio(mapping, dst, src, mode, false);
}
EXPORT_SYMBOL(buffer_migrate_folio);

/**
 * buffer_migrate_folio_norefs() - Migration function for folios with buffers.
 * @mapping: The address space containing @src.
 * @dst: The folio to migrate to.
 * @src: The folio to migrate from.
 * @mode: How to migrate the folio.
 *
 * Like buffer_migrate_folio() except that this variant is more careful
 * and checks that there are also no buffer head references. This function
 * is the right one for mappings where buffer heads are directly looked
 * up and referenced (such as block device mappings).
 *
 * Return: 0 on success or a negative errno on failure.
 */
int buffer_migrate_folio_norefs(struct address_space *mapping,
                struct folio *dst, struct folio *src, enum migrate_mode mode)
{
        return __buffer_migrate_folio(mapping, dst, src, mode, true);
}
EXPORT_SYMBOL_GPL(buffer_migrate_folio_norefs);
#endif /* CONFIG_BUFFER_HEAD */

int filemap_migrate_folio(struct address_space *mapping,
                struct folio *dst, struct folio *src, enum migrate_mode mode)
{
        return __migrate_folio(mapping, dst, src, folio_get_private(src), mode);
}
EXPORT_SYMBOL_GPL(filemap_migrate_folio);

/*
 * Writeback a folio to clean the dirty state
 */
static int writeout(struct address_space *mapping, struct folio *folio)
{
        struct writeback_control wbc = {
                .sync_mode = WB_SYNC_NONE,
                .nr_to_write = 1,
                .range_start = 0,
                .range_end = LLONG_MAX,
                .for_reclaim = 1
        };
        int rc;

        if (!mapping->a_ops->writepage)
                /* No write method for the address space */
                return -EINVAL;

        if (!folio_clear_dirty_for_io(folio))
                /* Someone else already triggered a write */
                return -EAGAIN;

        /*
         * A dirty folio may imply that the underlying filesystem has
         * the folio on some queue. So the folio must be clean for
         * migration. Writeout may mean we lose the lock and the
         * folio state is no longer what we checked for earlier.
         * At this point we know that the migration attempt cannot
         * be successful.
         */
        remove_migration_ptes(folio, folio, 0);

        rc = mapping->a_ops->writepage(&folio->page, &wbc);

        if (rc != AOP_WRITEPAGE_ACTIVATE)
                /* unlocked. Relock */
                folio_lock(folio);

        return (rc < 0) ? -EIO : -EAGAIN;
}

/*
 * Default handling if a filesystem does not provide a migration function.
 */
static int fallback_migrate_folio(struct address_space *mapping,
                struct folio *dst, struct folio *src, enum migrate_mode mode)
{
        if (folio_test_dirty(src)) {
                /* Only writeback folios in full synchronous migration */
                switch (mode) {
                case MIGRATE_SYNC:
                        break;
                default:
                        return -EBUSY;
                }
                return writeout(mapping, src);
        }

        /*
         * Buffers may be managed in a filesystem specific way.
         * We must have no buffers or drop them.
         */
        if (!filemap_release_folio(src, GFP_KERNEL))
                return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;

        return migrate_folio(mapping, dst, src, mode);
}

/*
 * Move a page to a newly allocated page
 * The page is locked and all ptes have been successfully removed.
 *
 * The new page will have replaced the old page if this function
 * is successful.
 *
 * Return value:
 *   < 0 - error code
 *  MIGRATEPAGE_SUCCESS - success
 */
static int move_to_new_folio(struct folio *dst, struct folio *src,
                                enum migrate_mode mode)
{
        int rc = -EAGAIN;
        bool is_lru = !__folio_test_movable(src);

        VM_BUG_ON_FOLIO(!folio_test_locked(src), src);
        VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst);

        if (likely(is_lru)) {
                struct address_space *mapping = folio_mapping(src);

                if (!mapping)
                        rc = migrate_folio(mapping, dst, src, mode);
                else if (mapping_inaccessible(mapping))
                        rc = -EOPNOTSUPP;
                else if (mapping->a_ops->migrate_folio)
                        /*
                         * Most folios have a mapping and most filesystems
                         * provide a migrate_folio callback. Anonymous folios
                         * are part of swap space which also has its own
                         * migrate_folio callback. This is the most common path
                         * for page migration.
                         */
                        rc = mapping->a_ops->migrate_folio(mapping, dst, src,
                                                                mode);
                else
                        rc = fallback_migrate_folio(mapping, dst, src, mode);
        } else {
                const struct movable_operations *mops;

                /*
                 * In case of non-lru page, it could be released after
                 * isolation step. In that case, we shouldn't try migration.
                 */
                VM_BUG_ON_FOLIO(!folio_test_isolated(src), src);
                if (!folio_test_movable(src)) {
                        rc = MIGRATEPAGE_SUCCESS;
                        folio_clear_isolated(src);
                        goto out;
                }

                mops = folio_movable_ops(src);
                rc = mops->migrate_page(&dst->page, &src->page, mode);
                WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
                                !folio_test_isolated(src));
        }

        /*
         * When successful, old pagecache src->mapping must be cleared before
         * src is freed; but stats require that PageAnon be left as PageAnon.
         */
        if (rc == MIGRATEPAGE_SUCCESS) {
                if (__folio_test_movable(src)) {
                        VM_BUG_ON_FOLIO(!folio_test_isolated(src), src);

                        /*
                         * We clear PG_movable under page_lock so any compactor
                         * cannot try to migrate this page.
                         */
                        folio_clear_isolated(src);
                }

                /*
                 * Anonymous and movable src->mapping will be cleared by
                 * free_pages_prepare so don't reset it here for keeping
                 * the type to work PageAnon, for example.
                 */
                if (!folio_mapping_flags(src))
                        src->mapping = NULL;

                if (likely(!folio_is_zone_device(dst)))
                        flush_dcache_folio(dst);
        }
out:
        return rc;
}

/*
 * To record some information during migration, we use unused private
 * field of struct folio of the newly allocated destination folio.
 * This is safe because nobody is using it except us.
 */
enum {
        PAGE_WAS_MAPPED = BIT(0),
        PAGE_WAS_MLOCKED = BIT(1),
        PAGE_OLD_STATES = PAGE_WAS_MAPPED | PAGE_WAS_MLOCKED,
};

static void __migrate_folio_record(struct folio *dst,
                                   int old_page_state,
                                   struct anon_vma *anon_vma)
{
        dst->private = (void *)anon_vma + old_page_state;
}

static void __migrate_folio_extract(struct folio *dst,
                                   int *old_page_state,
                                   struct anon_vma **anon_vmap)
{
        unsigned long private = (unsigned long)dst->private;

        *anon_vmap = (struct anon_vma *)(private & ~PAGE_OLD_STATES);
        *old_page_state = private & PAGE_OLD_STATES;
        dst->private = NULL;
}

/* Restore the source folio to the original state upon failure */
static void migrate_folio_undo_src(struct folio *src,
                                   int page_was_mapped,
                                   struct anon_vma *anon_vma,
                                   bool locked,
                                   struct list_head *ret)
{
        if (page_was_mapped)
                remove_migration_ptes(src, src, 0);
        /* Drop an anon_vma reference if we took one */
        if (anon_vma)
                put_anon_vma(anon_vma);
        if (locked)
                folio_unlock(src);
        if (ret)
                list_move_tail(&src->lru, ret);
}

/* Restore the destination folio to the original state upon failure */
static void migrate_folio_undo_dst(struct folio *dst, bool locked,
                free_folio_t put_new_folio, unsigned long private)
{
        if (locked)
                folio_unlock(dst);
        if (put_new_folio)
                put_new_folio(dst, private);
        else
                folio_put(dst);
}

/* Cleanup src folio upon migration success */
static void migrate_folio_done(struct folio *src,
                               enum migrate_reason reason)
{
        /*
         * Compaction can migrate also non-LRU pages which are
         * not accounted to NR_ISOLATED_*. They can be recognized
         * as __folio_test_movable
         */
        if (likely(!__folio_test_movable(src)) && reason != MR_DEMOTION)
                mod_node_page_state(folio_pgdat(src), NR_ISOLATED_ANON +
                                    folio_is_file_lru(src), -folio_nr_pages(src));

        if (reason != MR_MEMORY_FAILURE)
                /* We release the page in page_handle_poison. */
                folio_put(src);
}

/* Obtain the lock on page, remove all ptes. */
static int migrate_folio_unmap(new_folio_t get_new_folio,
                free_folio_t put_new_folio, unsigned long private,
                struct folio *src, struct folio **dstp, enum migrate_mode mode,
                enum migrate_reason reason, struct list_head *ret)
{
        struct folio *dst;
        int rc = -EAGAIN;
        int old_page_state = 0;
        struct anon_vma *anon_vma = NULL;
        bool is_lru = data_race(!__folio_test_movable(src));
        bool locked = false;
        bool dst_locked = false;

        if (folio_ref_count(src) == 1) {
                /* Folio was freed from under us. So we are done. */
                folio_clear_active(src);
                folio_clear_unevictable(src);
                /* free_pages_prepare() will clear PG_isolated. */
                list_del(&src->lru);
                migrate_folio_done(src, reason);
                return MIGRATEPAGE_SUCCESS;
        }

        dst = get_new_folio(src, private);
        if (!dst)
                return -ENOMEM;
        *dstp = dst;

        dst->private = NULL;

        if (!folio_trylock(src)) {
                if (mode == MIGRATE_ASYNC)
                        goto out;

                /*
                 * It's not safe for direct compaction to call lock_page.
                 * For example, during page readahead pages are added locked
                 * to the LRU. Later, when the IO completes the pages are
                 * marked uptodate and unlocked. However, the queueing
                 * could be merging multiple pages for one bio (e.g.
                 * mpage_readahead). If an allocation happens for the
                 * second or third page, the process can end up locking
                 * the same page twice and deadlocking. Rather than
                 * trying to be clever about what pages can be locked,
                 * avoid the use of lock_page for direct compaction
                 * altogether.
                 */
                if (current->flags & PF_MEMALLOC)
                        goto out;

                /*
                 * In "light" mode, we can wait for transient locks (eg
                 * inserting a page into the page table), but it's not
                 * worth waiting for I/O.
                 */
                if (mode == MIGRATE_SYNC_LIGHT && !folio_test_uptodate(src))
                        goto out;

                folio_lock(src);
        }
        locked = true;
        if (folio_test_mlocked(src))
                old_page_state |= PAGE_WAS_MLOCKED;

        if (folio_test_writeback(src)) {
                /*
                 * Only in the case of a full synchronous migration is it
                 * necessary to wait for PageWriteback. In the async case,
                 * the retry loop is too short and in the sync-light case,
                 * the overhead of stalling is too much
                 */
                switch (mode) {
                case MIGRATE_SYNC:
                        break;
                default:
                        rc = -EBUSY;
                        goto out;
                }
                folio_wait_writeback(src);
        }

        /*
         * By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
         * we cannot notice that anon_vma is freed while we migrate a page.
         * This get_anon_vma() delays freeing anon_vma pointer until the end
         * of migration. File cache pages are no problem because of page_lock()
         * File Caches may use write_page() or lock_page() in migration, then,
         * just care Anon page here.
         *
         * Only folio_get_anon_vma() understands the subtleties of
         * getting a hold on an anon_vma from outside one of its mms.
         * But if we cannot get anon_vma, then we won't need it anyway,
         * because that implies that the anon page is no longer mapped
         * (and cannot be remapped so long as we hold the page lock).
         */
        if (folio_test_anon(src) && !folio_test_ksm(src))
                anon_vma = folio_get_anon_vma(src);

        /*
         * Block others from accessing the new page when we get around to
         * establishing additional references. We are usually the only one
         * holding a reference to dst at this point. We used to have a BUG
         * here if folio_trylock(dst) fails, but would like to allow for
         * cases where there might be a race with the previous use of dst.
         * This is much like races on refcount of oldpage: just don't BUG().
         */
        if (unlikely(!folio_trylock(dst)))
                goto out;
        dst_locked = true;

        if (unlikely(!is_lru)) {
                __migrate_folio_record(dst, old_page_state, anon_vma);
                return MIGRATEPAGE_UNMAP;
        }

        /*
         * Corner case handling:
         * 1. When a new swap-cache page is read into, it is added to the LRU
         * and treated as swapcache but it has no rmap yet.
         * Calling try_to_unmap() against a src->mapping==NULL page will
         * trigger a BUG.  So handle it here.
         * 2. An orphaned page (see truncate_cleanup_page) might have
         * fs-private metadata. The page can be picked up due to memory
         * offlining.  Everywhere else except page reclaim, the page is
         * invisible to the vm, so the page can not be migrated.  So try to
         * free the metadata, so the page can be freed.
         */
        if (!src->mapping) {
                if (folio_test_private(src)) {
                        try_to_free_buffers(src);
                        goto out;
                }
        } else if (folio_mapped(src)) {
                /* Establish migration ptes */
                VM_BUG_ON_FOLIO(folio_test_anon(src) &&
                               !folio_test_ksm(src) && !anon_vma, src);
                try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
                old_page_state |= PAGE_WAS_MAPPED;
        }

        if (!folio_mapped(src)) {
                __migrate_folio_record(dst, old_page_state, anon_vma);
                return MIGRATEPAGE_UNMAP;
        }

out:
        /*
         * A folio that has not been unmapped will be restored to
         * right list unless we want to retry.
         */
        if (rc == -EAGAIN)
                ret = NULL;

        migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
                               anon_vma, locked, ret);
        migrate_folio_undo_dst(dst, dst_locked, put_new_folio, private);

        return rc;
}

/* Migrate the folio to the newly allocated folio in dst. */
static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
                              struct folio *src, struct folio *dst,
                              enum migrate_mode mode, enum migrate_reason reason,
                              struct list_head *ret)
{
        int rc;
        int old_page_state = 0;
        struct anon_vma *anon_vma = NULL;
        bool is_lru = !__folio_test_movable(src);
        struct list_head *prev;

        __migrate_folio_extract(dst, &old_page_state, &anon_vma);
        prev = dst->lru.prev;
        list_del(&dst->lru);

        rc = move_to_new_folio(dst, src, mode);
        if (rc)
                goto out;

        if (unlikely(!is_lru))
                goto out_unlock_both;

        /*
         * When successful, push dst to LRU immediately: so that if it
         * turns out to be an mlocked page, remove_migration_ptes() will
         * automatically build up the correct dst->mlock_count for it.
         *
         * We would like to do something similar for the old page, when
         * unsuccessful, and other cases when a page has been temporarily
         * isolated from the unevictable LRU: but this case is the easiest.
         */
        folio_add_lru(dst);
        if (old_page_state & PAGE_WAS_MLOCKED)
                lru_add_drain();

        if (old_page_state & PAGE_WAS_MAPPED)
                remove_migration_ptes(src, dst, 0);

out_unlock_both:
        folio_unlock(dst);
        set_page_owner_migrate_reason(&dst->page, reason);
        /*
         * If migration is successful, decrease refcount of dst,
         * which will not free the page because new page owner increased
         * refcounter.
         */
        folio_put(dst);

        /*
         * A folio that has been migrated has all references removed
         * and will be freed.
         */
        list_del(&src->lru);
        /* Drop an anon_vma reference if we took one */
        if (anon_vma)
                put_anon_vma(anon_vma);
        folio_unlock(src);
        migrate_folio_done(src, reason);

        return rc;
out:
        /*
         * A folio that has not been migrated will be restored to
         * right list unless we want to retry.
         */
        if (rc == -EAGAIN) {
                list_add(&dst->lru, prev);
                __migrate_folio_record(dst, old_page_state, anon_vma);
                return rc;
        }

        migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
                               anon_vma, true, ret);
        migrate_folio_undo_dst(dst, true, put_new_folio, private);

        return rc;
}

/*
 * Counterpart of unmap_and_move_page() for hugepage migration.
 *
 * This function doesn't wait the completion of hugepage I/O
 * because there is no race between I/O and migration for hugepage.
 * Note that currently hugepage I/O occurs only in direct I/O
 * where no lock is held and PG_writeback is irrelevant,
 * and writeback status of all subpages are counted in the reference
 * count of the head page (i.e. if all subpages of a 2MB hugepage are
 * under direct I/O, the reference of the head page is 512 and a bit more.)
 * This means that when we try to migrate hugepage whose subpages are
 * doing direct I/O, some references remain after try_to_unmap() and
 * hugepage migration fails without data corruption.
 *
 * There is also no race when direct I/O is issued on the page under migration,
 * because then pte is replaced with migration swap entry and direct I/O code
 * will wait in the page fault for migration to complete.
 */
static int unmap_and_move_huge_page(new_folio_t get_new_folio,
                free_folio_t put_new_folio, unsigned long private,
                struct folio *src, int force, enum migrate_mode mode,
                int reason, struct list_head *ret)
{
        struct folio *dst;
        int rc = -EAGAIN;
        int page_was_mapped = 0;
        struct anon_vma *anon_vma = NULL;
        struct address_space *mapping = NULL;

        if (folio_ref_count(src) == 1) {
                /* page was freed from under us. So we are done. */
                folio_putback_active_hugetlb(src);
                return MIGRATEPAGE_SUCCESS;
        }

        dst = get_new_folio(src, private);
        if (!dst)
                return -ENOMEM;

        if (!folio_trylock(src)) {
                if (!force)
                        goto out;
                switch (mode) {
                case MIGRATE_SYNC:
                        break;
                default:
                        goto out;
                }
                folio_lock(src);
        }

        /*
         * Check for pages which are in the process of being freed.  Without
         * folio_mapping() set, hugetlbfs specific move page routine will not
         * be called and we could leak usage counts for subpools.
         */
        if (hugetlb_folio_subpool(src) && !folio_mapping(src)) {
                rc = -EBUSY;
                goto out_unlock;
        }

        if (folio_test_anon(src))
                anon_vma = folio_get_anon_vma(src);

        if (unlikely(!folio_trylock(dst)))
                goto put_anon;

        if (folio_mapped(src)) {
                enum ttu_flags ttu = 0;

                if (!folio_test_anon(src)) {
                        /*
                         * In shared mappings, try_to_unmap could potentially
                         * call huge_pmd_unshare.  Because of this, take
                         * semaphore in write mode here and set TTU_RMAP_LOCKED
                         * to let lower levels know we have taken the lock.
                         */
                        mapping = hugetlb_folio_mapping_lock_write(src);
                        if (unlikely(!mapping))
                                goto unlock_put_anon;

                        ttu = TTU_RMAP_LOCKED;
                }

                try_to_migrate(src, ttu);
                page_was_mapped = 1;

                if (ttu & TTU_RMAP_LOCKED)
                        i_mmap_unlock_write(mapping);
        }

        if (!folio_mapped(src))
                rc = move_to_new_folio(dst, src, mode);

        if (page_was_mapped)
                remove_migration_ptes(src,
                        rc == MIGRATEPAGE_SUCCESS ? dst : src, 0);

unlock_put_anon:
        folio_unlock(dst);

put_anon:
        if (anon_vma)
                put_anon_vma(anon_vma);

        if (rc == MIGRATEPAGE_SUCCESS) {
                move_hugetlb_state(src, dst, reason);
                put_new_folio = NULL;
        }

out_unlock:
        folio_unlock(src);
out:
        if (rc == MIGRATEPAGE_SUCCESS)
                folio_putback_active_hugetlb(src);
        else if (rc != -EAGAIN)
                list_move_tail(&src->lru, ret);

        /*
         * If migration was not successful and there's a freeing callback, use
         * it.  Otherwise, put_page() will drop the reference grabbed during
         * isolation.
         */
        if (put_new_folio)
                put_new_folio(dst, private);
        else
                folio_putback_active_hugetlb(dst);

        return rc;
}

static inline int try_split_folio(struct folio *folio, struct list_head *split_folios,
                                  enum migrate_mode mode)
{
        int rc;

        if (mode == MIGRATE_ASYNC) {
                if (!folio_trylock(folio))
                        return -EAGAIN;
        } else {
                folio_lock(folio);
        }
        rc = split_folio_to_list(folio, split_folios);
        folio_unlock(folio);
        if (!rc)
                list_move_tail(&folio->lru, split_folios);

        return rc;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define NR_MAX_BATCHED_MIGRATION        HPAGE_PMD_NR
#else
#define NR_MAX_BATCHED_MIGRATION        512
#endif
#define NR_MAX_MIGRATE_PAGES_RETRY        10
#define NR_MAX_MIGRATE_ASYNC_RETRY        3
#define NR_MAX_MIGRATE_SYNC_RETRY                                        \
        (NR_MAX_MIGRATE_PAGES_RETRY - NR_MAX_MIGRATE_ASYNC_RETRY)

struct migrate_pages_stats {
        int nr_succeeded;        /* Normal and large folios migrated successfully, in
                                   units of base pages */
        int nr_failed_pages;        /* Normal and large folios failed to be migrated, in
                                   units of base pages.  Untried folios aren't counted */
        int nr_thp_succeeded;        /* THP migrated successfully */
        int nr_thp_failed;        /* THP failed to be migrated */
        int nr_thp_split;        /* THP split before migrating */
        int nr_split;        /* Large folio (include THP) split before migrating */
};

/*
 * Returns the number of hugetlb folios that were not migrated, or an error code
 * after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no hugetlb folios are movable
 * any more because the list has become empty or no retryable hugetlb folios
 * exist any more. It is caller's responsibility to call putback_movable_pages()
 * only if ret != 0.
 */
static int migrate_hugetlbs(struct list_head *from, new_folio_t get_new_folio,
                            free_folio_t put_new_folio, unsigned long private,
                            enum migrate_mode mode, int reason,
                            struct migrate_pages_stats *stats,
                            struct list_head *ret_folios)
{
        int retry = 1;
        int nr_failed = 0;
        int nr_retry_pages = 0;
        int pass = 0;
        struct folio *folio, *folio2;
        int rc, nr_pages;

        for (pass = 0; pass < NR_MAX_MIGRATE_PAGES_RETRY && retry; pass++) {
                retry = 0;
                nr_retry_pages = 0;

                list_for_each_entry_safe(folio, folio2, from, lru) {
                        if (!folio_test_hugetlb(folio))
                                continue;

                        nr_pages = folio_nr_pages(folio);

                        cond_resched();

                        /*
                         * Migratability of hugepages depends on architectures and
                         * their size.  This check is necessary because some callers
                         * of hugepage migration like soft offline and memory
                         * hotremove don't walk through page tables or check whether
                         * the hugepage is pmd-based or not before kicking migration.
                         */
                        if (!hugepage_migration_supported(folio_hstate(folio))) {
                                nr_failed++;
                                stats->nr_failed_pages += nr_pages;
                                list_move_tail(&folio->lru, ret_folios);
                                continue;
                        }

                        rc = unmap_and_move_huge_page(get_new_folio,
                                                      put_new_folio, private,
                                                      folio, pass > 2, mode,
                                                      reason, ret_folios);
                        /*
                         * The rules are:
                         *        Success: hugetlb folio will be put back
                         *        -EAGAIN: stay on the from list
                         *        -ENOMEM: stay on the from list
                         *        Other errno: put on ret_folios list
                         */
                        switch(rc) {
                        case -ENOMEM:
                                /*
                                 * When memory is low, don't bother to try to migrate
                                 * other folios, just exit.
                                 */
                                stats->nr_failed_pages += nr_pages + nr_retry_pages;
                                return -ENOMEM;
                        case -EAGAIN:
                                retry++;
                                nr_retry_pages += nr_pages;
                                break;
                        case MIGRATEPAGE_SUCCESS:
                                stats->nr_succeeded += nr_pages;
                                break;
                        default:
                                /*
                                 * Permanent failure (-EBUSY, etc.):
                                 * unlike -EAGAIN case, the failed folio is
                                 * removed from migration folio list and not
                                 * retried in the next outer loop.
                                 */
                                nr_failed++;
                                stats->nr_failed_pages += nr_pages;
                                break;
                        }
                }
        }
        /*
         * nr_failed is number of hugetlb folios failed to be migrated.  After
         * NR_MAX_MIGRATE_PAGES_RETRY attempts, give up and count retried hugetlb
         * folios as failed.
         */
        nr_failed += retry;
        stats->nr_failed_pages += nr_retry_pages;

        return nr_failed;
}

/*
 * migrate_pages_batch() first unmaps folios in the from list as many as
 * possible, then move the unmapped folios.
 *
 * We only batch migration if mode == MIGRATE_ASYNC to avoid to wait a
 * lock or bit when we have locked more than one folio.  Which may cause
 * deadlock (e.g., for loop device).  So, if mode != MIGRATE_ASYNC, the
 * length of the from list must be <= 1.
 */
static int migrate_pages_batch(struct list_head *from,
                new_folio_t get_new_folio, free_folio_t put_new_folio,
                unsigned long private, enum migrate_mode mode, int reason,
                struct list_head *ret_folios, struct list_head *split_folios,
                struct migrate_pages_stats *stats, int nr_pass)
{
        int retry = 1;
        int thp_retry = 1;
        int nr_failed = 0;
        int nr_retry_pages = 0;
        int pass = 0;
        bool is_thp = false;
        bool is_large = false;
        struct folio *folio, *folio2, *dst = NULL, *dst2;
        int rc, rc_saved = 0, nr_pages;
        LIST_HEAD(unmap_folios);
        LIST_HEAD(dst_folios);
        bool nosplit = (reason == MR_NUMA_MISPLACED);

        VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
                        !list_empty(from) && !list_is_singular(from));

        for (pass = 0; pass < nr_pass && retry; pass++) {
                retry = 0;
                thp_retry = 0;
                nr_retry_pages = 0;

                list_for_each_entry_safe(folio, folio2, from, lru) {
                        is_large = folio_test_large(folio);
                        is_thp = is_large && folio_test_pmd_mappable(folio);
                        nr_pages = folio_nr_pages(folio);

                        cond_resched();

                        /*
                         * The rare folio on the deferred split list should
                         * be split now. It should not count as a failure:
                         * but increment nr_failed because, without doing so,
                         * migrate_pages() may report success with (split but
                         * unmigrated) pages still on its fromlist; whereas it
                         * always reports success when its fromlist is empty.
                         * stats->nr_thp_failed should be increased too,
                         * otherwise stats inconsistency will happen when
                         * migrate_pages_batch is called via migrate_pages()
                         * with MIGRATE_SYNC and MIGRATE_ASYNC.
                         *
                         * Only check it without removing it from the list.
                         * Since the folio can be on deferred_split_scan()
                         * local list and removing it can cause the local list
                         * corruption. Folio split process below can handle it
                         * with the help of folio_ref_freeze().
                         *
                         * nr_pages > 2 is needed to avoid checking order-1
                         * page cache folios. They exist, in contrast to
                         * non-existent order-1 anonymous folios, and do not
                         * use _deferred_list.
                         */
                        if (nr_pages > 2 &&
                           !list_empty(&folio->_deferred_list) &&
                           folio_test_partially_mapped(folio)) {
                                if (!try_split_folio(folio, split_folios, mode)) {
                                        nr_failed++;
                                        stats->nr_thp_failed += is_thp;
                                        stats->nr_thp_split += is_thp;
                                        stats->nr_split++;
                                        continue;
                                }
                        }

                        /*
                         * Large folio migration might be unsupported or
                         * the allocation might be failed so we should retry
                         * on the same folio with the large folio split
                         * to normal folios.
                         *
                         * Split folios are put in split_folios, and
                         * we will migrate them after the rest of the
                         * list is processed.
                         */
                        if (!thp_migration_supported() && is_thp) {
                                nr_failed++;
                                stats->nr_thp_failed++;
                                if (!try_split_folio(folio, split_folios, mode)) {
                                        stats->nr_thp_split++;
                                        stats->nr_split++;
                                        continue;
                                }
                                stats->nr_failed_pages += nr_pages;
                                list_move_tail(&folio->lru, ret_folios);
                                continue;
                        }

                        rc = migrate_folio_unmap(get_new_folio, put_new_folio,
                                        private, folio, &dst, mode, reason,
                                        ret_folios);
                        /*
                         * The rules are:
                         *        Success: folio will be freed
                         *        Unmap: folio will be put on unmap_folios list,
                         *               dst folio put on dst_folios list
                         *        -EAGAIN: stay on the from list
                         *        -ENOMEM: stay on the from list
                         *        Other errno: put on ret_folios list
                         */
                        switch(rc) {
                        case -ENOMEM:
                                /*
                                 * When memory is low, don't bother to try to migrate
                                 * other folios, move unmapped folios, then exit.
                                 */
                                nr_failed++;
                                stats->nr_thp_failed += is_thp;
                                /* Large folio NUMA faulting doesn't split to retry. */
                                if (is_large && !nosplit) {
                                        int ret = try_split_folio(folio, split_folios, mode);

                                        if (!ret) {
                                                stats->nr_thp_split += is_thp;
                                                stats->nr_split++;
                                                break;
                                        } else if (reason == MR_LONGTERM_PIN &&
                                                   ret == -EAGAIN) {
                                                /*
                                                 * Try again to split large folio to
                                                 * mitigate the failure of longterm pinning.
                                                 */
                                                retry++;
                                                thp_retry += is_thp;
                                                nr_retry_pages += nr_pages;
                                                /* Undo duplicated failure counting. */
                                                nr_failed--;
                                                stats->nr_thp_failed -= is_thp;
                                                break;
                                        }
                                }

                                stats->nr_failed_pages += nr_pages + nr_retry_pages;
                                /* nr_failed isn't updated for not used */
                                stats->nr_thp_failed += thp_retry;
                                rc_saved = rc;
                                if (list_empty(&unmap_folios))
                                        goto out;
                                else
                                        goto move;
                        case -EAGAIN:
                                retry++;
                                thp_retry += is_thp;
                                nr_retry_pages += nr_pages;
                                break;
                        case MIGRATEPAGE_SUCCESS:
                                stats->nr_succeeded += nr_pages;
                                stats->nr_thp_succeeded += is_thp;
                                break;
                        case MIGRATEPAGE_UNMAP:
                                list_move_tail(&folio->lru, &unmap_folios);
                                list_add_tail(&dst->lru, &dst_folios);
                                break;
                        default:
                                /*
                                 * Permanent failure (-EBUSY, etc.):
                                 * unlike -EAGAIN case, the failed folio is
                                 * removed from migration folio list and not
                                 * retried in the next outer loop.
                                 */
                                nr_failed++;
                                stats->nr_thp_failed += is_thp;
                                stats->nr_failed_pages += nr_pages;
                                break;
                        }
                }
        }
        nr_failed += retry;
        stats->nr_thp_failed += thp_retry;
        stats->nr_failed_pages += nr_retry_pages;
move:
        /* Flush TLBs for all unmapped folios */
        try_to_unmap_flush();

        retry = 1;
        for (pass = 0; pass < nr_pass && retry; pass++) {
                retry = 0;
                thp_retry = 0;
                nr_retry_pages = 0;

                dst = list_first_entry(&dst_folios, struct folio, lru);
                dst2 = list_next_entry(dst, lru);
                list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
                        is_thp = folio_test_large(folio) && folio_test_pmd_mappable(folio);
                        nr_pages = folio_nr_pages(folio);

                        cond_resched();

                        rc = migrate_folio_move(put_new_folio, private,
                                                folio, dst, mode,
                                                reason, ret_folios);
                        /*
                         * The rules are:
                         *        Success: folio will be freed
                         *        -EAGAIN: stay on the unmap_folios list
                         *        Other errno: put on ret_folios list
                         */
                        switch(rc) {
                        case -EAGAIN:
                                retry++;
                                thp_retry += is_thp;
                                nr_retry_pages += nr_pages;
                                break;
                        case MIGRATEPAGE_SUCCESS:
                                stats->nr_succeeded += nr_pages;
                                stats->nr_thp_succeeded += is_thp;
                                break;
                        default:
                                nr_failed++;
                                stats->nr_thp_failed += is_thp;
                                stats->nr_failed_pages += nr_pages;
                                break;
                        }
                        dst = dst2;
                        dst2 = list_next_entry(dst, lru);
                }
        }
        nr_failed += retry;
        stats->nr_thp_failed += thp_retry;
        stats->nr_failed_pages += nr_retry_pages;

        rc = rc_saved ? : nr_failed;
out:
        /* Cleanup remaining folios */
        dst = list_first_entry(&dst_folios, struct folio, lru);
        dst2 = list_next_entry(dst, lru);
        list_for_each_entry_safe(folio, folio2, &unmap_folios, lru) {
                int old_page_state = 0;
                struct anon_vma *anon_vma = NULL;

                __migrate_folio_extract(dst, &old_page_state, &anon_vma);
                migrate_folio_undo_src(folio, old_page_state & PAGE_WAS_MAPPED,
                                       anon_vma, true, ret_folios);
                list_del(&dst->lru);
                migrate_folio_undo_dst(dst, true, put_new_folio, private);
                dst = dst2;
                dst2 = list_next_entry(dst, lru);
        }

        return rc;
}

static int migrate_pages_sync(struct list_head *from, new_folio_t get_new_folio,
                free_folio_t put_new_folio, unsigned long private,
                enum migrate_mode mode, int reason,
                struct list_head *ret_folios, struct list_head *split_folios,
                struct migrate_pages_stats *stats)
{
        int rc, nr_failed = 0;
        LIST_HEAD(folios);
        struct migrate_pages_stats astats;

        memset(&astats, 0, sizeof(astats));
        /* Try to migrate in batch with MIGRATE_ASYNC mode firstly */
        rc = migrate_pages_batch(from, get_new_folio, put_new_folio, private, MIGRATE_ASYNC,
                                 reason, &folios, split_folios, &astats,
                                 NR_MAX_MIGRATE_ASYNC_RETRY);
        stats->nr_succeeded += astats.nr_succeeded;
        stats->nr_thp_succeeded += astats.nr_thp_succeeded;
        stats->nr_thp_split += astats.nr_thp_split;
        stats->nr_split += astats.nr_split;
        if (rc < 0) {
                stats->nr_failed_pages += astats.nr_failed_pages;
                stats->nr_thp_failed += astats.nr_thp_failed;
                list_splice_tail(&folios, ret_folios);
                return rc;
        }
        stats->nr_thp_failed += astats.nr_thp_split;
        /*
         * Do not count rc, as pages will be retried below.
         * Count nr_split only, since it includes nr_thp_split.
         */
        nr_failed += astats.nr_split;
        /*
         * Fall back to migrate all failed folios one by one synchronously. All
         * failed folios except split THPs will be retried, so their failure
         * isn't counted
         */
        list_splice_tail_init(&folios, from);
        while (!list_empty(from)) {
                list_move(from->next, &folios);
                rc = migrate_pages_batch(&folios, get_new_folio, put_new_folio,
                                         private, mode, reason, ret_folios,
                                         split_folios, stats, NR_MAX_MIGRATE_SYNC_RETRY);
                list_splice_tail_init(&folios, ret_folios);
                if (rc < 0)
                        return rc;
                nr_failed += rc;
        }

        return nr_failed;
}

/*
 * migrate_pages - migrate the folios specified in a list, to the free folios
 *                   supplied as the target for the page migration
 *
 * @from:                The list of folios to be migrated.
 * @get_new_folio:        The function used to allocate free folios to be used
 *                        as the target of the folio migration.
 * @put_new_folio:        The function used to free target folios if migration
 *                        fails, or NULL if no special handling is necessary.
 * @private:                Private data to be passed on to get_new_folio()
 * @mode:                The migration mode that specifies the constraints for
 *                        folio migration, if any.
 * @reason:                The reason for folio migration.
 * @ret_succeeded:        Set to the number of folios migrated successfully if
 *                        the caller passes a non-NULL pointer.
 *
 * The function returns after NR_MAX_MIGRATE_PAGES_RETRY attempts or if no folios
 * are movable any more because the list has become empty or no retryable folios
 * exist any more. It is caller's responsibility to call putback_movable_pages()
 * only if ret != 0.
 *
 * Returns the number of {normal folio, large folio, hugetlb} that were not
 * migrated, or an error code. The number of large folio splits will be
 * considered as the number of non-migrated large folio, no matter how many
 * split folios of the large folio are migrated successfully.
 */
int migrate_pages(struct list_head *from, new_folio_t get_new_folio,
                free_folio_t put_new_folio, unsigned long private,
                enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
{
        int rc, rc_gather;
        int nr_pages;
        struct folio *folio, *folio2;
        LIST_HEAD(folios);
        LIST_HEAD(ret_folios);
        LIST_HEAD(split_folios);
        struct migrate_pages_stats stats;

        trace_mm_migrate_pages_start(mode, reason);

        memset(&stats, 0, sizeof(stats));

        rc_gather = migrate_hugetlbs(from, get_new_folio, put_new_folio, private,
                                     mode, reason, &stats, &ret_folios);
        if (rc_gather < 0)
                goto out;

again:
        nr_pages = 0;
        list_for_each_entry_safe(folio, folio2, from, lru) {
                /* Retried hugetlb folios will be kept in list  */
                if (folio_test_hugetlb(folio)) {
                        list_move_tail(&folio->lru, &ret_folios);
                        continue;
                }

                nr_pages += folio_nr_pages(folio);
                if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
                        break;
        }
        if (nr_pages >= NR_MAX_BATCHED_MIGRATION)
                list_cut_before(&folios, from, &folio2->lru);
        else
                list_splice_init(from, &folios);
        if (mode == MIGRATE_ASYNC)
                rc = migrate_pages_batch(&folios, get_new_folio, put_new_folio,
                                private, mode, reason, &ret_folios,
                                &split_folios, &stats,
                                NR_MAX_MIGRATE_PAGES_RETRY);
        else
                rc = migrate_pages_sync(&folios, get_new_folio, put_new_folio,
                                private, mode, reason, &ret_folios,
                                &split_folios, &stats);
        list_splice_tail_init(&folios, &ret_folios);
        if (rc < 0) {
                rc_gather = rc;
                list_splice_tail(&split_folios, &ret_folios);
                goto out;
        }
        if (!list_empty(&split_folios)) {
                /*
                 * Failure isn't counted since all split folios of a large folio
                 * is counted as 1 failure already.  And, we only try to migrate
                 * with minimal effort, force MIGRATE_ASYNC mode and retry once.
                 */
                migrate_pages_batch(&split_folios, get_new_folio,
                                put_new_folio, private, MIGRATE_ASYNC, reason,
                                &ret_folios, NULL, &stats, 1);
                list_splice_tail_init(&split_folios, &ret_folios);
        }
        rc_gather += rc;
        if (!list_empty(from))
                goto again;
out:
        /*
         * Put the permanent failure folio back to migration list, they
         * will be put back to the right list by the caller.
         */
        list_splice(&ret_folios, from);

        /*
         * Return 0 in case all split folios of fail-to-migrate large folios
         * are migrated successfully.
         */
        if (list_empty(from))
                rc_gather = 0;

        count_vm_events(PGMIGRATE_SUCCESS, stats.nr_succeeded);
        count_vm_events(PGMIGRATE_FAIL, stats.nr_failed_pages);
        count_vm_events(THP_MIGRATION_SUCCESS, stats.nr_thp_succeeded);
        count_vm_events(THP_MIGRATION_FAIL, stats.nr_thp_failed);
        count_vm_events(THP_MIGRATION_SPLIT, stats.nr_thp_split);
        trace_mm_migrate_pages(stats.nr_succeeded, stats.nr_failed_pages,
                               stats.nr_thp_succeeded, stats.nr_thp_failed,
                               stats.nr_thp_split, stats.nr_split, mode,
                               reason);

        if (ret_succeeded)
                *ret_succeeded = stats.nr_succeeded;

        return rc_gather;
}

struct folio *alloc_migration_target(struct folio *src, unsigned long private)
{
        struct migration_target_control *mtc;
        gfp_t gfp_mask;
        unsigned int order = 0;
        int nid;
        int zidx;

        mtc = (struct migration_target_control *)private;
        gfp_mask = mtc->gfp_mask;
        nid = mtc->nid;
        if (nid == NUMA_NO_NODE)
                nid = folio_nid(src);

        if (folio_test_hugetlb(src)) {
                struct hstate *h = folio_hstate(src);

                gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
                return alloc_hugetlb_folio_nodemask(h, nid,
                                                mtc->nmask, gfp_mask,
                                                htlb_allow_alloc_fallback(mtc->reason));
        }

        if (folio_test_large(src)) {
                /*
                 * clear __GFP_RECLAIM to make the migration callback
                 * consistent with regular THP allocations.
                 */
                gfp_mask &= ~__GFP_RECLAIM;
                gfp_mask |= GFP_TRANSHUGE;
                order = folio_order(src);
        }
        zidx = zone_idx(folio_zone(src));
        if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
                gfp_mask |= __GFP_HIGHMEM;

        return __folio_alloc(gfp_mask, order, nid, mtc->nmask);
}

#ifdef CONFIG_NUMA

static int store_status(int __user *status, int start, int value, int nr)
{
        while (nr-- > 0) {
                if (put_user(value, status + start))
                        return -EFAULT;
                start++;
        }

        return 0;
}

static int do_move_pages_to_node(struct list_head *pagelist, int node)
{
        int err;
        struct migration_target_control mtc = {
                .nid = node,
                .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
                .reason = MR_SYSCALL,
        };

        err = migrate_pages(pagelist, alloc_migration_target, NULL,
                (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
        if (err)
                putback_movable_pages(pagelist);
        return err;
}

static int __add_folio_for_migration(struct folio *folio, int node,
                struct list_head *pagelist, bool migrate_all)
{
        if (is_zero_folio(folio) || is_huge_zero_folio(folio))
                return -EFAULT;

        if (folio_is_zone_device(folio))
                return -ENOENT;

        if (folio_nid(folio) == node)
                return 0;

        if (folio_likely_mapped_shared(folio) && !migrate_all)
                return -EACCES;

        if (folio_test_hugetlb(folio)) {
                if (isolate_hugetlb(folio, pagelist))
                        return 1;
        } else if (folio_isolate_lru(folio)) {
                list_add_tail(&folio->lru, pagelist);
                node_stat_mod_folio(folio,
                        NR_ISOLATED_ANON + folio_is_file_lru(folio),
                        folio_nr_pages(folio));
                return 1;
        }
        return -EBUSY;
}

/*
 * Resolves the given address to a struct folio, isolates it from the LRU and
 * puts it to the given pagelist.
 * Returns:
 *     errno - if the folio cannot be found/isolated
 *     0 - when it doesn't have to be migrated because it is already on the
 *         target node
 *     1 - when it has been queued
 */
static int add_folio_for_migration(struct mm_struct *mm, const void __user *p,
                int node, struct list_head *pagelist, bool migrate_all)
{
        struct vm_area_struct *vma;
        struct folio_walk fw;
        struct folio *folio;
        unsigned long addr;
        int err = -EFAULT;

        mmap_read_lock(mm);
        addr = (unsigned long)untagged_addr_remote(mm, p);

        vma = vma_lookup(mm, addr);
        if (vma && vma_migratable(vma)) {
                folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
                if (folio) {
                        err = __add_folio_for_migration(folio, node, pagelist,
                                                        migrate_all);
                        folio_walk_end(&fw, vma);
                } else {
                        err = -ENOENT;
                }
        }
        mmap_read_unlock(mm);
        return err;
}

static int move_pages_and_store_status(int node,
                struct list_head *pagelist, int __user *status,
                int start, int i, unsigned long nr_pages)
{
        int err;

        if (list_empty(pagelist))
                return 0;

        err = do_move_pages_to_node(pagelist, node);
        if (err) {
                /*
                 * Positive err means the number of failed
                 * pages to migrate.  Since we are going to
                 * abort and return the number of non-migrated
                 * pages, so need to include the rest of the
                 * nr_pages that have not been attempted as
                 * well.
                 */
                if (err > 0)
                        err += nr_pages - i;
                return err;
        }
        return store_status(status, start, node, i - start);
}

/*
 * Migrate an array of page address onto an array of nodes and fill
 * the corresponding array of status.
 */
static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
                         unsigned long nr_pages,
                         const void __user * __user *pages,
                         const int __user *nodes,
                         int __user *status, int flags)
{
        compat_uptr_t __user *compat_pages = (void __user *)pages;
        int current_node = NUMA_NO_NODE;
        LIST_HEAD(pagelist);
        int start, i;
        int err = 0, err1;

        lru_cache_disable();

        for (i = start = 0; i < nr_pages; i++) {
                const void __user *p;
                int node;

                err = -EFAULT;
                if (in_compat_syscall()) {
                        compat_uptr_t cp;

                        if (get_user(cp, compat_pages + i))
                                goto out_flush;

                        p = compat_ptr(cp);
                } else {
                        if (get_user(p, pages + i))
                                goto out_flush;
                }
                if (get_user(node, nodes + i))
                        goto out_flush;

                err = -ENODEV;
                if (node < 0 || node >= MAX_NUMNODES)
                        goto out_flush;
                if (!node_state(node, N_MEMORY))
                        goto out_flush;

                err = -EACCES;
                if (!node_isset(node, task_nodes))
                        goto out_flush;

                if (current_node == NUMA_NO_NODE) {
                        current_node = node;
                        start = i;
                } else if (node != current_node) {
                        err = move_pages_and_store_status(current_node,
                                        &pagelist, status, start, i, nr_pages);
                        if (err)
                                goto out;
                        start = i;
                        current_node = node;
                }

                /*
                 * Errors in the page lookup or isolation are not fatal and we simply
                 * report them via status
                 */
                err = add_folio_for_migration(mm, p, current_node, &pagelist,
                                              flags & MPOL_MF_MOVE_ALL);

                if (err > 0) {
                        /* The page is successfully queued for migration */
                        continue;
                }

                /*
                 * The move_pages() man page does not have an -EEXIST choice, so
                 * use -EFAULT instead.
                 */
                if (err == -EEXIST)
                        err = -EFAULT;

                /*
                 * If the page is already on the target node (!err), store the
                 * node, otherwise, store the err.
                 */
                err = store_status(status, i, err ? : current_node, 1);
                if (err)
                        goto out_flush;

                err = move_pages_and_store_status(current_node, &pagelist,
                                status, start, i, nr_pages);
                if (err) {
                        /* We have accounted for page i */
                        if (err > 0)
                                err--;
                        goto out;
                }
                current_node = NUMA_NO_NODE;
        }
out_flush:
        /* Make sure we do not overwrite the existing error */
        err1 = move_pages_and_store_status(current_node, &pagelist,
                                status, start, i, nr_pages);
        if (err >= 0)
                err = err1;
out:
        lru_cache_enable();
        return err;
}

/*
 * Determine the nodes of an array of pages and store it in an array of status.
 */
static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
                                const void __user **pages, int *status)
{
        unsigned long i;

        mmap_read_lock(mm);

        for (i = 0; i < nr_pages; i++) {
                unsigned long addr = (unsigned long)(*pages);
                struct vm_area_struct *vma;
                struct folio_walk fw;
                struct folio *folio;
                int err = -EFAULT;

                vma = vma_lookup(mm, addr);
                if (!vma)
                        goto set_status;

                folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
                if (folio) {
                        if (is_zero_folio(folio) || is_huge_zero_folio(folio))
                                err = -EFAULT;
                        else if (folio_is_zone_device(folio))
                                err = -ENOENT;
                        else
                                err = folio_nid(folio);
                        folio_walk_end(&fw, vma);
                } else {
                        err = -ENOENT;
                }
set_status:
                *status = err;

                pages++;
                status++;
        }

        mmap_read_unlock(mm);
}

static int get_compat_pages_array(const void __user *chunk_pages[],
                                  const void __user * __user *pages,
                                  unsigned long chunk_nr)
{
        compat_uptr_t __user *pages32 = (compat_uptr_t __user *)pages;
        compat_uptr_t p;
        int i;

        for (i = 0; i < chunk_nr; i++) {
                if (get_user(p, pages32 + i))
                        return -EFAULT;
                chunk_pages[i] = compat_ptr(p);
        }

        return 0;
}

/*
 * Determine the nodes of a user array of pages and store it in
 * a user array of status.
 */
static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
                         const void __user * __user *pages,
                         int __user *status)
{
#define DO_PAGES_STAT_CHUNK_NR 16UL
        const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
        int chunk_status[DO_PAGES_STAT_CHUNK_NR];

        while (nr_pages) {
                unsigned long chunk_nr = min(nr_pages, DO_PAGES_STAT_CHUNK_NR);

                if (in_compat_syscall()) {
                        if (get_compat_pages_array(chunk_pages, pages,
                                                   chunk_nr))
                                break;
                } else {
                        if (copy_from_user(chunk_pages, pages,
                                      chunk_nr * sizeof(*chunk_pages)))
                                break;
                }

                do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);

                if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
                        break;

                pages += chunk_nr;
                status += chunk_nr;
                nr_pages -= chunk_nr;
        }
        return nr_pages ? -EFAULT : 0;
}

static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
{
        struct task_struct *task;
        struct mm_struct *mm;

        /*
         * There is no need to check if current process has the right to modify
         * the specified process when they are same.
         */
        if (!pid) {
                mmget(current->mm);
                *mem_nodes = cpuset_mems_allowed(current);
                return current->mm;
        }

        task = find_get_task_by_vpid(pid);
        if (!task) {
                return ERR_PTR(-ESRCH);
        }

        /*
         * Check if this process has the right to modify the specified
         * process. Use the regular "ptrace_may_access()" checks.
         */
        if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
                mm = ERR_PTR(-EPERM);
                goto out;
        }

        mm = ERR_PTR(security_task_movememory(task));
        if (IS_ERR(mm))
                goto out;
        *mem_nodes = cpuset_mems_allowed(task);
        mm = get_task_mm(task);
out:
        put_task_struct(task);
        if (!mm)
                mm = ERR_PTR(-EINVAL);
        return mm;
}

/*
 * Move a list of pages in the address space of the currently executing
 * process.
 */
static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
                             const void __user * __user *pages,
                             const int __user *nodes,
                             int __user *status, int flags)
{
        struct mm_struct *mm;
        int err;
        nodemask_t task_nodes;

        /* Check flags */
        if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
                return -EINVAL;

        if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
                return -EPERM;

        mm = find_mm_struct(pid, &task_nodes);
        if (IS_ERR(mm))
                return PTR_ERR(mm);

        if (nodes)
                err = do_pages_move(mm, task_nodes, nr_pages, pages,
                                    nodes, status, flags);
        else
                err = do_pages_stat(mm, nr_pages, pages, status);

        mmput(mm);
        return err;
}

SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
                const void __user * __user *, pages,
                const int __user *, nodes,
                int __user *, status, int, flags)
{
        return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
}

#ifdef CONFIG_NUMA_BALANCING
/*
 * Returns true if this is a safe migration target node for misplaced NUMA
 * pages. Currently it only checks the watermarks which is crude.
 */
static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
                                   unsigned long nr_migrate_pages)
{
        int z;

        for (z = pgdat->nr_zones - 1; z >= 0; z--) {
                struct zone *zone = pgdat->node_zones + z;

                if (!managed_zone(zone))
                        continue;

                /* Avoid waking kswapd by allocating pages_to_migrate pages. */
                if (!zone_watermark_ok(zone, 0,
                                       high_wmark_pages(zone) +
                                       nr_migrate_pages,
                                       ZONE_MOVABLE, ALLOC_CMA))
                        continue;
                return true;
        }
        return false;
}

static struct folio *alloc_misplaced_dst_folio(struct folio *src,
                                           unsigned long data)
{
        int nid = (int) data;
        int order = folio_order(src);
        gfp_t gfp = __GFP_THISNODE;

        if (order > 0)
                gfp |= GFP_TRANSHUGE_LIGHT;
        else {
                gfp |= GFP_HIGHUSER_MOVABLE | __GFP_NOMEMALLOC | __GFP_NORETRY |
                        __GFP_NOWARN;
                gfp &= ~__GFP_RECLAIM;
        }
        return __folio_alloc_node(gfp, order, nid);
}

/*
 * Prepare for calling migrate_misplaced_folio() by isolating the folio if
 * permitted. Must be called with the PTL still held.
 */
int migrate_misplaced_folio_prepare(struct folio *folio,
                struct vm_area_struct *vma, int node)
{
        int nr_pages = folio_nr_pages(folio);
        pg_data_t *pgdat = NODE_DATA(node);

        if (folio_is_file_lru(folio)) {
                /*
                 * Do not migrate file folios that are mapped in multiple
                 * processes with execute permissions as they are probably
                 * shared libraries.
                 *
                 * See folio_likely_mapped_shared() on possible imprecision
                 * when we cannot easily detect if a folio is shared.
                 */
                if ((vma->vm_flags & VM_EXEC) &&
                    folio_likely_mapped_shared(folio))
                        return -EACCES;

                /*
                 * Do not migrate dirty folios as not all filesystems can move
                 * dirty folios in MIGRATE_ASYNC mode which is a waste of
                 * cycles.
                 */
                if (folio_test_dirty(folio))
                        return -EAGAIN;
        }

        /* Avoid migrating to a node that is nearly full */
        if (!migrate_balanced_pgdat(pgdat, nr_pages)) {
                int z;

                if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING))
                        return -EAGAIN;
                for (z = pgdat->nr_zones - 1; z >= 0; z--) {
                        if (managed_zone(pgdat->node_zones + z))
                                break;
                }

                /*
                 * If there are no managed zones, it should not proceed
                 * further.
                 */
                if (z < 0)
                        return -EAGAIN;

                wakeup_kswapd(pgdat->node_zones + z, 0,
                              folio_order(folio), ZONE_MOVABLE);
                return -EAGAIN;
        }

        if (!folio_isolate_lru(folio))
                return -EAGAIN;

        node_stat_mod_folio(folio, NR_ISOLATED_ANON + folio_is_file_lru(folio),
                            nr_pages);
        return 0;
}

/*
 * Attempt to migrate a misplaced folio to the specified destination
 * node. Caller is expected to have isolated the folio by calling
 * migrate_misplaced_folio_prepare(), which will result in an
 * elevated reference count on the folio. This function will un-isolate the
 * folio, dereferencing the folio before returning.
 */
int migrate_misplaced_folio(struct folio *folio, struct vm_area_struct *vma,
                            int node)
{
        pg_data_t *pgdat = NODE_DATA(node);
        int nr_remaining;
        unsigned int nr_succeeded;
        LIST_HEAD(migratepages);
        struct mem_cgroup *memcg = get_mem_cgroup_from_folio(folio);
        struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);

        list_add(&folio->lru, &migratepages);
        nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_folio,
                                     NULL, node, MIGRATE_ASYNC,
                                     MR_NUMA_MISPLACED, &nr_succeeded);
        if (nr_remaining && !list_empty(&migratepages))
                putback_movable_pages(&migratepages);
        if (nr_succeeded) {
                count_vm_numa_events(NUMA_PAGE_MIGRATE, nr_succeeded);
                count_memcg_events(memcg, NUMA_PAGE_MIGRATE, nr_succeeded);
                if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
                    && !node_is_toptier(folio_nid(folio))
                    && node_is_toptier(node))
                        mod_lruvec_state(lruvec, PGPROMOTE_SUCCESS, nr_succeeded);
        }
        mem_cgroup_put(memcg);
        BUG_ON(!list_empty(&migratepages));
        return nr_remaining ? -EAGAIN : 0;
}
#endif /* CONFIG_NUMA_BALANCING */
#endif /* CONFIG_NUMA */





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 













































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
// SPDX-License-Identifier: GPL-2.0

#include <linux/ceph/ceph_debug.h>

#include <linux/module.h>
#include <linux/slab.h>

#include <linux/ceph/libceph.h>
#include <linux/ceph/osdmap.h>
#include <linux/ceph/decode.h>
#include <linux/crush/hash.h>
#include <linux/crush/mapper.h>

static __printf(2, 3)
void osdmap_info(const struct ceph_osdmap *map, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);
        vaf.fmt = fmt;
        vaf.va = &args;

        printk(KERN_INFO "%s (%pU e%u): %pV", KBUILD_MODNAME, &map->fsid,
               map->epoch, &vaf);

        va_end(args);
}

char *ceph_osdmap_state_str(char *str, int len, u32 state)
{
        if (!len)
                return str;

        if ((state & CEPH_OSD_EXISTS) && (state & CEPH_OSD_UP))
                snprintf(str, len, "exists, up");
        else if (state & CEPH_OSD_EXISTS)
                snprintf(str, len, "exists");
        else if (state & CEPH_OSD_UP)
                snprintf(str, len, "up");
        else
                snprintf(str, len, "doesn't exist");

        return str;
}

/* maps */

static int calc_bits_of(unsigned int t)
{
        int b = 0;
        while (t) {
                t = t >> 1;
                b++;
        }
        return b;
}

/*
 * the foo_mask is the smallest value 2^n-1 that is >= foo.
 */
static void calc_pg_masks(struct ceph_pg_pool_info *pi)
{
        pi->pg_num_mask = (1 << calc_bits_of(pi->pg_num-1)) - 1;
        pi->pgp_num_mask = (1 << calc_bits_of(pi->pgp_num-1)) - 1;
}

/*
 * decode crush map
 */
static int crush_decode_uniform_bucket(void **p, void *end,
                                       struct crush_bucket_uniform *b)
{
        dout("crush_decode_uniform_bucket %p to %p\n", *p, end);
        ceph_decode_need(p, end, (1+b->h.size) * sizeof(u32), bad);
        b->item_weight = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_list_bucket(void **p, void *end,
                                    struct crush_bucket_list *b)
{
        int j;
        dout("crush_decode_list_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        b->sum_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->sum_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, 2 * b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++) {
                b->item_weights[j] = ceph_decode_32(p);
                b->sum_weights[j] = ceph_decode_32(p);
        }
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_tree_bucket(void **p, void *end,
                                    struct crush_bucket_tree *b)
{
        int j;
        dout("crush_decode_tree_bucket %p to %p\n", *p, end);
        ceph_decode_8_safe(p, end, b->num_nodes, bad);
        b->node_weights = kcalloc(b->num_nodes, sizeof(u32), GFP_NOFS);
        if (b->node_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, b->num_nodes * sizeof(u32), bad);
        for (j = 0; j < b->num_nodes; j++)
                b->node_weights[j] = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_straw_bucket(void **p, void *end,
                                     struct crush_bucket_straw *b)
{
        int j;
        dout("crush_decode_straw_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        b->straws = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->straws == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, 2 * b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++) {
                b->item_weights[j] = ceph_decode_32(p);
                b->straws[j] = ceph_decode_32(p);
        }
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_straw2_bucket(void **p, void *end,
                                      struct crush_bucket_straw2 *b)
{
        int j;
        dout("crush_decode_straw2_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++)
                b->item_weights[j] = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

struct crush_name_node {
        struct rb_node cn_node;
        int cn_id;
        char cn_name[];
};

static struct crush_name_node *alloc_crush_name(size_t name_len)
{
        struct crush_name_node *cn;

        cn = kmalloc(sizeof(*cn) + name_len + 1, GFP_NOIO);
        if (!cn)
                return NULL;

        RB_CLEAR_NODE(&cn->cn_node);
        return cn;
}

static void free_crush_name(struct crush_name_node *cn)
{
        WARN_ON(!RB_EMPTY_NODE(&cn->cn_node));

        kfree(cn);
}

DEFINE_RB_FUNCS(crush_name, struct crush_name_node, cn_id, cn_node)

static int decode_crush_names(void **p, void *end, struct rb_root *root)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct crush_name_node *cn;
                int id;
                u32 name_len;

                ceph_decode_32_safe(p, end, id, e_inval);
                ceph_decode_32_safe(p, end, name_len, e_inval);
                ceph_decode_need(p, end, name_len, e_inval);

                cn = alloc_crush_name(name_len);
                if (!cn)
                        return -ENOMEM;

                cn->cn_id = id;
                memcpy(cn->cn_name, *p, name_len);
                cn->cn_name[name_len] = '\0';
                *p += name_len;

                if (!__insert_crush_name(root, cn)) {
                        free_crush_name(cn);
                        return -EEXIST;
                }
        }

        return 0;

e_inval:
        return -EINVAL;
}

void clear_crush_names(struct rb_root *root)
{
        while (!RB_EMPTY_ROOT(root)) {
                struct crush_name_node *cn =
                    rb_entry(rb_first(root), struct crush_name_node, cn_node);

                erase_crush_name(root, cn);
                free_crush_name(cn);
        }
}

static struct crush_choose_arg_map *alloc_choose_arg_map(void)
{
        struct crush_choose_arg_map *arg_map;

        arg_map = kzalloc(sizeof(*arg_map), GFP_NOIO);
        if (!arg_map)
                return NULL;

        RB_CLEAR_NODE(&arg_map->node);
        return arg_map;
}

static void free_choose_arg_map(struct crush_choose_arg_map *arg_map)
{
        if (arg_map) {
                int i, j;

                WARN_ON(!RB_EMPTY_NODE(&arg_map->node));

                for (i = 0; i < arg_map->size; i++) {
                        struct crush_choose_arg *arg = &arg_map->args[i];

                        for (j = 0; j < arg->weight_set_size; j++)
                                kfree(arg->weight_set[j].weights);
                        kfree(arg->weight_set);
                        kfree(arg->ids);
                }
                kfree(arg_map->args);
                kfree(arg_map);
        }
}

DEFINE_RB_FUNCS(choose_arg_map, struct crush_choose_arg_map, choose_args_index,
                node);

void clear_choose_args(struct crush_map *c)
{
        while (!RB_EMPTY_ROOT(&c->choose_args)) {
                struct crush_choose_arg_map *arg_map =
                    rb_entry(rb_first(&c->choose_args),
                             struct crush_choose_arg_map, node);

                erase_choose_arg_map(&c->choose_args, arg_map);
                free_choose_arg_map(arg_map);
        }
}

static u32 *decode_array_32_alloc(void **p, void *end, u32 *plen)
{
        u32 *a = NULL;
        u32 len;
        int ret;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len) {
                u32 i;

                a = kmalloc_array(len, sizeof(u32), GFP_NOIO);
                if (!a) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_need(p, end, len * sizeof(u32), e_inval);
                for (i = 0; i < len; i++)
                        a[i] = ceph_decode_32(p);
        }

        *plen = len;
        return a;

e_inval:
        ret = -EINVAL;
fail:
        kfree(a);
        return ERR_PTR(ret);
}

/*
 * Assumes @arg is zero-initialized.
 */
static int decode_choose_arg(void **p, void *end, struct crush_choose_arg *arg)
{
        int ret;

        ceph_decode_32_safe(p, end, arg->weight_set_size, e_inval);
        if (arg->weight_set_size) {
                u32 i;

                arg->weight_set = kmalloc_array(arg->weight_set_size,
                                                sizeof(*arg->weight_set),
                                                GFP_NOIO);
                if (!arg->weight_set)
                        return -ENOMEM;

                for (i = 0; i < arg->weight_set_size; i++) {
                        struct crush_weight_set *w = &arg->weight_set[i];

                        w->weights = decode_array_32_alloc(p, end, &w->size);
                        if (IS_ERR(w->weights)) {
                                ret = PTR_ERR(w->weights);
                                w->weights = NULL;
                                return ret;
                        }
                }
        }

        arg->ids = decode_array_32_alloc(p, end, &arg->ids_size);
        if (IS_ERR(arg->ids)) {
                ret = PTR_ERR(arg->ids);
                arg->ids = NULL;
                return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_choose_args(void **p, void *end, struct crush_map *c)
{
        struct crush_choose_arg_map *arg_map = NULL;
        u32 num_choose_arg_maps, num_buckets;
        int ret;

        ceph_decode_32_safe(p, end, num_choose_arg_maps, e_inval);
        while (num_choose_arg_maps--) {
                arg_map = alloc_choose_arg_map();
                if (!arg_map) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_64_safe(p, end, arg_map->choose_args_index,
                                    e_inval);
                arg_map->size = c->max_buckets;
                arg_map->args = kcalloc(arg_map->size, sizeof(*arg_map->args),
                                        GFP_NOIO);
                if (!arg_map->args) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_32_safe(p, end, num_buckets, e_inval);
                while (num_buckets--) {
                        struct crush_choose_arg *arg;
                        u32 bucket_index;

                        ceph_decode_32_safe(p, end, bucket_index, e_inval);
                        if (bucket_index >= arg_map->size)
                                goto e_inval;

                        arg = &arg_map->args[bucket_index];
                        ret = decode_choose_arg(p, end, arg);
                        if (ret)
                                goto fail;

                        if (arg->ids_size &&
                            arg->ids_size != c->buckets[bucket_index]->size)
                                goto e_inval;
                }

                insert_choose_arg_map(&c->choose_args, arg_map);
        }

        return 0;

e_inval:
        ret = -EINVAL;
fail:
        free_choose_arg_map(arg_map);
        return ret;
}

static void crush_finalize(struct crush_map *c)
{
        __s32 b;

        /* Space for the array of pointers to per-bucket workspace */
        c->working_size = sizeof(struct crush_work) +
            c->max_buckets * sizeof(struct crush_work_bucket *);

        for (b = 0; b < c->max_buckets; b++) {
                if (!c->buckets[b])
                        continue;

                switch (c->buckets[b]->alg) {
                default:
                        /*
                         * The base case, permutation variables and
                         * the pointer to the permutation array.
                         */
                        c->working_size += sizeof(struct crush_work_bucket);
                        break;
                }
                /* Every bucket has a permutation array. */
                c->working_size += c->buckets[b]->size * sizeof(__u32);
        }
}

static struct crush_map *crush_decode(void *pbyval, void *end)
{
        struct crush_map *c;
        int err;
        int i, j;
        void **p = &pbyval;
        void *start = pbyval;
        u32 magic;

        dout("crush_decode %p to %p len %d\n", *p, end, (int)(end - *p));

        c = kzalloc(sizeof(*c), GFP_NOFS);
        if (c == NULL)
                return ERR_PTR(-ENOMEM);

        c->type_names = RB_ROOT;
        c->names = RB_ROOT;
        c->choose_args = RB_ROOT;

        /* set tunables to default values */
        c->choose_local_tries = 2;
        c->choose_local_fallback_tries = 5;
        c->choose_total_tries = 19;
        c->chooseleaf_descend_once = 0;

        ceph_decode_need(p, end, 4*sizeof(u32), bad);
        magic = ceph_decode_32(p);
        if (magic != CRUSH_MAGIC) {
                pr_err("crush_decode magic %x != current %x\n",
                       (unsigned int)magic, (unsigned int)CRUSH_MAGIC);
                goto bad;
        }
        c->max_buckets = ceph_decode_32(p);
        c->max_rules = ceph_decode_32(p);
        c->max_devices = ceph_decode_32(p);

        c->buckets = kcalloc(c->max_buckets, sizeof(*c->buckets), GFP_NOFS);
        if (c->buckets == NULL)
                goto badmem;
        c->rules = kcalloc(c->max_rules, sizeof(*c->rules), GFP_NOFS);
        if (c->rules == NULL)
                goto badmem;

        /* buckets */
        for (i = 0; i < c->max_buckets; i++) {
                int size = 0;
                u32 alg;
                struct crush_bucket *b;

                ceph_decode_32_safe(p, end, alg, bad);
                if (alg == 0) {
                        c->buckets[i] = NULL;
                        continue;
                }
                dout("crush_decode bucket %d off %x %p to %p\n",
                     i, (int)(*p-start), *p, end);

                switch (alg) {
                case CRUSH_BUCKET_UNIFORM:
                        size = sizeof(struct crush_bucket_uniform);
                        break;
                case CRUSH_BUCKET_LIST:
                        size = sizeof(struct crush_bucket_list);
                        break;
                case CRUSH_BUCKET_TREE:
                        size = sizeof(struct crush_bucket_tree);
                        break;
                case CRUSH_BUCKET_STRAW:
                        size = sizeof(struct crush_bucket_straw);
                        break;
                case CRUSH_BUCKET_STRAW2:
                        size = sizeof(struct crush_bucket_straw2);
                        break;
                default:
                        goto bad;
                }
                BUG_ON(size == 0);
                b = c->buckets[i] = kzalloc(size, GFP_NOFS);
                if (b == NULL)
                        goto badmem;

                ceph_decode_need(p, end, 4*sizeof(u32), bad);
                b->id = ceph_decode_32(p);
                b->type = ceph_decode_16(p);
                b->alg = ceph_decode_8(p);
                b->hash = ceph_decode_8(p);
                b->weight = ceph_decode_32(p);
                b->size = ceph_decode_32(p);

                dout("crush_decode bucket size %d off %x %p to %p\n",
                     b->size, (int)(*p-start), *p, end);

                b->items = kcalloc(b->size, sizeof(__s32), GFP_NOFS);
                if (b->items == NULL)
                        goto badmem;

                ceph_decode_need(p, end, b->size*sizeof(u32), bad);
                for (j = 0; j < b->size; j++)
                        b->items[j] = ceph_decode_32(p);

                switch (b->alg) {
                case CRUSH_BUCKET_UNIFORM:
                        err = crush_decode_uniform_bucket(p, end,
                                  (struct crush_bucket_uniform *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_LIST:
                        err = crush_decode_list_bucket(p, end,
                               (struct crush_bucket_list *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_TREE:
                        err = crush_decode_tree_bucket(p, end,
                                (struct crush_bucket_tree *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_STRAW:
                        err = crush_decode_straw_bucket(p, end,
                                (struct crush_bucket_straw *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_STRAW2:
                        err = crush_decode_straw2_bucket(p, end,
                                (struct crush_bucket_straw2 *)b);
                        if (err < 0)
                                goto fail;
                        break;
                }
        }

        /* rules */
        dout("rule vec is %p\n", c->rules);
        for (i = 0; i < c->max_rules; i++) {
                u32 yes;
                struct crush_rule *r;

                ceph_decode_32_safe(p, end, yes, bad);
                if (!yes) {
                        dout("crush_decode NO rule %d off %x %p to %p\n",
                             i, (int)(*p-start), *p, end);
                        c->rules[i] = NULL;
                        continue;
                }

                dout("crush_decode rule %d off %x %p to %p\n",
                     i, (int)(*p-start), *p, end);

                /* len */
                ceph_decode_32_safe(p, end, yes, bad);
#if BITS_PER_LONG == 32
                if (yes > (ULONG_MAX - sizeof(*r))
                          / sizeof(struct crush_rule_step))
                        goto bad;
#endif
                r = kmalloc(struct_size(r, steps, yes), GFP_NOFS);
                if (r == NULL)
                        goto badmem;
                dout(" rule %d is at %p\n", i, r);
                c->rules[i] = r;
                r->len = yes;
                ceph_decode_copy_safe(p, end, &r->mask, 4, bad); /* 4 u8's */
                ceph_decode_need(p, end, r->len*3*sizeof(u32), bad);
                for (j = 0; j < r->len; j++) {
                        r->steps[j].op = ceph_decode_32(p);
                        r->steps[j].arg1 = ceph_decode_32(p);
                        r->steps[j].arg2 = ceph_decode_32(p);
                }
        }

        err = decode_crush_names(p, end, &c->type_names);
        if (err)
                goto fail;

        err = decode_crush_names(p, end, &c->names);
        if (err)
                goto fail;

        ceph_decode_skip_map(p, end, 32, string, bad); /* rule_name_map */

        /* tunables */
        ceph_decode_need(p, end, 3*sizeof(u32), done);
        c->choose_local_tries = ceph_decode_32(p);
        c->choose_local_fallback_tries =  ceph_decode_32(p);
        c->choose_total_tries = ceph_decode_32(p);
        dout("crush decode tunable choose_local_tries = %d\n",
             c->choose_local_tries);
        dout("crush decode tunable choose_local_fallback_tries = %d\n",
             c->choose_local_fallback_tries);
        dout("crush decode tunable choose_total_tries = %d\n",
             c->choose_total_tries);

        ceph_decode_need(p, end, sizeof(u32), done);
        c->chooseleaf_descend_once = ceph_decode_32(p);
        dout("crush decode tunable chooseleaf_descend_once = %d\n",
             c->chooseleaf_descend_once);

        ceph_decode_need(p, end, sizeof(u8), done);
        c->chooseleaf_vary_r = ceph_decode_8(p);
        dout("crush decode tunable chooseleaf_vary_r = %d\n",
             c->chooseleaf_vary_r);

        /* skip straw_calc_version, allowed_bucket_algs */
        ceph_decode_need(p, end, sizeof(u8) + sizeof(u32), done);
        *p += sizeof(u8) + sizeof(u32);

        ceph_decode_need(p, end, sizeof(u8), done);
        c->chooseleaf_stable = ceph_decode_8(p);
        dout("crush decode tunable chooseleaf_stable = %d\n",
             c->chooseleaf_stable);

        if (*p != end) {
                /* class_map */
                ceph_decode_skip_map(p, end, 32, 32, bad);
                /* class_name */
                ceph_decode_skip_map(p, end, 32, string, bad);
                /* class_bucket */
                ceph_decode_skip_map_of_map(p, end, 32, 32, 32, bad);
        }

        if (*p != end) {
                err = decode_choose_args(p, end, c);
                if (err)
                        goto fail;
        }

done:
        crush_finalize(c);
        dout("crush_decode success\n");
        return c;

badmem:
        err = -ENOMEM;
fail:
        dout("crush_decode fail %d\n", err);
        crush_destroy(c);
        return ERR_PTR(err);

bad:
        err = -EINVAL;
        goto fail;
}

int ceph_pg_compare(const struct ceph_pg *lhs, const struct ceph_pg *rhs)
{
        if (lhs->pool < rhs->pool)
                return -1;
        if (lhs->pool > rhs->pool)
                return 1;
        if (lhs->seed < rhs->seed)
                return -1;
        if (lhs->seed > rhs->seed)
                return 1;

        return 0;
}

int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs)
{
        int ret;

        ret = ceph_pg_compare(&lhs->pgid, &rhs->pgid);
        if (ret)
                return ret;

        if (lhs->shard < rhs->shard)
                return -1;
        if (lhs->shard > rhs->shard)
                return 1;

        return 0;
}

static struct ceph_pg_mapping *alloc_pg_mapping(size_t payload_len)
{
        struct ceph_pg_mapping *pg;

        pg = kmalloc(sizeof(*pg) + payload_len, GFP_NOIO);
        if (!pg)
                return NULL;

        RB_CLEAR_NODE(&pg->node);
        return pg;
}

static void free_pg_mapping(struct ceph_pg_mapping *pg)
{
        WARN_ON(!RB_EMPTY_NODE(&pg->node));

        kfree(pg);
}

/*
 * rbtree of pg_mapping for handling pg_temp (explicit mapping of pgid
 * to a set of osds) and primary_temp (explicit primary setting)
 */
DEFINE_RB_FUNCS2(pg_mapping, struct ceph_pg_mapping, pgid, ceph_pg_compare,
                 RB_BYPTR, const struct ceph_pg *, node)

/*
 * rbtree of pg pool info
 */
DEFINE_RB_FUNCS(pg_pool, struct ceph_pg_pool_info, id, node)

struct ceph_pg_pool_info *ceph_pg_pool_by_id(struct ceph_osdmap *map, u64 id)
{
        return lookup_pg_pool(&map->pg_pools, id);
}

const char *ceph_pg_pool_name_by_id(struct ceph_osdmap *map, u64 id)
{
        struct ceph_pg_pool_info *pi;

        if (id == CEPH_NOPOOL)
                return NULL;

        if (WARN_ON_ONCE(id > (u64) INT_MAX))
                return NULL;

        pi = lookup_pg_pool(&map->pg_pools, id);
        return pi ? pi->name : NULL;
}
EXPORT_SYMBOL(ceph_pg_pool_name_by_id);

int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char *name)
{
        struct rb_node *rbp;

        for (rbp = rb_first(&map->pg_pools); rbp; rbp = rb_next(rbp)) {
                struct ceph_pg_pool_info *pi =
                        rb_entry(rbp, struct ceph_pg_pool_info, node);
                if (pi->name && strcmp(pi->name, name) == 0)
                        return pi->id;
        }
        return -ENOENT;
}
EXPORT_SYMBOL(ceph_pg_poolid_by_name);

u64 ceph_pg_pool_flags(struct ceph_osdmap *map, u64 id)
{
        struct ceph_pg_pool_info *pi;

        pi = lookup_pg_pool(&map->pg_pools, id);
        return pi ? pi->flags : 0;
}
EXPORT_SYMBOL(ceph_pg_pool_flags);

static void __remove_pg_pool(struct rb_root *root, struct ceph_pg_pool_info *pi)
{
        erase_pg_pool(root, pi);
        kfree(pi->name);
        kfree(pi);
}

static int decode_pool(void **p, void *end, struct ceph_pg_pool_info *pi)
{
        u8 ev, cv;
        unsigned len, num;
        void *pool_end;

        ceph_decode_need(p, end, 2 + 4, bad);
        ev = ceph_decode_8(p);  /* encoding version */
        cv = ceph_decode_8(p); /* compat version */
        if (ev < 5) {
                pr_warn("got v %d < 5 cv %d of ceph_pg_pool\n", ev, cv);
                return -EINVAL;
        }
        if (cv > 9) {
                pr_warn("got v %d cv %d > 9 of ceph_pg_pool\n", ev, cv);
                return -EINVAL;
        }
        len = ceph_decode_32(p);
        ceph_decode_need(p, end, len, bad);
        pool_end = *p + len;

        pi->type = ceph_decode_8(p);
        pi->size = ceph_decode_8(p);
        pi->crush_ruleset = ceph_decode_8(p);
        pi->object_hash = ceph_decode_8(p);

        pi->pg_num = ceph_decode_32(p);
        pi->pgp_num = ceph_decode_32(p);

        *p += 4 + 4;  /* skip lpg* */
        *p += 4;      /* skip last_change */
        *p += 8 + 4;  /* skip snap_seq, snap_epoch */

        /* skip snaps */
        num = ceph_decode_32(p);
        while (num--) {
                *p += 8;  /* snapid key */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;
        }

        /* skip removed_snaps */
        num = ceph_decode_32(p);
        *p += num * (8 + 8);

        *p += 8;  /* skip auid */
        pi->flags = ceph_decode_64(p);
        *p += 4;  /* skip crash_replay_interval */

        if (ev >= 7)
                pi->min_size = ceph_decode_8(p);
        else
                pi->min_size = pi->size - pi->size / 2;

        if (ev >= 8)
                *p += 8 + 8;  /* skip quota_max_* */

        if (ev >= 9) {
                /* skip tiers */
                num = ceph_decode_32(p);
                *p += num * 8;

                *p += 8;  /* skip tier_of */
                *p += 1;  /* skip cache_mode */

                pi->read_tier = ceph_decode_64(p);
                pi->write_tier = ceph_decode_64(p);
        } else {
                pi->read_tier = -1;
                pi->write_tier = -1;
        }

        if (ev >= 10) {
                /* skip properties */
                num = ceph_decode_32(p);
                while (num--) {
                        len = ceph_decode_32(p);
                        *p += len; /* key */
                        len = ceph_decode_32(p);
                        *p += len; /* val */
                }
        }

        if (ev >= 11) {
                /* skip hit_set_params */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;

                *p += 4; /* skip hit_set_period */
                *p += 4; /* skip hit_set_count */
        }

        if (ev >= 12)
                *p += 4; /* skip stripe_width */

        if (ev >= 13) {
                *p += 8; /* skip target_max_bytes */
                *p += 8; /* skip target_max_objects */
                *p += 4; /* skip cache_target_dirty_ratio_micro */
                *p += 4; /* skip cache_target_full_ratio_micro */
                *p += 4; /* skip cache_min_flush_age */
                *p += 4; /* skip cache_min_evict_age */
        }

        if (ev >=  14) {
                /* skip erasure_code_profile */
                len = ceph_decode_32(p);
                *p += len;
        }

        /*
         * last_force_op_resend_preluminous, will be overridden if the
         * map was encoded with RESEND_ON_SPLIT
         */
        if (ev >= 15)
                pi->last_force_request_resend = ceph_decode_32(p);
        else
                pi->last_force_request_resend = 0;

        if (ev >= 16)
                *p += 4; /* skip min_read_recency_for_promote */

        if (ev >= 17)
                *p += 8; /* skip expected_num_objects */

        if (ev >= 19)
                *p += 4; /* skip cache_target_dirty_high_ratio_micro */

        if (ev >= 20)
                *p += 4; /* skip min_write_recency_for_promote */

        if (ev >= 21)
                *p += 1; /* skip use_gmt_hitset */

        if (ev >= 22)
                *p += 1; /* skip fast_read */

        if (ev >= 23) {
                *p += 4; /* skip hit_set_grade_decay_rate */
                *p += 4; /* skip hit_set_search_last_n */
        }

        if (ev >= 24) {
                /* skip opts */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;
        }

        if (ev >= 25)
                pi->last_force_request_resend = ceph_decode_32(p);

        /* ignore the rest */

        *p = pool_end;
        calc_pg_masks(pi);
        return 0;

bad:
        return -EINVAL;
}

static int decode_pool_names(void **p, void *end, struct ceph_osdmap *map)
{
        struct ceph_pg_pool_info *pi;
        u32 num, len;
        u64 pool;

        ceph_decode_32_safe(p, end, num, bad);
        dout(" %d pool names\n", num);
        while (num--) {
                ceph_decode_64_safe(p, end, pool, bad);
                ceph_decode_32_safe(p, end, len, bad);
                dout("  pool %llu len %d\n", pool, len);
                ceph_decode_need(p, end, len, bad);
                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (pi) {
                        char *name = kstrndup(*p, len, GFP_NOFS);

                        if (!name)
                                return -ENOMEM;
                        kfree(pi->name);
                        pi->name = name;
                        dout("  name is %s\n", pi->name);
                }
                *p += len;
        }
        return 0;

bad:
        return -EINVAL;
}

/*
 * CRUSH workspaces
 *
 * workspace_manager framework borrowed from fs/btrfs/compression.c.
 * Two simplifications: there is only one type of workspace and there
 * is always at least one workspace.
 */
static struct crush_work *alloc_workspace(const struct crush_map *c)
{
        struct crush_work *work;
        size_t work_size;

        WARN_ON(!c->working_size);
        work_size = crush_work_size(c, CEPH_PG_MAX_SIZE);
        dout("%s work_size %zu bytes\n", __func__, work_size);

        work = kvmalloc(work_size, GFP_NOIO);
        if (!work)
                return NULL;

        INIT_LIST_HEAD(&work->item);
        crush_init_workspace(c, work);
        return work;
}

static void free_workspace(struct crush_work *work)
{
        WARN_ON(!list_empty(&work->item));
        kvfree(work);
}

static void init_workspace_manager(struct workspace_manager *wsm)
{
        INIT_LIST_HEAD(&wsm->idle_ws);
        spin_lock_init(&wsm->ws_lock);
        atomic_set(&wsm->total_ws, 0);
        wsm->free_ws = 0;
        init_waitqueue_head(&wsm->ws_wait);
}

static void add_initial_workspace(struct workspace_manager *wsm,
                                  struct crush_work *work)
{
        WARN_ON(!list_empty(&wsm->idle_ws));

        list_add(&work->item, &wsm->idle_ws);
        atomic_set(&wsm->total_ws, 1);
        wsm->free_ws = 1;
}

static void cleanup_workspace_manager(struct workspace_manager *wsm)
{
        struct crush_work *work;

        while (!list_empty(&wsm->idle_ws)) {
                work = list_first_entry(&wsm->idle_ws, struct crush_work,
                                        item);
                list_del_init(&work->item);
                free_workspace(work);
        }
        atomic_set(&wsm->total_ws, 0);
        wsm->free_ws = 0;
}

/*
 * Finds an available workspace or allocates a new one.  If it's not
 * possible to allocate a new one, waits until there is one.
 */
static struct crush_work *get_workspace(struct workspace_manager *wsm,
                                        const struct crush_map *c)
{
        struct crush_work *work;
        int cpus = num_online_cpus();

again:
        spin_lock(&wsm->ws_lock);
        if (!list_empty(&wsm->idle_ws)) {
                work = list_first_entry(&wsm->idle_ws, struct crush_work,
                                        item);
                list_del_init(&work->item);
                wsm->free_ws--;
                spin_unlock(&wsm->ws_lock);
                return work;

        }
        if (atomic_read(&wsm->total_ws) > cpus) {
                DEFINE_WAIT(wait);

                spin_unlock(&wsm->ws_lock);
                prepare_to_wait(&wsm->ws_wait, &wait, TASK_UNINTERRUPTIBLE);
                if (atomic_read(&wsm->total_ws) > cpus && !wsm->free_ws)
                        schedule();
                finish_wait(&wsm->ws_wait, &wait);
                goto again;
        }
        atomic_inc(&wsm->total_ws);
        spin_unlock(&wsm->ws_lock);

        work = alloc_workspace(c);
        if (!work) {
                atomic_dec(&wsm->total_ws);
                wake_up(&wsm->ws_wait);

                /*
                 * Do not return the error but go back to waiting.  We
                 * have the initial workspace and the CRUSH computation
                 * time is bounded so we will get it eventually.
                 */
                WARN_ON(atomic_read(&wsm->total_ws) < 1);
                goto again;
        }
        return work;
}

/*
 * Puts a workspace back on the list or frees it if we have enough
 * idle ones sitting around.
 */
static void put_workspace(struct workspace_manager *wsm,
                          struct crush_work *work)
{
        spin_lock(&wsm->ws_lock);
        if (wsm->free_ws <= num_online_cpus()) {
                list_add(&work->item, &wsm->idle_ws);
                wsm->free_ws++;
                spin_unlock(&wsm->ws_lock);
                goto wake;
        }
        spin_unlock(&wsm->ws_lock);

        free_workspace(work);
        atomic_dec(&wsm->total_ws);
wake:
        if (wq_has_sleeper(&wsm->ws_wait))
                wake_up(&wsm->ws_wait);
}

/*
 * osd map
 */
struct ceph_osdmap *ceph_osdmap_alloc(void)
{
        struct ceph_osdmap *map;

        map = kzalloc(sizeof(*map), GFP_NOIO);
        if (!map)
                return NULL;

        map->pg_pools = RB_ROOT;
        map->pool_max = -1;
        map->pg_temp = RB_ROOT;
        map->primary_temp = RB_ROOT;
        map->pg_upmap = RB_ROOT;
        map->pg_upmap_items = RB_ROOT;

        init_workspace_manager(&map->crush_wsm);

        return map;
}

void ceph_osdmap_destroy(struct ceph_osdmap *map)
{
        dout("osdmap_destroy %p\n", map);

        if (map->crush)
                crush_destroy(map->crush);
        cleanup_workspace_manager(&map->crush_wsm);

        while (!RB_EMPTY_ROOT(&map->pg_temp)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_temp),
                                 struct ceph_pg_mapping, node);
                erase_pg_mapping(&map->pg_temp, pg);
                free_pg_mapping(pg);
        }
        while (!RB_EMPTY_ROOT(&map->primary_temp)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->primary_temp),
                                 struct ceph_pg_mapping, node);
                erase_pg_mapping(&map->primary_temp, pg);
                free_pg_mapping(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_upmap)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_upmap),
                                 struct ceph_pg_mapping, node);
                rb_erase(&pg->node, &map->pg_upmap);
                kfree(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_upmap_items)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_upmap_items),
                                 struct ceph_pg_mapping, node);
                rb_erase(&pg->node, &map->pg_upmap_items);
                kfree(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_pools)) {
                struct ceph_pg_pool_info *pi =
                        rb_entry(rb_first(&map->pg_pools),
                                 struct ceph_pg_pool_info, node);
                __remove_pg_pool(&map->pg_pools, pi);
        }
        kvfree(map->osd_state);
        kvfree(map->osd_weight);
        kvfree(map->osd_addr);
        kvfree(map->osd_primary_affinity);
        kfree(map);
}

/*
 * Adjust max_osd value, (re)allocate arrays.
 *
 * The new elements are properly initialized.
 */
static int osdmap_set_max_osd(struct ceph_osdmap *map, u32 max)
{
        u32 *state;
        u32 *weight;
        struct ceph_entity_addr *addr;
        u32 to_copy;
        int i;

        dout("%s old %u new %u\n", __func__, map->max_osd, max);
        if (max == map->max_osd)
                return 0;

        state = kvmalloc(array_size(max, sizeof(*state)), GFP_NOFS);
        weight = kvmalloc(array_size(max, sizeof(*weight)), GFP_NOFS);
        addr = kvmalloc(array_size(max, sizeof(*addr)), GFP_NOFS);
        if (!state || !weight || !addr) {
                kvfree(state);
                kvfree(weight);
                kvfree(addr);
                return -ENOMEM;
        }

        to_copy = min(map->max_osd, max);
        if (map->osd_state) {
                memcpy(state, map->osd_state, to_copy * sizeof(*state));
                memcpy(weight, map->osd_weight, to_copy * sizeof(*weight));
                memcpy(addr, map->osd_addr, to_copy * sizeof(*addr));
                kvfree(map->osd_state);
                kvfree(map->osd_weight);
                kvfree(map->osd_addr);
        }

        map->osd_state = state;
        map->osd_weight = weight;
        map->osd_addr = addr;
        for (i = map->max_osd; i < max; i++) {
                map->osd_state[i] = 0;
                map->osd_weight[i] = CEPH_OSD_OUT;
                memset(map->osd_addr + i, 0, sizeof(*map->osd_addr));
        }

        if (map->osd_primary_affinity) {
                u32 *affinity;

                affinity = kvmalloc(array_size(max, sizeof(*affinity)),
                                         GFP_NOFS);
                if (!affinity)
                        return -ENOMEM;

                memcpy(affinity, map->osd_primary_affinity,
                       to_copy * sizeof(*affinity));
                kvfree(map->osd_primary_affinity);

                map->osd_primary_affinity = affinity;
                for (i = map->max_osd; i < max; i++)
                        map->osd_primary_affinity[i] =
                            CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
        }

        map->max_osd = max;

        return 0;
}

static int osdmap_set_crush(struct ceph_osdmap *map, struct crush_map *crush)
{
        struct crush_work *work;

        if (IS_ERR(crush))
                return PTR_ERR(crush);

        work = alloc_workspace(crush);
        if (!work) {
                crush_destroy(crush);
                return -ENOMEM;
        }

        if (map->crush)
                crush_destroy(map->crush);
        cleanup_workspace_manager(&map->crush_wsm);
        map->crush = crush;
        add_initial_workspace(&map->crush_wsm, work);
        return 0;
}

#define OSDMAP_WRAPPER_COMPAT_VER        7
#define OSDMAP_CLIENT_DATA_COMPAT_VER        1

/*
 * Return 0 or error.  On success, *v is set to 0 for old (v6) osdmaps,
 * to struct_v of the client_data section for new (v7 and above)
 * osdmaps.
 */
static int get_osdmap_client_data_v(void **p, void *end,
                                    const char *prefix, u8 *v)
{
        u8 struct_v;

        ceph_decode_8_safe(p, end, struct_v, e_inval);
        if (struct_v >= 7) {
                u8 struct_compat;

                ceph_decode_8_safe(p, end, struct_compat, e_inval);
                if (struct_compat > OSDMAP_WRAPPER_COMPAT_VER) {
                        pr_warn("got v %d cv %d > %d of %s ceph_osdmap\n",
                                struct_v, struct_compat,
                                OSDMAP_WRAPPER_COMPAT_VER, prefix);
                        return -EINVAL;
                }
                *p += 4; /* ignore wrapper struct_len */

                ceph_decode_8_safe(p, end, struct_v, e_inval);
                ceph_decode_8_safe(p, end, struct_compat, e_inval);
                if (struct_compat > OSDMAP_CLIENT_DATA_COMPAT_VER) {
                        pr_warn("got v %d cv %d > %d of %s ceph_osdmap client data\n",
                                struct_v, struct_compat,
                                OSDMAP_CLIENT_DATA_COMPAT_VER, prefix);
                        return -EINVAL;
                }
                *p += 4; /* ignore client data struct_len */
        } else {
                u16 version;

                *p -= 1;
                ceph_decode_16_safe(p, end, version, e_inval);
                if (version < 6) {
                        pr_warn("got v %d < 6 of %s ceph_osdmap\n",
                                version, prefix);
                        return -EINVAL;
                }

                /* old osdmap encoding */
                struct_v = 0;
        }

        *v = struct_v;
        return 0;

e_inval:
        return -EINVAL;
}

static int __decode_pools(void **p, void *end, struct ceph_osdmap *map,
                          bool incremental)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct ceph_pg_pool_info *pi;
                u64 pool;
                int ret;

                ceph_decode_64_safe(p, end, pool, e_inval);

                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (!incremental || !pi) {
                        pi = kzalloc(sizeof(*pi), GFP_NOFS);
                        if (!pi)
                                return -ENOMEM;

                        RB_CLEAR_NODE(&pi->node);
                        pi->id = pool;

                        if (!__insert_pg_pool(&map->pg_pools, pi)) {
                                kfree(pi);
                                return -EEXIST;
                        }
                }

                ret = decode_pool(p, end, pi);
                if (ret)
                        return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_pools(void **p, void *end, struct ceph_osdmap *map)
{
        return __decode_pools(p, end, map, false);
}

static int decode_new_pools(void **p, void *end, struct ceph_osdmap *map)
{
        return __decode_pools(p, end, map, true);
}

typedef struct ceph_pg_mapping *(*decode_mapping_fn_t)(void **, void *, bool);

static int decode_pg_mapping(void **p, void *end, struct rb_root *mapping_root,
                             decode_mapping_fn_t fn, bool incremental)
{
        u32 n;

        WARN_ON(!incremental && !fn);

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct ceph_pg_mapping *pg;
                struct ceph_pg pgid;
                int ret;

                ret = ceph_decode_pgid(p, end, &pgid);
                if (ret)
                        return ret;

                pg = lookup_pg_mapping(mapping_root, &pgid);
                if (pg) {
                        WARN_ON(!incremental);
                        erase_pg_mapping(mapping_root, pg);
                        free_pg_mapping(pg);
                }

                if (fn) {
                        pg = fn(p, end, incremental);
                        if (IS_ERR(pg))
                                return PTR_ERR(pg);

                        if (pg) {
                                pg->pgid = pgid; /* struct */
                                insert_pg_mapping(mapping_root, pg);
                        }
                }
        }

        return 0;

e_inval:
        return -EINVAL;
}

static struct ceph_pg_mapping *__decode_pg_temp(void **p, void *end,
                                                bool incremental)
{
        struct ceph_pg_mapping *pg;
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len == 0 && incremental)
                return NULL;        /* new_pg_temp: [] to remove */
        if (len > (SIZE_MAX - sizeof(*pg)) / sizeof(u32))
                return ERR_PTR(-EINVAL);

        ceph_decode_need(p, end, len * sizeof(u32), e_inval);
        pg = alloc_pg_mapping(len * sizeof(u32));
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->pg_temp.len = len;
        for (i = 0; i < len; i++)
                pg->pg_temp.osds[i] = ceph_decode_32(p);

        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_pg_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_temp, __decode_pg_temp,
                                 false);
}

static int decode_new_pg_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_temp, __decode_pg_temp,
                                 true);
}

static struct ceph_pg_mapping *__decode_primary_temp(void **p, void *end,
                                                     bool incremental)
{
        struct ceph_pg_mapping *pg;
        u32 osd;

        ceph_decode_32_safe(p, end, osd, e_inval);
        if (osd == (u32)-1 && incremental)
                return NULL;        /* new_primary_temp: -1 to remove */

        pg = alloc_pg_mapping(0);
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->primary_temp.osd = osd;
        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_primary_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->primary_temp,
                                 __decode_primary_temp, false);
}

static int decode_new_primary_temp(void **p, void *end,
                                   struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->primary_temp,
                                 __decode_primary_temp, true);
}

u32 ceph_get_primary_affinity(struct ceph_osdmap *map, int osd)
{
        BUG_ON(osd >= map->max_osd);

        if (!map->osd_primary_affinity)
                return CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;

        return map->osd_primary_affinity[osd];
}

static int set_primary_affinity(struct ceph_osdmap *map, int osd, u32 aff)
{
        BUG_ON(osd >= map->max_osd);

        if (!map->osd_primary_affinity) {
                int i;

                map->osd_primary_affinity = kvmalloc(
                    array_size(map->max_osd, sizeof(*map->osd_primary_affinity)),
                    GFP_NOFS);
                if (!map->osd_primary_affinity)
                        return -ENOMEM;

                for (i = 0; i < map->max_osd; i++)
                        map->osd_primary_affinity[i] =
                            CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
        }

        map->osd_primary_affinity[osd] = aff;

        return 0;
}

static int decode_primary_affinity(void **p, void *end,
                                   struct ceph_osdmap *map)
{
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len == 0) {
                kvfree(map->osd_primary_affinity);
                map->osd_primary_affinity = NULL;
                return 0;
        }
        if (len != map->max_osd)
                goto e_inval;

        ceph_decode_need(p, end, map->max_osd*sizeof(u32), e_inval);

        for (i = 0; i < map->max_osd; i++) {
                int ret;

                ret = set_primary_affinity(map, i, ceph_decode_32(p));
                if (ret)
                        return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_new_primary_affinity(void **p, void *end,
                                       struct ceph_osdmap *map)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                u32 osd, aff;
                int ret;

                ceph_decode_32_safe(p, end, osd, e_inval);
                ceph_decode_32_safe(p, end, aff, e_inval);

                ret = set_primary_affinity(map, osd, aff);
                if (ret)
                        return ret;

                osdmap_info(map, "osd%d primary-affinity 0x%x\n", osd, aff);
        }

        return 0;

e_inval:
        return -EINVAL;
}

static struct ceph_pg_mapping *__decode_pg_upmap(void **p, void *end,
                                                 bool __unused)
{
        return __decode_pg_temp(p, end, false);
}

static int decode_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, __decode_pg_upmap,
                                 false);
}

static int decode_new_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, __decode_pg_upmap,
                                 true);
}

static int decode_old_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, NULL, true);
}

static struct ceph_pg_mapping *__decode_pg_upmap_items(void **p, void *end,
                                                       bool __unused)
{
        struct ceph_pg_mapping *pg;
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > (SIZE_MAX - sizeof(*pg)) / (2 * sizeof(u32)))
                return ERR_PTR(-EINVAL);

        ceph_decode_need(p, end, 2 * len * sizeof(u32), e_inval);
        pg = alloc_pg_mapping(2 * len * sizeof(u32));
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->pg_upmap_items.len = len;
        for (i = 0; i < len; i++) {
                pg->pg_upmap_items.from_to[i][0] = ceph_decode_32(p);
                pg->pg_upmap_items.from_to[i][1] = ceph_decode_32(p);
        }

        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_pg_upmap_items(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items,
                                 __decode_pg_upmap_items, false);
}

static int decode_new_pg_upmap_items(void **p, void *end,
                                     struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items,
                                 __decode_pg_upmap_items, true);
}

static int decode_old_pg_upmap_items(void **p, void *end,
                                     struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items, NULL, true);
}

/*
 * decode a full map.
 */
static int osdmap_decode(void **p, void *end, bool msgr2,
                         struct ceph_osdmap *map)
{
        u8 struct_v;
        u32 epoch = 0;
        void *start = *p;
        u32 max;
        u32 len, i;
        int err;

        dout("%s %p to %p len %d\n", __func__, *p, end, (int)(end - *p));

        err = get_osdmap_client_data_v(p, end, "full", &struct_v);
        if (err)
                goto bad;

        /* fsid, epoch, created, modified */
        ceph_decode_need(p, end, sizeof(map->fsid) + sizeof(u32) +
                         sizeof(map->created) + sizeof(map->modified), e_inval);
        ceph_decode_copy(p, &map->fsid, sizeof(map->fsid));
        epoch = map->epoch = ceph_decode_32(p);
        ceph_decode_copy(p, &map->created, sizeof(map->created));
        ceph_decode_copy(p, &map->modified, sizeof(map->modified));

        /* pools */
        err = decode_pools(p, end, map);
        if (err)
                goto bad;

        /* pool_name */
        err = decode_pool_names(p, end, map);
        if (err)
                goto bad;

        ceph_decode_32_safe(p, end, map->pool_max, e_inval);

        ceph_decode_32_safe(p, end, map->flags, e_inval);

        /* max_osd */
        ceph_decode_32_safe(p, end, max, e_inval);

        /* (re)alloc osd arrays */
        err = osdmap_set_max_osd(map, max);
        if (err)
                goto bad;

        /* osd_state, osd_weight, osd_addrs->client_addr */
        ceph_decode_need(p, end, 3*sizeof(u32) +
                         map->max_osd*(struct_v >= 5 ? sizeof(u32) :
                                                       sizeof(u8)) +
                                       sizeof(*map->osd_weight), e_inval);
        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        if (struct_v >= 5) {
                for (i = 0; i < map->max_osd; i++)
                        map->osd_state[i] = ceph_decode_32(p);
        } else {
                for (i = 0; i < map->max_osd; i++)
                        map->osd_state[i] = ceph_decode_8(p);
        }

        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        for (i = 0; i < map->max_osd; i++)
                map->osd_weight[i] = ceph_decode_32(p);

        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        for (i = 0; i < map->max_osd; i++) {
                struct ceph_entity_addr *addr = &map->osd_addr[i];

                if (struct_v >= 8)
                        err = ceph_decode_entity_addrvec(p, end, msgr2, addr);
                else
                        err = ceph_decode_entity_addr(p, end, addr);
                if (err)
                        goto bad;

                dout("%s osd%d addr %s\n", __func__, i, ceph_pr_addr(addr));
        }

        /* pg_temp */
        err = decode_pg_temp(p, end, map);
        if (err)
                goto bad;

        /* primary_temp */
        if (struct_v >= 1) {
                err = decode_primary_temp(p, end, map);
                if (err)
                        goto bad;
        }

        /* primary_affinity */
        if (struct_v >= 2) {
                err = decode_primary_affinity(p, end, map);
                if (err)
                        goto bad;
        } else {
                WARN_ON(map->osd_primary_affinity);
        }

        /* crush */
        ceph_decode_32_safe(p, end, len, e_inval);
        err = osdmap_set_crush(map, crush_decode(*p, min(*p + len, end)));
        if (err)
                goto bad;

        *p += len;
        if (struct_v >= 3) {
                /* erasure_code_profiles */
                ceph_decode_skip_map_of_map(p, end, string, string, string,
                                            e_inval);
        }

        if (struct_v >= 4) {
                err = decode_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;
        } else {
                WARN_ON(!RB_EMPTY_ROOT(&map->pg_upmap));
                WARN_ON(!RB_EMPTY_ROOT(&map->pg_upmap_items));
        }

        /* ignore the rest */
        *p = end;

        dout("full osdmap epoch %d max_osd %d\n", map->epoch, map->max_osd);
        return 0;

e_inval:
        err = -EINVAL;
bad:
        pr_err("corrupt full osdmap (%d) epoch %d off %d (%p of %p-%p)\n",
               err, epoch, (int)(*p - start), *p, start, end);
        print_hex_dump(KERN_DEBUG, "osdmap: ",
                       DUMP_PREFIX_OFFSET, 16, 1,
                       start, end - start, true);
        return err;
}

/*
 * Allocate and decode a full map.
 */
struct ceph_osdmap *ceph_osdmap_decode(void **p, void *end, bool msgr2)
{
        struct ceph_osdmap *map;
        int ret;

        map = ceph_osdmap_alloc();
        if (!map)
                return ERR_PTR(-ENOMEM);

        ret = osdmap_decode(p, end, msgr2, map);
        if (ret) {
                ceph_osdmap_destroy(map);
                return ERR_PTR(ret);
        }

        return map;
}

/*
 * Encoding order is (new_up_client, new_state, new_weight).  Need to
 * apply in the (new_weight, new_state, new_up_client) order, because
 * an incremental map may look like e.g.
 *
 *     new_up_client: { osd=6, addr=... } # set osd_state and addr
 *     new_state: { osd=6, xorstate=EXISTS } # clear osd_state
 */
static int decode_new_up_state_weight(void **p, void *end, u8 struct_v,
                                      bool msgr2, struct ceph_osdmap *map)
{
        void *new_up_client;
        void *new_state;
        void *new_weight_end;
        u32 len;
        int ret;
        int i;

        new_up_client = *p;
        ceph_decode_32_safe(p, end, len, e_inval);
        for (i = 0; i < len; ++i) {
                struct ceph_entity_addr addr;

                ceph_decode_skip_32(p, end, e_inval);
                if (struct_v >= 7)
                        ret = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
                else
                        ret = ceph_decode_entity_addr(p, end, &addr);
                if (ret)
                        return ret;
        }

        new_state = *p;
        ceph_decode_32_safe(p, end, len, e_inval);
        len *= sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8));
        ceph_decode_need(p, end, len, e_inval);
        *p += len;

        /* new_weight */
        ceph_decode_32_safe(p, end, len, e_inval);
        while (len--) {
                s32 osd;
                u32 w;

                ceph_decode_need(p, end, 2*sizeof(u32), e_inval);
                osd = ceph_decode_32(p);
                w = ceph_decode_32(p);
                BUG_ON(osd >= map->max_osd);
                osdmap_info(map, "osd%d weight 0x%x %s\n", osd, w,
                            w == CEPH_OSD_IN ? "(in)" :
                            (w == CEPH_OSD_OUT ? "(out)" : ""));
                map->osd_weight[osd] = w;

                /*
                 * If we are marking in, set the EXISTS, and clear the
                 * AUTOOUT and NEW bits.
                 */
                if (w) {
                        map->osd_state[osd] |= CEPH_OSD_EXISTS;
                        map->osd_state[osd] &= ~(CEPH_OSD_AUTOOUT |
                                                 CEPH_OSD_NEW);
                }
        }
        new_weight_end = *p;

        /* new_state (up/down) */
        *p = new_state;
        len = ceph_decode_32(p);
        while (len--) {
                s32 osd;
                u32 xorstate;

                osd = ceph_decode_32(p);
                if (struct_v >= 5)
                        xorstate = ceph_decode_32(p);
                else
                        xorstate = ceph_decode_8(p);
                if (xorstate == 0)
                        xorstate = CEPH_OSD_UP;
                BUG_ON(osd >= map->max_osd);
                if ((map->osd_state[osd] & CEPH_OSD_UP) &&
                    (xorstate & CEPH_OSD_UP))
                        osdmap_info(map, "osd%d down\n", osd);
                if ((map->osd_state[osd] & CEPH_OSD_EXISTS) &&
                    (xorstate & CEPH_OSD_EXISTS)) {
                        osdmap_info(map, "osd%d does not exist\n", osd);
                        ret = set_primary_affinity(map, osd,
                                                   CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
                        if (ret)
                                return ret;
                        memset(map->osd_addr + osd, 0, sizeof(*map->osd_addr));
                        map->osd_state[osd] = 0;
                } else {
                        map->osd_state[osd] ^= xorstate;
                }
        }

        /* new_up_client */
        *p = new_up_client;
        len = ceph_decode_32(p);
        while (len--) {
                s32 osd;
                struct ceph_entity_addr addr;

                osd = ceph_decode_32(p);
                BUG_ON(osd >= map->max_osd);
                if (struct_v >= 7)
                        ret = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
                else
                        ret = ceph_decode_entity_addr(p, end, &addr);
                if (ret)
                        return ret;

                dout("%s osd%d addr %s\n", __func__, osd, ceph_pr_addr(&addr));

                osdmap_info(map, "osd%d up\n", osd);
                map->osd_state[osd] |= CEPH_OSD_EXISTS | CEPH_OSD_UP;
                map->osd_addr[osd] = addr;
        }

        *p = new_weight_end;
        return 0;

e_inval:
        return -EINVAL;
}

/*
 * decode and apply an incremental map update.
 */
struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, bool msgr2,
                                             struct ceph_osdmap *map)
{
        struct ceph_fsid fsid;
        u32 epoch = 0;
        struct ceph_timespec modified;
        s32 len;
        u64 pool;
        __s64 new_pool_max;
        __s32 new_flags, max;
        void *start = *p;
        int err;
        u8 struct_v;

        dout("%s %p to %p len %d\n", __func__, *p, end, (int)(end - *p));

        err = get_osdmap_client_data_v(p, end, "inc", &struct_v);
        if (err)
                goto bad;

        /* fsid, epoch, modified, new_pool_max, new_flags */
        ceph_decode_need(p, end, sizeof(fsid) + sizeof(u32) + sizeof(modified) +
                         sizeof(u64) + sizeof(u32), e_inval);
        ceph_decode_copy(p, &fsid, sizeof(fsid));
        epoch = ceph_decode_32(p);
        BUG_ON(epoch != map->epoch+1);
        ceph_decode_copy(p, &modified, sizeof(modified));
        new_pool_max = ceph_decode_64(p);
        new_flags = ceph_decode_32(p);

        /* full map? */
        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > 0) {
                dout("apply_incremental full map len %d, %p to %p\n",
                     len, *p, end);
                return ceph_osdmap_decode(p, min(*p+len, end), msgr2);
        }

        /* new crush? */
        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > 0) {
                err = osdmap_set_crush(map,
                                       crush_decode(*p, min(*p + len, end)));
                if (err)
                        goto bad;
                *p += len;
        }

        /* new flags? */
        if (new_flags >= 0)
                map->flags = new_flags;
        if (new_pool_max >= 0)
                map->pool_max = new_pool_max;

        /* new max? */
        ceph_decode_32_safe(p, end, max, e_inval);
        if (max >= 0) {
                err = osdmap_set_max_osd(map, max);
                if (err)
                        goto bad;
        }

        map->epoch++;
        map->modified = modified;

        /* new_pools */
        err = decode_new_pools(p, end, map);
        if (err)
                goto bad;

        /* new_pool_names */
        err = decode_pool_names(p, end, map);
        if (err)
                goto bad;

        /* old_pool */
        ceph_decode_32_safe(p, end, len, e_inval);
        while (len--) {
                struct ceph_pg_pool_info *pi;

                ceph_decode_64_safe(p, end, pool, e_inval);
                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (pi)
                        __remove_pg_pool(&map->pg_pools, pi);
        }

        /* new_up_client, new_state, new_weight */
        err = decode_new_up_state_weight(p, end, struct_v, msgr2, map);
        if (err)
                goto bad;

        /* new_pg_temp */
        err = decode_new_pg_temp(p, end, map);
        if (err)
                goto bad;

        /* new_primary_temp */
        if (struct_v >= 1) {
                err = decode_new_primary_temp(p, end, map);
                if (err)
                        goto bad;
        }

        /* new_primary_affinity */
        if (struct_v >= 2) {
                err = decode_new_primary_affinity(p, end, map);
                if (err)
                        goto bad;
        }

        if (struct_v >= 3) {
                /* new_erasure_code_profiles */
                ceph_decode_skip_map_of_map(p, end, string, string, string,
                                            e_inval);
                /* old_erasure_code_profiles */
                ceph_decode_skip_set(p, end, string, e_inval);
        }

        if (struct_v >= 4) {
                err = decode_new_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_old_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_new_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;

                err = decode_old_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;
        }

        /* ignore the rest */
        *p = end;

        dout("inc osdmap epoch %d max_osd %d\n", map->epoch, map->max_osd);
        return map;

e_inval:
        err = -EINVAL;
bad:
        pr_err("corrupt inc osdmap (%d) epoch %d off %d (%p of %p-%p)\n",
               err, epoch, (int)(*p - start), *p, start, end);
        print_hex_dump(KERN_DEBUG, "osdmap: ",
                       DUMP_PREFIX_OFFSET, 16, 1,
                       start, end - start, true);
        return ERR_PTR(err);
}

void ceph_oloc_copy(struct ceph_object_locator *dest,
                    const struct ceph_object_locator *src)
{
        ceph_oloc_destroy(dest);

        dest->pool = src->pool;
        if (src->pool_ns)
                dest->pool_ns = ceph_get_string(src->pool_ns);
        else
                dest->pool_ns = NULL;
}
EXPORT_SYMBOL(ceph_oloc_copy);

void ceph_oloc_destroy(struct ceph_object_locator *oloc)
{
        ceph_put_string(oloc->pool_ns);
}
EXPORT_SYMBOL(ceph_oloc_destroy);

void ceph_oid_copy(struct ceph_object_id *dest,
                   const struct ceph_object_id *src)
{
        ceph_oid_destroy(dest);

        if (src->name != src->inline_name) {
                /* very rare, see ceph_object_id definition */
                dest->name = kmalloc(src->name_len + 1,
                                     GFP_NOIO | __GFP_NOFAIL);
        } else {
                dest->name = dest->inline_name;
        }
        memcpy(dest->name, src->name, src->name_len + 1);
        dest->name_len = src->name_len;
}
EXPORT_SYMBOL(ceph_oid_copy);

static __printf(2, 0)
int oid_printf_vargs(struct ceph_object_id *oid, const char *fmt, va_list ap)
{
        int len;

        WARN_ON(!ceph_oid_empty(oid));

        len = vsnprintf(oid->inline_name, sizeof(oid->inline_name), fmt, ap);
        if (len >= sizeof(oid->inline_name))
                return len;

        oid->name_len = len;
        return 0;
}

/*
 * If oid doesn't fit into inline buffer, BUG.
 */
void ceph_oid_printf(struct ceph_object_id *oid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        BUG_ON(oid_printf_vargs(oid, fmt, ap));
        va_end(ap);
}
EXPORT_SYMBOL(ceph_oid_printf);

static __printf(3, 0)
int oid_aprintf_vargs(struct ceph_object_id *oid, gfp_t gfp,
                      const char *fmt, va_list ap)
{
        va_list aq;
        int len;

        va_copy(aq, ap);
        len = oid_printf_vargs(oid, fmt, aq);
        va_end(aq);

        if (len) {
                char *external_name;

                external_name = kmalloc(len + 1, gfp);
                if (!external_name)
                        return -ENOMEM;

                oid->name = external_name;
                WARN_ON(vsnprintf(oid->name, len + 1, fmt, ap) != len);
                oid->name_len = len;
        }

        return 0;
}

/*
 * If oid doesn't fit into inline buffer, allocate.
 */
int ceph_oid_aprintf(struct ceph_object_id *oid, gfp_t gfp,
                     const char *fmt, ...)
{
        va_list ap;
        int ret;

        va_start(ap, fmt);
        ret = oid_aprintf_vargs(oid, gfp, fmt, ap);
        va_end(ap);

        return ret;
}
EXPORT_SYMBOL(ceph_oid_aprintf);

void ceph_oid_destroy(struct ceph_object_id *oid)
{
        if (oid->name != oid->inline_name)
                kfree(oid->name);
}
EXPORT_SYMBOL(ceph_oid_destroy);

/*
 * osds only
 */
static bool __osds_equal(const struct ceph_osds *lhs,
                         const struct ceph_osds *rhs)
{
        if (lhs->size == rhs->size &&
            !memcmp(lhs->osds, rhs->osds, rhs->size * sizeof(rhs->osds[0])))
                return true;

        return false;
}

/*
 * osds + primary
 */
static bool osds_equal(const struct ceph_osds *lhs,
                       const struct ceph_osds *rhs)
{
        if (__osds_equal(lhs, rhs) &&
            lhs->primary == rhs->primary)
                return true;

        return false;
}

static bool osds_valid(const struct ceph_osds *set)
{
        /* non-empty set */
        if (set->size > 0 && set->primary >= 0)
                return true;

        /* empty can_shift_osds set */
        if (!set->size && set->primary == -1)
                return true;

        /* empty !can_shift_osds set - all NONE */
        if (set->size > 0 && set->primary == -1) {
                int i;

                for (i = 0; i < set->size; i++) {
                        if (set->osds[i] != CRUSH_ITEM_NONE)
                                break;
                }
                if (i == set->size)
                        return true;
        }

        return false;
}

void ceph_osds_copy(struct ceph_osds *dest, const struct ceph_osds *src)
{
        memcpy(dest->osds, src->osds, src->size * sizeof(src->osds[0]));
        dest->size = src->size;
        dest->primary = src->primary;
}

bool ceph_pg_is_split(const struct ceph_pg *pgid, u32 old_pg_num,
                      u32 new_pg_num)
{
        int old_bits = calc_bits_of(old_pg_num);
        int old_mask = (1 << old_bits) - 1;
        int n;

        WARN_ON(pgid->seed >= old_pg_num);
        if (new_pg_num <= old_pg_num)
                return false;

        for (n = 1; ; n++) {
                int next_bit = n << (old_bits - 1);
                u32 s = next_bit | pgid->seed;

                if (s < old_pg_num || s == pgid->seed)
                        continue;
                if (s >= new_pg_num)
                        break;

                s = ceph_stable_mod(s, old_pg_num, old_mask);
                if (s == pgid->seed)
                        return true;
        }

        return false;
}

bool ceph_is_new_interval(const struct ceph_osds *old_acting,
                          const struct ceph_osds *new_acting,
                          const struct ceph_osds *old_up,
                          const struct ceph_osds *new_up,
                          int old_size,
                          int new_size,
                          int old_min_size,
                          int new_min_size,
                          u32 old_pg_num,
                          u32 new_pg_num,
                          bool old_sort_bitwise,
                          bool new_sort_bitwise,
                          bool old_recovery_deletes,
                          bool new_recovery_deletes,
                          const struct ceph_pg *pgid)
{
        return !osds_equal(old_acting, new_acting) ||
               !osds_equal(old_up, new_up) ||
               old_size != new_size ||
               old_min_size != new_min_size ||
               ceph_pg_is_split(pgid, old_pg_num, new_pg_num) ||
               old_sort_bitwise != new_sort_bitwise ||
               old_recovery_deletes != new_recovery_deletes;
}

static int calc_pg_rank(int osd, const struct ceph_osds *acting)
{
        int i;

        for (i = 0; i < acting->size; i++) {
                if (acting->osds[i] == osd)
                        return i;
        }

        return -1;
}

static bool primary_changed(const struct ceph_osds *old_acting,
                            const struct ceph_osds *new_acting)
{
        if (!old_acting->size && !new_acting->size)
                return false; /* both still empty */

        if (!old_acting->size ^ !new_acting->size)
                return true; /* was empty, now not, or vice versa */

        if (old_acting->primary != new_acting->primary)
                return true; /* primary changed */

        if (calc_pg_rank(old_acting->primary, old_acting) !=
            calc_pg_rank(new_acting->primary, new_acting))
                return true;

        return false; /* same primary (tho replicas may have changed) */
}

bool ceph_osds_changed(const struct ceph_osds *old_acting,
                       const struct ceph_osds *new_acting,
                       bool any_change)
{
        if (primary_changed(old_acting, new_acting))
                return true;

        if (any_change && !__osds_equal(old_acting, new_acting))
                return true;

        return false;
}

/*
 * Map an object into a PG.
 *
 * Should only be called with target_oid and target_oloc (as opposed to
 * base_oid and base_oloc), since tiering isn't taken into account.
 */
void __ceph_object_locator_to_pg(struct ceph_pg_pool_info *pi,
                                 const struct ceph_object_id *oid,
                                 const struct ceph_object_locator *oloc,
                                 struct ceph_pg *raw_pgid)
{
        WARN_ON(pi->id != oloc->pool);

        if (!oloc->pool_ns) {
                raw_pgid->pool = oloc->pool;
                raw_pgid->seed = ceph_str_hash(pi->object_hash, oid->name,
                                             oid->name_len);
                dout("%s %s -> raw_pgid %llu.%x\n", __func__, oid->name,
                     raw_pgid->pool, raw_pgid->seed);
        } else {
                char stack_buf[256];
                char *buf = stack_buf;
                int nsl = oloc->pool_ns->len;
                size_t total = nsl + 1 + oid->name_len;

                if (total > sizeof(stack_buf))
                        buf = kmalloc(total, GFP_NOIO | __GFP_NOFAIL);
                memcpy(buf, oloc->pool_ns->str, nsl);
                buf[nsl] = '\037';
                memcpy(buf + nsl + 1, oid->name, oid->name_len);
                raw_pgid->pool = oloc->pool;
                raw_pgid->seed = ceph_str_hash(pi->object_hash, buf, total);
                if (buf != stack_buf)
                        kfree(buf);
                dout("%s %s ns %.*s -> raw_pgid %llu.%x\n", __func__,
                     oid->name, nsl, oloc->pool_ns->str,
                     raw_pgid->pool, raw_pgid->seed);
        }
}

int ceph_object_locator_to_pg(struct ceph_osdmap *osdmap,
                              const struct ceph_object_id *oid,
                              const struct ceph_object_locator *oloc,
                              struct ceph_pg *raw_pgid)
{
        struct ceph_pg_pool_info *pi;

        pi = ceph_pg_pool_by_id(osdmap, oloc->pool);
        if (!pi)
                return -ENOENT;

        __ceph_object_locator_to_pg(pi, oid, oloc, raw_pgid);
        return 0;
}
EXPORT_SYMBOL(ceph_object_locator_to_pg);

/*
 * Map a raw PG (full precision ps) into an actual PG.
 */
static void raw_pg_to_pg(struct ceph_pg_pool_info *pi,
                         const struct ceph_pg *raw_pgid,
                         struct ceph_pg *pgid)
{
        pgid->pool = raw_pgid->pool;
        pgid->seed = ceph_stable_mod(raw_pgid->seed, pi->pg_num,
                                     pi->pg_num_mask);
}

/*
 * Map a raw PG (full precision ps) into a placement ps (placement
 * seed).  Include pool id in that value so that different pools don't
 * use the same seeds.
 */
static u32 raw_pg_to_pps(struct ceph_pg_pool_info *pi,
                         const struct ceph_pg *raw_pgid)
{
        if (pi->flags & CEPH_POOL_FLAG_HASHPSPOOL) {
                /* hash pool id and seed so that pool PGs do not overlap */
                return crush_hash32_2(CRUSH_HASH_RJENKINS1,
                                      ceph_stable_mod(raw_pgid->seed,
                                                      pi->pgp_num,
                                                      pi->pgp_num_mask),
                                      raw_pgid->pool);
        } else {
                /*
                 * legacy behavior: add ps and pool together.  this is
                 * not a great approach because the PGs from each pool
                 * will overlap on top of each other: 0.5 == 1.4 ==
                 * 2.3 == ...
                 */
                return ceph_stable_mod(raw_pgid->seed, pi->pgp_num,
                                       pi->pgp_num_mask) +
                       (unsigned)raw_pgid->pool;
        }
}

/*
 * Magic value used for a "default" fallback choose_args, used if the
 * crush_choose_arg_map passed to do_crush() does not exist.  If this
 * also doesn't exist, fall back to canonical weights.
 */
#define CEPH_DEFAULT_CHOOSE_ARGS        -1

static int do_crush(struct ceph_osdmap *map, int ruleno, int x,
                    int *result, int result_max,
                    const __u32 *weight, int weight_max,
                    s64 choose_args_index)
{
        struct crush_choose_arg_map *arg_map;
        struct crush_work *work;
        int r;

        BUG_ON(result_max > CEPH_PG_MAX_SIZE);

        arg_map = lookup_choose_arg_map(&map->crush->choose_args,
                                        choose_args_index);
        if (!arg_map)
                arg_map = lookup_choose_arg_map(&map->crush->choose_args,
                                                CEPH_DEFAULT_CHOOSE_ARGS);

        work = get_workspace(&map->crush_wsm, map->crush);
        r = crush_do_rule(map->crush, ruleno, x, result, result_max,
                          weight, weight_max, work,
                          arg_map ? arg_map->args : NULL);
        put_workspace(&map->crush_wsm, work);
        return r;
}

static void remove_nonexistent_osds(struct ceph_osdmap *osdmap,
                                    struct ceph_pg_pool_info *pi,
                                    struct ceph_osds *set)
{
        int i;

        if (ceph_can_shift_osds(pi)) {
                int removed = 0;

                /* shift left */
                for (i = 0; i < set->size; i++) {
                        if (!ceph_osd_exists(osdmap, set->osds[i])) {
                                removed++;
                                continue;
                        }
                        if (removed)
                                set->osds[i - removed] = set->osds[i];
                }
                set->size -= removed;
        } else {
                /* set dne devices to NONE */
                for (i = 0; i < set->size; i++) {
                        if (!ceph_osd_exists(osdmap, set->osds[i]))
                                set->osds[i] = CRUSH_ITEM_NONE;
                }
        }
}

/*
 * Calculate raw set (CRUSH output) for given PG and filter out
 * nonexistent OSDs.  ->primary is undefined for a raw set.
 *
 * Placement seed (CRUSH input) is returned through @ppps.
 */
static void pg_to_raw_osds(struct ceph_osdmap *osdmap,
                           struct ceph_pg_pool_info *pi,
                           const struct ceph_pg *raw_pgid,
                           struct ceph_osds *raw,
                           u32 *ppps)
{
        u32 pps = raw_pg_to_pps(pi, raw_pgid);
        int ruleno;
        int len;

        ceph_osds_init(raw);
        if (ppps)
                *ppps = pps;

        ruleno = crush_find_rule(osdmap->crush, pi->crush_ruleset, pi->type,
                                 pi->size);
        if (ruleno < 0) {
                pr_err("no crush rule: pool %lld ruleset %d type %d size %d\n",
                       pi->id, pi->crush_ruleset, pi->type, pi->size);
                return;
        }

        if (pi->size > ARRAY_SIZE(raw->osds)) {
                pr_err_ratelimited("pool %lld ruleset %d type %d too wide: size %d > %zu\n",
                       pi->id, pi->crush_ruleset, pi->type, pi->size,
                       ARRAY_SIZE(raw->osds));
                return;
        }

        len = do_crush(osdmap, ruleno, pps, raw->osds, pi->size,
                       osdmap->osd_weight, osdmap->max_osd, pi->id);
        if (len < 0) {
                pr_err("error %d from crush rule %d: pool %lld ruleset %d type %d size %d\n",
                       len, ruleno, pi->id, pi->crush_ruleset, pi->type,
                       pi->size);
                return;
        }

        raw->size = len;
        remove_nonexistent_osds(osdmap, pi, raw);
}

/* apply pg_upmap[_items] mappings */
static void apply_upmap(struct ceph_osdmap *osdmap,
                        const struct ceph_pg *pgid,
                        struct ceph_osds *raw)
{
        struct ceph_pg_mapping *pg;
        int i, j;

        pg = lookup_pg_mapping(&osdmap->pg_upmap, pgid);
        if (pg) {
                /* make sure targets aren't marked out */
                for (i = 0; i < pg->pg_upmap.len; i++) {
                        int osd = pg->pg_upmap.osds[i];

                        if (osd != CRUSH_ITEM_NONE &&
                            osd < osdmap->max_osd &&
                            osdmap->osd_weight[osd] == 0) {
                                /* reject/ignore explicit mapping */
                                return;
                        }
                }
                for (i = 0; i < pg->pg_upmap.len; i++)
                        raw->osds[i] = pg->pg_upmap.osds[i];
                raw->size = pg->pg_upmap.len;
                /* check and apply pg_upmap_items, if any */
        }

        pg = lookup_pg_mapping(&osdmap->pg_upmap_items, pgid);
        if (pg) {
                /*
                 * Note: this approach does not allow a bidirectional swap,
                 * e.g., [[1,2],[2,1]] applied to [0,1,2] -> [0,2,1].
                 */
                for (i = 0; i < pg->pg_upmap_items.len; i++) {
                        int from = pg->pg_upmap_items.from_to[i][0];
                        int to = pg->pg_upmap_items.from_to[i][1];
                        int pos = -1;
                        bool exists = false;

                        /* make sure replacement doesn't already appear */
                        for (j = 0; j < raw->size; j++) {
                                int osd = raw->osds[j];

                                if (osd == to) {
                                        exists = true;
                                        break;
                                }
                                /* ignore mapping if target is marked out */
                                if (osd == from && pos < 0 &&
                                    !(to != CRUSH_ITEM_NONE &&
                                      to < osdmap->max_osd &&
                                      osdmap->osd_weight[to] == 0)) {
                                        pos = j;
                                }
                        }
                        if (!exists && pos >= 0)
                                raw->osds[pos] = to;
                }
        }
}

/*
 * Given raw set, calculate up set and up primary.  By definition of an
 * up set, the result won't contain nonexistent or down OSDs.
 *
 * This is done in-place - on return @set is the up set.  If it's
 * empty, ->primary will remain undefined.
 */
static void raw_to_up_osds(struct ceph_osdmap *osdmap,
                           struct ceph_pg_pool_info *pi,
                           struct ceph_osds *set)
{
        int i;

        /* ->primary is undefined for a raw set */
        BUG_ON(set->primary != -1);

        if (ceph_can_shift_osds(pi)) {
                int removed = 0;

                /* shift left */
                for (i = 0; i < set->size; i++) {
                        if (ceph_osd_is_down(osdmap, set->osds[i])) {
                                removed++;
                                continue;
                        }
                        if (removed)
                                set->osds[i - removed] = set->osds[i];
                }
                set->size -= removed;
                if (set->size > 0)
                        set->primary = set->osds[0];
        } else {
                /* set down/dne devices to NONE */
                for (i = set->size - 1; i >= 0; i--) {
                        if (ceph_osd_is_down(osdmap, set->osds[i]))
                                set->osds[i] = CRUSH_ITEM_NONE;
                        else
                                set->primary = set->osds[i];
                }
        }
}

static void apply_primary_affinity(struct ceph_osdmap *osdmap,
                                   struct ceph_pg_pool_info *pi,
                                   u32 pps,
                                   struct ceph_osds *up)
{
        int i;
        int pos = -1;

        /*
         * Do we have any non-default primary_affinity values for these
         * osds?
         */
        if (!osdmap->osd_primary_affinity)
                return;

        for (i = 0; i < up->size; i++) {
                int osd = up->osds[i];

                if (osd != CRUSH_ITEM_NONE &&
                    osdmap->osd_primary_affinity[osd] !=
                                        CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
                        break;
                }
        }
        if (i == up->size)
                return;

        /*
         * Pick the primary.  Feed both the seed (for the pg) and the
         * osd into the hash/rng so that a proportional fraction of an
         * osd's pgs get rejected as primary.
         */
        for (i = 0; i < up->size; i++) {
                int osd = up->osds[i];
                u32 aff;

                if (osd == CRUSH_ITEM_NONE)
                        continue;

                aff = osdmap->osd_primary_affinity[osd];
                if (aff < CEPH_OSD_MAX_PRIMARY_AFFINITY &&
                    (crush_hash32_2(CRUSH_HASH_RJENKINS1,
                                    pps, osd) >> 16) >= aff) {
                        /*
                         * We chose not to use this primary.  Note it
                         * anyway as a fallback in case we don't pick
                         * anyone else, but keep looking.
                         */
                        if (pos < 0)
                                pos = i;
                } else {
                        pos = i;
                        break;
                }
        }
        if (pos < 0)
                return;

        up->primary = up->osds[pos];

        if (ceph_can_shift_osds(pi) && pos > 0) {
                /* move the new primary to the front */
                for (i = pos; i > 0; i--)
                        up->osds[i] = up->osds[i - 1];
                up->osds[0] = up->primary;
        }
}

/*
 * Get pg_temp and primary_temp mappings for given PG.
 *
 * Note that a PG may have none, only pg_temp, only primary_temp or
 * both pg_temp and primary_temp mappings.  This means @temp isn't
 * always a valid OSD set on return: in the "only primary_temp" case,
 * @temp will have its ->primary >= 0 but ->size == 0.
 */
static void get_temp_osds(struct ceph_osdmap *osdmap,
                          struct ceph_pg_pool_info *pi,
                          const struct ceph_pg *pgid,
                          struct ceph_osds *temp)
{
        struct ceph_pg_mapping *pg;
        int i;

        ceph_osds_init(temp);

        /* pg_temp? */
        pg = lookup_pg_mapping(&osdmap->pg_temp, pgid);
        if (pg) {
                for (i = 0; i < pg->pg_temp.len; i++) {
                        if (ceph_osd_is_down(osdmap, pg->pg_temp.osds[i])) {
                                if (ceph_can_shift_osds(pi))
                                        continue;

                                temp->osds[temp->size++] = CRUSH_ITEM_NONE;
                        } else {
                                temp->osds[temp->size++] = pg->pg_temp.osds[i];
                        }
                }

                /* apply pg_temp's primary */
                for (i = 0; i < temp->size; i++) {
                        if (temp->osds[i] != CRUSH_ITEM_NONE) {
                                temp->primary = temp->osds[i];
                                break;
                        }
                }
        }

        /* primary_temp? */
        pg = lookup_pg_mapping(&osdmap->primary_temp, pgid);
        if (pg)
                temp->primary = pg->primary_temp.osd;
}

/*
 * Map a PG to its acting set as well as its up set.
 *
 * Acting set is used for data mapping purposes, while up set can be
 * recorded for detecting interval changes and deciding whether to
 * resend a request.
 */
void ceph_pg_to_up_acting_osds(struct ceph_osdmap *osdmap,
                               struct ceph_pg_pool_info *pi,
                               const struct ceph_pg *raw_pgid,
                               struct ceph_osds *up,
                               struct ceph_osds *acting)
{
        struct ceph_pg pgid;
        u32 pps;

        WARN_ON(pi->id != raw_pgid->pool);
        raw_pg_to_pg(pi, raw_pgid, &pgid);

        pg_to_raw_osds(osdmap, pi, raw_pgid, up, &pps);
        apply_upmap(osdmap, &pgid, up);
        raw_to_up_osds(osdmap, pi, up);
        apply_primary_affinity(osdmap, pi, pps, up);
        get_temp_osds(osdmap, pi, &pgid, acting);
        if (!acting->size) {
                memcpy(acting->osds, up->osds, up->size * sizeof(up->osds[0]));
                acting->size = up->size;
                if (acting->primary == -1)
                        acting->primary = up->primary;
        }
        WARN_ON(!osds_valid(up) || !osds_valid(acting));
}

bool ceph_pg_to_primary_shard(struct ceph_osdmap *osdmap,
                              struct ceph_pg_pool_info *pi,
                              const struct ceph_pg *raw_pgid,
                              struct ceph_spg *spgid)
{
        struct ceph_pg pgid;
        struct ceph_osds up, acting;
        int i;

        WARN_ON(pi->id != raw_pgid->pool);
        raw_pg_to_pg(pi, raw_pgid, &pgid);

        if (ceph_can_shift_osds(pi)) {
                spgid->pgid = pgid; /* struct */
                spgid->shard = CEPH_SPG_NOSHARD;
                return true;
        }

        ceph_pg_to_up_acting_osds(osdmap, pi, &pgid, &up, &acting);
        for (i = 0; i < acting.size; i++) {
                if (acting.osds[i] == acting.primary) {
                        spgid->pgid = pgid; /* struct */
                        spgid->shard = i;
                        return true;
                }
        }

        return false;
}

/*
 * Return acting primary for given PG, or -1 if none.
 */
int ceph_pg_to_acting_primary(struct ceph_osdmap *osdmap,
                              const struct ceph_pg *raw_pgid)
{
        struct ceph_pg_pool_info *pi;
        struct ceph_osds up, acting;

        pi = ceph_pg_pool_by_id(osdmap, raw_pgid->pool);
        if (!pi)
                return -1;

        ceph_pg_to_up_acting_osds(osdmap, pi, raw_pgid, &up, &acting);
        return acting.primary;
}
EXPORT_SYMBOL(ceph_pg_to_acting_primary);

static struct crush_loc_node *alloc_crush_loc(size_t type_name_len,
                                              size_t name_len)
{
        struct crush_loc_node *loc;

        loc = kmalloc(sizeof(*loc) + type_name_len + name_len + 2, GFP_NOIO);
        if (!loc)
                return NULL;

        RB_CLEAR_NODE(&loc->cl_node);
        return loc;
}

static void free_crush_loc(struct crush_loc_node *loc)
{
        WARN_ON(!RB_EMPTY_NODE(&loc->cl_node));

        kfree(loc);
}

static int crush_loc_compare(const struct crush_loc *loc1,
                             const struct crush_loc *loc2)
{
        return strcmp(loc1->cl_type_name, loc2->cl_type_name) ?:
               strcmp(loc1->cl_name, loc2->cl_name);
}

DEFINE_RB_FUNCS2(crush_loc, struct crush_loc_node, cl_loc, crush_loc_compare,
                 RB_BYPTR, const struct crush_loc *, cl_node)

/*
 * Parses a set of <bucket type name>':'<bucket name> pairs separated
 * by '|', e.g. "rack:foo1|rack:foo2|datacenter:bar".
 *
 * Note that @crush_location is modified by strsep().
 */
int ceph_parse_crush_location(char *crush_location, struct rb_root *locs)
{
        struct crush_loc_node *loc;
        const char *type_name, *name, *colon;
        size_t type_name_len, name_len;

        dout("%s '%s'\n", __func__, crush_location);
        while ((type_name = strsep(&crush_location, "|"))) {
                colon = strchr(type_name, ':');
                if (!colon)
                        return -EINVAL;

                type_name_len = colon - type_name;
                if (type_name_len == 0)
                        return -EINVAL;

                name = colon + 1;
                name_len = strlen(name);
                if (name_len == 0)
                        return -EINVAL;

                loc = alloc_crush_loc(type_name_len, name_len);
                if (!loc)
                        return -ENOMEM;

                loc->cl_loc.cl_type_name = loc->cl_data;
                memcpy(loc->cl_loc.cl_type_name, type_name, type_name_len);
                loc->cl_loc.cl_type_name[type_name_len] = '\0';

                loc->cl_loc.cl_name = loc->cl_data + type_name_len + 1;
                memcpy(loc->cl_loc.cl_name, name, name_len);
                loc->cl_loc.cl_name[name_len] = '\0';

                if (!__insert_crush_loc(locs, loc)) {
                        free_crush_loc(loc);
                        return -EEXIST;
                }

                dout("%s type_name '%s' name '%s'\n", __func__,
                     loc->cl_loc.cl_type_name, loc->cl_loc.cl_name);
        }

        return 0;
}

int ceph_compare_crush_locs(struct rb_root *locs1, struct rb_root *locs2)
{
        struct rb_node *n1 = rb_first(locs1);
        struct rb_node *n2 = rb_first(locs2);
        int ret;

        for ( ; n1 && n2; n1 = rb_next(n1), n2 = rb_next(n2)) {
                struct crush_loc_node *loc1 =
                    rb_entry(n1, struct crush_loc_node, cl_node);
                struct crush_loc_node *loc2 =
                    rb_entry(n2, struct crush_loc_node, cl_node);

                ret = crush_loc_compare(&loc1->cl_loc, &loc2->cl_loc);
                if (ret)
                        return ret;
        }

        if (!n1 && n2)
                return -1;
        if (n1 && !n2)
                return 1;
        return 0;
}

void ceph_clear_crush_locs(struct rb_root *locs)
{
        while (!RB_EMPTY_ROOT(locs)) {
                struct crush_loc_node *loc =
                    rb_entry(rb_first(locs), struct crush_loc_node, cl_node);

                erase_crush_loc(locs, loc);
                free_crush_loc(loc);
        }
}

/*
 * [a-zA-Z0-9-_.]+
 */
static bool is_valid_crush_name(const char *name)
{
        do {
                if (!('a' <= *name && *name <= 'z') &&
                    !('A' <= *name && *name <= 'Z') &&
                    !('0' <= *name && *name <= '9') &&
                    *name != '-' && *name != '_' && *name != '.')
                        return false;
        } while (*++name != '\0');

        return true;
}

/*
 * Gets the parent of an item.  Returns its id (<0 because the
 * parent is always a bucket), type id (>0 for the same reason,
 * via @parent_type_id) and location (via @parent_loc).  If no
 * parent, returns 0.
 *
 * Does a linear search, as there are no parent pointers of any
 * kind.  Note that the result is ambiguous for items that occur
 * multiple times in the map.
 */
static int get_immediate_parent(struct crush_map *c, int id,
                                u16 *parent_type_id,
                                struct crush_loc *parent_loc)
{
        struct crush_bucket *b;
        struct crush_name_node *type_cn, *cn;
        int i, j;

        for (i = 0; i < c->max_buckets; i++) {
                b = c->buckets[i];
                if (!b)
                        continue;

                /* ignore per-class shadow hierarchy */
                cn = lookup_crush_name(&c->names, b->id);
                if (!cn || !is_valid_crush_name(cn->cn_name))
                        continue;

                for (j = 0; j < b->size; j++) {
                        if (b->items[j] != id)
                                continue;

                        *parent_type_id = b->type;
                        type_cn = lookup_crush_name(&c->type_names, b->type);
                        parent_loc->cl_type_name = type_cn->cn_name;
                        parent_loc->cl_name = cn->cn_name;
                        return b->id;
                }
        }

        return 0;  /* no parent */
}

/*
 * Calculates the locality/distance from an item to a client
 * location expressed in terms of CRUSH hierarchy as a set of
 * (bucket type name, bucket name) pairs.  Specifically, looks
 * for the lowest-valued bucket type for which the location of
 * @id matches one of the locations in @locs, so for standard
 * bucket types (host = 1, rack = 3, datacenter = 8, zone = 9)
 * a matching host is closer than a matching rack and a matching
 * data center is closer than a matching zone.
 *
 * Specifying multiple locations (a "multipath" location) such
 * as "rack=foo1 rack=foo2 datacenter=bar" is allowed -- @locs
 * is a multimap.  The locality will be:
 *
 * - 3 for OSDs in racks foo1 and foo2
 * - 8 for OSDs in data center bar
 * - -1 for all other OSDs
 *
 * The lowest possible bucket type is 1, so the best locality
 * for an OSD is 1 (i.e. a matching host).  Locality 0 would be
 * the OSD itself.
 */
int ceph_get_crush_locality(struct ceph_osdmap *osdmap, int id,
                            struct rb_root *locs)
{
        struct crush_loc loc;
        u16 type_id;

        /*
         * Instead of repeated get_immediate_parent() calls,
         * the location of @id could be obtained with a single
         * depth-first traversal.
         */
        for (;;) {
                id = get_immediate_parent(osdmap->crush, id, &type_id, &loc);
                if (id >= 0)
                        return -1;  /* not local */

                if (lookup_crush_loc(locs, &loc))
                        return type_id;
        }
}























    1 












    1 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2011 Instituto Nokia de Tecnologia
 *
 * Authors:
 *    Aloisio Almeida Jr <aloisio.almeida@openbossa.org>
 *    Lauro Ramos Venancio <lauro.venancio@openbossa.org>
 */

#include <linux/nfc.h>
#include <linux/module.h>

#include "nfc.h"

static DEFINE_RWLOCK(proto_tab_lock);
static const struct nfc_protocol *proto_tab[NFC_SOCKPROTO_MAX];

static int nfc_sock_create(struct net *net, struct socket *sock, int proto,
                           int kern)
{
        int rc = -EPROTONOSUPPORT;

        if (net != &init_net)
                return -EAFNOSUPPORT;

        if (proto < 0 || proto >= NFC_SOCKPROTO_MAX)
                return -EINVAL;

        read_lock(&proto_tab_lock);
        if (proto_tab[proto] &&        try_module_get(proto_tab[proto]->owner)) {
                rc = proto_tab[proto]->create(net, sock, proto_tab[proto], kern);
                module_put(proto_tab[proto]->owner);
        }
        read_unlock(&proto_tab_lock);

        return rc;
}

static const struct net_proto_family nfc_sock_family_ops = {
        .owner  = THIS_MODULE,
        .family = PF_NFC,
        .create = nfc_sock_create,
};

int nfc_proto_register(const struct nfc_protocol *nfc_proto)
{
        int rc;

        if (nfc_proto->id < 0 || nfc_proto->id >= NFC_SOCKPROTO_MAX)
                return -EINVAL;

        rc = proto_register(nfc_proto->proto, 0);
        if (rc)
                return rc;

        write_lock(&proto_tab_lock);
        if (proto_tab[nfc_proto->id])
                rc = -EBUSY;
        else
                proto_tab[nfc_proto->id] = nfc_proto;
        write_unlock(&proto_tab_lock);

        if (rc)
                proto_unregister(nfc_proto->proto);

        return rc;
}
EXPORT_SYMBOL(nfc_proto_register);

void nfc_proto_unregister(const struct nfc_protocol *nfc_proto)
{
        write_lock(&proto_tab_lock);
        proto_tab[nfc_proto->id] = NULL;
        write_unlock(&proto_tab_lock);

        proto_unregister(nfc_proto->proto);
}
EXPORT_SYMBOL(nfc_proto_unregister);

int __init af_nfc_init(void)
{
        return sock_register(&nfc_sock_family_ops);
}

void __exit af_nfc_exit(void)
{
        sock_unregister(PF_NFC);
}






































































































































































































































































































  538 




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
/*
 * Linux Security plug
 *
 * Copyright (C) 2001 WireX Communications, Inc <chris@wirex.com>
 * Copyright (C) 2001 Greg Kroah-Hartman <greg@kroah.com>
 * Copyright (C) 2001 Networks Associates Technology, Inc <ssmalley@nai.com>
 * Copyright (C) 2001 James Morris <jmorris@intercode.com.au>
 * Copyright (C) 2001 Silicon Graphics, Inc. (Trust Technology Group)
 * Copyright (C) 2016 Mellanox Techonologies
 *
 *        This program is free software; you can redistribute it and/or modify
 *        it under the terms of the GNU General Public License as published by
 *        the Free Software Foundation; either version 2 of the License, or
 *        (at your option) any later version.
 *
 *        Due to this file being licensed under the GPL there is controversy over
 *        whether this permits you to write a module that #includes this file
 *        without placing your module under the GPL.  Please consult a lawyer for
 *        advice before doing this.
 *
 */

#ifndef __LINUX_SECURITY_H
#define __LINUX_SECURITY_H

#include <linux/kernel_read_file.h>
#include <linux/key.h>
#include <linux/capability.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/sockptr.h>
#include <linux/bpf.h>
#include <uapi/linux/lsm.h>
#include <linux/lsm/selinux.h>
#include <linux/lsm/smack.h>
#include <linux/lsm/apparmor.h>
#include <linux/lsm/bpf.h>

struct linux_binprm;
struct cred;
struct rlimit;
struct kernel_siginfo;
struct sembuf;
struct kern_ipc_perm;
struct audit_context;
struct super_block;
struct inode;
struct dentry;
struct file;
struct vfsmount;
struct path;
struct qstr;
struct iattr;
struct fown_struct;
struct file_operations;
struct msg_msg;
struct xattr;
struct kernfs_node;
struct xfrm_sec_ctx;
struct mm_struct;
struct fs_context;
struct fs_parameter;
enum fs_value_type;
struct watch;
struct watch_notification;
struct lsm_ctx;

/* Default (no) options for the capable function */
#define CAP_OPT_NONE 0x0
/* If capable should audit the security request */
#define CAP_OPT_NOAUDIT BIT(1)
/* If capable is being called by a setid function */
#define CAP_OPT_INSETID BIT(2)

/* LSM Agnostic defines for security_sb_set_mnt_opts() flags */
#define SECURITY_LSM_NATIVE_LABELS        1

struct ctl_table;
struct audit_krule;
struct user_namespace;
struct timezone;

enum lsm_event {
        LSM_POLICY_CHANGE,
};

struct dm_verity_digest {
        const char *alg;
        const u8 *digest;
        size_t digest_len;
};

enum lsm_integrity_type {
        LSM_INT_DMVERITY_SIG_VALID,
        LSM_INT_DMVERITY_ROOTHASH,
        LSM_INT_FSVERITY_BUILTINSIG_VALID,
};

/*
 * These are reasons that can be passed to the security_locked_down()
 * LSM hook. Lockdown reasons that protect kernel integrity (ie, the
 * ability for userland to modify kernel code) are placed before
 * LOCKDOWN_INTEGRITY_MAX.  Lockdown reasons that protect kernel
 * confidentiality (ie, the ability for userland to extract
 * information from the running kernel that would otherwise be
 * restricted) are placed before LOCKDOWN_CONFIDENTIALITY_MAX.
 *
 * LSM authors should note that the semantics of any given lockdown
 * reason are not guaranteed to be stable - the same reason may block
 * one set of features in one kernel release, and a slightly different
 * set of features in a later kernel release. LSMs that seek to expose
 * lockdown policy at any level of granularity other than "none",
 * "integrity" or "confidentiality" are responsible for either
 * ensuring that they expose a consistent level of functionality to
 * userland, or ensuring that userland is aware that this is
 * potentially a moving target. It is easy to misuse this information
 * in a way that could break userspace. Please be careful not to do
 * so.
 *
 * If you add to this, remember to extend lockdown_reasons in
 * security/lockdown/lockdown.c.
 */
enum lockdown_reason {
        LOCKDOWN_NONE,
        LOCKDOWN_MODULE_SIGNATURE,
        LOCKDOWN_DEV_MEM,
        LOCKDOWN_EFI_TEST,
        LOCKDOWN_KEXEC,
        LOCKDOWN_HIBERNATION,
        LOCKDOWN_PCI_ACCESS,
        LOCKDOWN_IOPORT,
        LOCKDOWN_MSR,
        LOCKDOWN_ACPI_TABLES,
        LOCKDOWN_DEVICE_TREE,
        LOCKDOWN_PCMCIA_CIS,
        LOCKDOWN_TIOCSSERIAL,
        LOCKDOWN_MODULE_PARAMETERS,
        LOCKDOWN_MMIOTRACE,
        LOCKDOWN_DEBUGFS,
        LOCKDOWN_XMON_WR,
        LOCKDOWN_BPF_WRITE_USER,
        LOCKDOWN_DBG_WRITE_KERNEL,
        LOCKDOWN_RTAS_ERROR_INJECTION,
        LOCKDOWN_INTEGRITY_MAX,
        LOCKDOWN_KCORE,
        LOCKDOWN_KPROBES,
        LOCKDOWN_BPF_READ_KERNEL,
        LOCKDOWN_DBG_READ_KERNEL,
        LOCKDOWN_PERF,
        LOCKDOWN_TRACEFS,
        LOCKDOWN_XMON_RW,
        LOCKDOWN_XFRM_SECRET,
        LOCKDOWN_CONFIDENTIALITY_MAX,
};

/*
 * Data exported by the security modules
 */
struct lsm_prop {
        struct lsm_prop_selinux selinux;
        struct lsm_prop_smack smack;
        struct lsm_prop_apparmor apparmor;
        struct lsm_prop_bpf bpf;
};

extern const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX+1];
extern u32 lsm_active_cnt;
extern const struct lsm_id *lsm_idlist[];

/* These functions are in security/commoncap.c */
extern int cap_capable(const struct cred *cred, struct user_namespace *ns,
                       int cap, unsigned int opts);
extern int cap_settime(const struct timespec64 *ts, const struct timezone *tz);
extern int cap_ptrace_access_check(struct task_struct *child, unsigned int mode);
extern int cap_ptrace_traceme(struct task_struct *parent);
extern int cap_capget(const struct task_struct *target, kernel_cap_t *effective,
                      kernel_cap_t *inheritable, kernel_cap_t *permitted);
extern int cap_capset(struct cred *new, const struct cred *old,
                      const kernel_cap_t *effective,
                      const kernel_cap_t *inheritable,
                      const kernel_cap_t *permitted);
extern int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file);
int cap_inode_setxattr(struct dentry *dentry, const char *name,
                       const void *value, size_t size, int flags);
int cap_inode_removexattr(struct mnt_idmap *idmap,
                          struct dentry *dentry, const char *name);
int cap_inode_need_killpriv(struct dentry *dentry);
int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry);
int cap_inode_getsecurity(struct mnt_idmap *idmap,
                          struct inode *inode, const char *name, void **buffer,
                          bool alloc);
extern int cap_mmap_addr(unsigned long addr);
extern int cap_mmap_file(struct file *file, unsigned long reqprot,
                         unsigned long prot, unsigned long flags);
extern int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags);
extern int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
                          unsigned long arg4, unsigned long arg5);
extern int cap_task_setscheduler(struct task_struct *p);
extern int cap_task_setioprio(struct task_struct *p, int ioprio);
extern int cap_task_setnice(struct task_struct *p, int nice);
extern int cap_vm_enough_memory(struct mm_struct *mm, long pages);

struct msghdr;
struct sk_buff;
struct sock;
struct sockaddr;
struct socket;
struct flowi_common;
struct dst_entry;
struct xfrm_selector;
struct xfrm_policy;
struct xfrm_state;
struct xfrm_user_sec_ctx;
struct seq_file;
struct sctp_association;

#ifdef CONFIG_MMU
extern unsigned long mmap_min_addr;
extern unsigned long dac_mmap_min_addr;
#else
#define mmap_min_addr                0UL
#define dac_mmap_min_addr        0UL
#endif

/*
 * Values used in the task_security_ops calls
 */
/* setuid or setgid, id0 == uid or gid */
#define LSM_SETID_ID        1

/* setreuid or setregid, id0 == real, id1 == eff */
#define LSM_SETID_RE        2

/* setresuid or setresgid, id0 == real, id1 == eff, uid2 == saved */
#define LSM_SETID_RES        4

/* setfsuid or setfsgid, id0 == fsuid or fsgid */
#define LSM_SETID_FS        8

/* Flags for security_task_prlimit(). */
#define LSM_PRLIMIT_READ  1
#define LSM_PRLIMIT_WRITE 2

/* forward declares to avoid warnings */
struct sched_param;
struct request_sock;

/* bprm->unsafe reasons */
#define LSM_UNSAFE_SHARE        1
#define LSM_UNSAFE_PTRACE        2
#define LSM_UNSAFE_NO_NEW_PRIVS        4

#ifdef CONFIG_MMU
extern int mmap_min_addr_handler(const struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos);
#endif

/* security_inode_init_security callback function to write xattrs */
typedef int (*initxattrs) (struct inode *inode,
                           const struct xattr *xattr_array, void *fs_data);


/* Keep the kernel_load_data_id enum in sync with kernel_read_file_id */
#define __data_id_enumify(ENUM, dummy) LOADING_ ## ENUM,
#define __data_id_stringify(dummy, str) #str,

enum kernel_load_data_id {
        __kernel_read_file_id(__data_id_enumify)
};

static const char * const kernel_load_data_str[] = {
        __kernel_read_file_id(__data_id_stringify)
};

static inline const char *kernel_load_data_id_str(enum kernel_load_data_id id)
{
        if ((unsigned)id >= LOADING_MAX_ID)
                return kernel_load_data_str[LOADING_UNKNOWN];

        return kernel_load_data_str[id];
}

/**
 * lsmprop_init - initialize a lsm_prop structure
 * @prop: Pointer to the data to initialize
 *
 * Set all secid for all modules to the specified value.
 */
static inline void lsmprop_init(struct lsm_prop *prop)
{
        memset(prop, 0, sizeof(*prop));
}

#ifdef CONFIG_SECURITY

/**
 * lsmprop_is_set - report if there is a value in the lsm_prop
 * @prop: Pointer to the exported LSM data
 *
 * Returns true if there is a value set, false otherwise
 */
static inline bool lsmprop_is_set(struct lsm_prop *prop)
{
        const struct lsm_prop empty = {};

        return !!memcmp(prop, &empty, sizeof(*prop));
}

int call_blocking_lsm_notifier(enum lsm_event event, void *data);
int register_blocking_lsm_notifier(struct notifier_block *nb);
int unregister_blocking_lsm_notifier(struct notifier_block *nb);

/* prototypes */
extern int security_init(void);
extern int early_security_init(void);
extern u64 lsm_name_to_attr(const char *name);

/* Security operations */
int security_binder_set_context_mgr(const struct cred *mgr);
int security_binder_transaction(const struct cred *from,
                                const struct cred *to);
int security_binder_transfer_binder(const struct cred *from,
                                    const struct cred *to);
int security_binder_transfer_file(const struct cred *from,
                                  const struct cred *to, const struct file *file);
int security_ptrace_access_check(struct task_struct *child, unsigned int mode);
int security_ptrace_traceme(struct task_struct *parent);
int security_capget(const struct task_struct *target,
                    kernel_cap_t *effective,
                    kernel_cap_t *inheritable,
                    kernel_cap_t *permitted);
int security_capset(struct cred *new, const struct cred *old,
                    const kernel_cap_t *effective,
                    const kernel_cap_t *inheritable,
                    const kernel_cap_t *permitted);
int security_capable(const struct cred *cred,
                       struct user_namespace *ns,
                       int cap,
                       unsigned int opts);
int security_quotactl(int cmds, int type, int id, const struct super_block *sb);
int security_quota_on(struct dentry *dentry);
int security_syslog(int type);
int security_settime64(const struct timespec64 *ts, const struct timezone *tz);
int security_vm_enough_memory_mm(struct mm_struct *mm, long pages);
int security_bprm_creds_for_exec(struct linux_binprm *bprm);
int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file);
int security_bprm_check(struct linux_binprm *bprm);
void security_bprm_committing_creds(const struct linux_binprm *bprm);
void security_bprm_committed_creds(const struct linux_binprm *bprm);
int security_fs_context_submount(struct fs_context *fc, struct super_block *reference);
int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc);
int security_fs_context_parse_param(struct fs_context *fc, struct fs_parameter *param);
int security_sb_alloc(struct super_block *sb);
void security_sb_delete(struct super_block *sb);
void security_sb_free(struct super_block *sb);
void security_free_mnt_opts(void **mnt_opts);
int security_sb_eat_lsm_opts(char *options, void **mnt_opts);
int security_sb_mnt_opts_compat(struct super_block *sb, void *mnt_opts);
int security_sb_remount(struct super_block *sb, void *mnt_opts);
int security_sb_kern_mount(const struct super_block *sb);
int security_sb_show_options(struct seq_file *m, struct super_block *sb);
int security_sb_statfs(struct dentry *dentry);
int security_sb_mount(const char *dev_name, const struct path *path,
                      const char *type, unsigned long flags, void *data);
int security_sb_umount(struct vfsmount *mnt, int flags);
int security_sb_pivotroot(const struct path *old_path, const struct path *new_path);
int security_sb_set_mnt_opts(struct super_block *sb,
                                void *mnt_opts,
                                unsigned long kern_flags,
                                unsigned long *set_kern_flags);
int security_sb_clone_mnt_opts(const struct super_block *oldsb,
                                struct super_block *newsb,
                                unsigned long kern_flags,
                                unsigned long *set_kern_flags);
int security_move_mount(const struct path *from_path, const struct path *to_path);
int security_dentry_init_security(struct dentry *dentry, int mode,
                                  const struct qstr *name,
                                  const char **xattr_name, void **ctx,
                                  u32 *ctxlen);
int security_dentry_create_files_as(struct dentry *dentry, int mode,
                                        struct qstr *name,
                                        const struct cred *old,
                                        struct cred *new);
int security_path_notify(const struct path *path, u64 mask,
                                        unsigned int obj_type);
int security_inode_alloc(struct inode *inode, gfp_t gfp);
void security_inode_free(struct inode *inode);
int security_inode_init_security(struct inode *inode, struct inode *dir,
                                 const struct qstr *qstr,
                                 initxattrs initxattrs, void *fs_data);
int security_inode_init_security_anon(struct inode *inode,
                                      const struct qstr *name,
                                      const struct inode *context_inode);
int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode);
void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
                                        struct inode *inode);
int security_inode_link(struct dentry *old_dentry, struct inode *dir,
                         struct dentry *new_dentry);
int security_inode_unlink(struct inode *dir, struct dentry *dentry);
int security_inode_symlink(struct inode *dir, struct dentry *dentry,
                           const char *old_name);
int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
int security_inode_rmdir(struct inode *dir, struct dentry *dentry);
int security_inode_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev);
int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry,
                          struct inode *new_dir, struct dentry *new_dentry,
                          unsigned int flags);
int security_inode_readlink(struct dentry *dentry);
int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
                               bool rcu);
int security_inode_permission(struct inode *inode, int mask);
int security_inode_setattr(struct mnt_idmap *idmap,
                           struct dentry *dentry, struct iattr *attr);
void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                                 int ia_valid);
int security_inode_getattr(const struct path *path);
int security_inode_setxattr(struct mnt_idmap *idmap,
                            struct dentry *dentry, const char *name,
                            const void *value, size_t size, int flags);
int security_inode_set_acl(struct mnt_idmap *idmap,
                           struct dentry *dentry, const char *acl_name,
                           struct posix_acl *kacl);
void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name,
                                 struct posix_acl *kacl);
int security_inode_get_acl(struct mnt_idmap *idmap,
                           struct dentry *dentry, const char *acl_name);
int security_inode_remove_acl(struct mnt_idmap *idmap,
                              struct dentry *dentry, const char *acl_name);
void security_inode_post_remove_acl(struct mnt_idmap *idmap,
                                    struct dentry *dentry,
                                    const char *acl_name);
void security_inode_post_setxattr(struct dentry *dentry, const char *name,
                                  const void *value, size_t size, int flags);
int security_inode_getxattr(struct dentry *dentry, const char *name);
int security_inode_listxattr(struct dentry *dentry);
int security_inode_removexattr(struct mnt_idmap *idmap,
                               struct dentry *dentry, const char *name);
void security_inode_post_removexattr(struct dentry *dentry, const char *name);
int security_inode_need_killpriv(struct dentry *dentry);
int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry);
int security_inode_getsecurity(struct mnt_idmap *idmap,
                               struct inode *inode, const char *name,
                               void **buffer, bool alloc);
int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags);
int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size);
void security_inode_getlsmprop(struct inode *inode, struct lsm_prop *prop);
int security_inode_copy_up(struct dentry *src, struct cred **new);
int security_inode_copy_up_xattr(struct dentry *src, const char *name);
int security_inode_setintegrity(const struct inode *inode,
                                enum lsm_integrity_type type, const void *value,
                                size_t size);
int security_kernfs_init_security(struct kernfs_node *kn_dir,
                                  struct kernfs_node *kn);
int security_file_permission(struct file *file, int mask);
int security_file_alloc(struct file *file);
void security_file_release(struct file *file);
void security_file_free(struct file *file);
int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
int security_file_ioctl_compat(struct file *file, unsigned int cmd,
                               unsigned long arg);
int security_mmap_file(struct file *file, unsigned long prot,
                        unsigned long flags);
int security_mmap_addr(unsigned long addr);
int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
                           unsigned long prot);
int security_file_lock(struct file *file, unsigned int cmd);
int security_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg);
void security_file_set_fowner(struct file *file);
int security_file_send_sigiotask(struct task_struct *tsk,
                                 struct fown_struct *fown, int sig);
int security_file_receive(struct file *file);
int security_file_open(struct file *file);
int security_file_post_open(struct file *file, int mask);
int security_file_truncate(struct file *file);
int security_task_alloc(struct task_struct *task, unsigned long clone_flags);
void security_task_free(struct task_struct *task);
int security_cred_alloc_blank(struct cred *cred, gfp_t gfp);
void security_cred_free(struct cred *cred);
int security_prepare_creds(struct cred *new, const struct cred *old, gfp_t gfp);
void security_transfer_creds(struct cred *new, const struct cred *old);
void security_cred_getsecid(const struct cred *c, u32 *secid);
void security_cred_getlsmprop(const struct cred *c, struct lsm_prop *prop);
int security_kernel_act_as(struct cred *new, u32 secid);
int security_kernel_create_files_as(struct cred *new, struct inode *inode);
int security_kernel_module_request(char *kmod_name);
int security_kernel_load_data(enum kernel_load_data_id id, bool contents);
int security_kernel_post_load_data(char *buf, loff_t size,
                                   enum kernel_load_data_id id,
                                   char *description);
int security_kernel_read_file(struct file *file, enum kernel_read_file_id id,
                              bool contents);
int security_kernel_post_read_file(struct file *file, char *buf, loff_t size,
                                   enum kernel_read_file_id id);
int security_task_fix_setuid(struct cred *new, const struct cred *old,
                             int flags);
int security_task_fix_setgid(struct cred *new, const struct cred *old,
                             int flags);
int security_task_fix_setgroups(struct cred *new, const struct cred *old);
int security_task_setpgid(struct task_struct *p, pid_t pgid);
int security_task_getpgid(struct task_struct *p);
int security_task_getsid(struct task_struct *p);
void security_current_getlsmprop_subj(struct lsm_prop *prop);
void security_task_getlsmprop_obj(struct task_struct *p, struct lsm_prop *prop);
int security_task_setnice(struct task_struct *p, int nice);
int security_task_setioprio(struct task_struct *p, int ioprio);
int security_task_getioprio(struct task_struct *p);
int security_task_prlimit(const struct cred *cred, const struct cred *tcred,
                          unsigned int flags);
int security_task_setrlimit(struct task_struct *p, unsigned int resource,
                struct rlimit *new_rlim);
int security_task_setscheduler(struct task_struct *p);
int security_task_getscheduler(struct task_struct *p);
int security_task_movememory(struct task_struct *p);
int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
                        int sig, const struct cred *cred);
int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
                        unsigned long arg4, unsigned long arg5);
void security_task_to_inode(struct task_struct *p, struct inode *inode);
int security_create_user_ns(const struct cred *cred);
int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag);
void security_ipc_getlsmprop(struct kern_ipc_perm *ipcp, struct lsm_prop *prop);
int security_msg_msg_alloc(struct msg_msg *msg);
void security_msg_msg_free(struct msg_msg *msg);
int security_msg_queue_alloc(struct kern_ipc_perm *msq);
void security_msg_queue_free(struct kern_ipc_perm *msq);
int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg);
int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd);
int security_msg_queue_msgsnd(struct kern_ipc_perm *msq,
                              struct msg_msg *msg, int msqflg);
int security_msg_queue_msgrcv(struct kern_ipc_perm *msq, struct msg_msg *msg,
                              struct task_struct *target, long type, int mode);
int security_shm_alloc(struct kern_ipc_perm *shp);
void security_shm_free(struct kern_ipc_perm *shp);
int security_shm_associate(struct kern_ipc_perm *shp, int shmflg);
int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd);
int security_shm_shmat(struct kern_ipc_perm *shp, char __user *shmaddr, int shmflg);
int security_sem_alloc(struct kern_ipc_perm *sma);
void security_sem_free(struct kern_ipc_perm *sma);
int security_sem_associate(struct kern_ipc_perm *sma, int semflg);
int security_sem_semctl(struct kern_ipc_perm *sma, int cmd);
int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops,
                        unsigned nsops, int alter);
void security_d_instantiate(struct dentry *dentry, struct inode *inode);
int security_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
                         u32 __user *size, u32 flags);
int security_setselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
                         u32 size, u32 flags);
int security_getprocattr(struct task_struct *p, int lsmid, const char *name,
                         char **value);
int security_setprocattr(int lsmid, const char *name, void *value, size_t size);
int security_netlink_send(struct sock *sk, struct sk_buff *skb);
int security_ismaclabel(const char *name);
int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen);
int security_lsmprop_to_secctx(struct lsm_prop *prop, char **secdata, u32 *seclen);
int security_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid);
void security_release_secctx(char *secdata, u32 seclen);
void security_inode_invalidate_secctx(struct inode *inode);
int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen);
int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen);
int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen);
int security_locked_down(enum lockdown_reason what);
int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, u32 *uctx_len,
                      void *val, size_t val_len, u64 id, u64 flags);
int security_bdev_alloc(struct block_device *bdev);
void security_bdev_free(struct block_device *bdev);
int security_bdev_setintegrity(struct block_device *bdev,
                               enum lsm_integrity_type type, const void *value,
                               size_t size);
#else /* CONFIG_SECURITY */

/**
 * lsmprop_is_set - report if there is a value in the lsm_prop
 * @prop: Pointer to the exported LSM data
 *
 * Returns true if there is a value set, false otherwise
 */
static inline bool lsmprop_is_set(struct lsm_prop *prop)
{
        return false;
}

static inline int call_blocking_lsm_notifier(enum lsm_event event, void *data)
{
        return 0;
}

static inline int register_blocking_lsm_notifier(struct notifier_block *nb)
{
        return 0;
}

static inline  int unregister_blocking_lsm_notifier(struct notifier_block *nb)
{
        return 0;
}

static inline u64 lsm_name_to_attr(const char *name)
{
        return LSM_ATTR_UNDEF;
}

static inline void security_free_mnt_opts(void **mnt_opts)
{
}

/*
 * This is the default capabilities functionality.  Most of these functions
 * are just stubbed out, but a few must call the proper capable code.
 */

static inline int security_init(void)
{
        return 0;
}

static inline int early_security_init(void)
{
        return 0;
}

static inline int security_binder_set_context_mgr(const struct cred *mgr)
{
        return 0;
}

static inline int security_binder_transaction(const struct cred *from,
                                              const struct cred *to)
{
        return 0;
}

static inline int security_binder_transfer_binder(const struct cred *from,
                                                  const struct cred *to)
{
        return 0;
}

static inline int security_binder_transfer_file(const struct cred *from,
                                                const struct cred *to,
                                                const struct file *file)
{
        return 0;
}

static inline int security_ptrace_access_check(struct task_struct *child,
                                             unsigned int mode)
{
        return cap_ptrace_access_check(child, mode);
}

static inline int security_ptrace_traceme(struct task_struct *parent)
{
        return cap_ptrace_traceme(parent);
}

static inline int security_capget(const struct task_struct *target,
                                   kernel_cap_t *effective,
                                   kernel_cap_t *inheritable,
                                   kernel_cap_t *permitted)
{
        return cap_capget(target, effective, inheritable, permitted);
}

static inline int security_capset(struct cred *new,
                                   const struct cred *old,
                                   const kernel_cap_t *effective,
                                   const kernel_cap_t *inheritable,
                                   const kernel_cap_t *permitted)
{
        return cap_capset(new, old, effective, inheritable, permitted);
}

static inline int security_capable(const struct cred *cred,
                                   struct user_namespace *ns,
                                   int cap,
                                   unsigned int opts)
{
        return cap_capable(cred, ns, cap, opts);
}

static inline int security_quotactl(int cmds, int type, int id,
                                     const struct super_block *sb)
{
        return 0;
}

static inline int security_quota_on(struct dentry *dentry)
{
        return 0;
}

static inline int security_syslog(int type)
{
        return 0;
}

static inline int security_settime64(const struct timespec64 *ts,
                                     const struct timezone *tz)
{
        return cap_settime(ts, tz);
}

static inline int security_vm_enough_memory_mm(struct mm_struct *mm, long pages)
{
        return __vm_enough_memory(mm, pages, !cap_vm_enough_memory(mm, pages));
}

static inline int security_bprm_creds_for_exec(struct linux_binprm *bprm)
{
        return 0;
}

static inline int security_bprm_creds_from_file(struct linux_binprm *bprm,
                                                const struct file *file)
{
        return cap_bprm_creds_from_file(bprm, file);
}

static inline int security_bprm_check(struct linux_binprm *bprm)
{
        return 0;
}

static inline void security_bprm_committing_creds(const struct linux_binprm *bprm)
{
}

static inline void security_bprm_committed_creds(const struct linux_binprm *bprm)
{
}

static inline int security_fs_context_submount(struct fs_context *fc,
                                           struct super_block *reference)
{
        return 0;
}
static inline int security_fs_context_dup(struct fs_context *fc,
                                          struct fs_context *src_fc)
{
        return 0;
}
static inline int security_fs_context_parse_param(struct fs_context *fc,
                                                  struct fs_parameter *param)
{
        return -ENOPARAM;
}

static inline int security_sb_alloc(struct super_block *sb)
{
        return 0;
}

static inline void security_sb_delete(struct super_block *sb)
{ }

static inline void security_sb_free(struct super_block *sb)
{ }

static inline int security_sb_eat_lsm_opts(char *options,
                                           void **mnt_opts)
{
        return 0;
}

static inline int security_sb_remount(struct super_block *sb,
                                      void *mnt_opts)
{
        return 0;
}

static inline int security_sb_mnt_opts_compat(struct super_block *sb,
                                              void *mnt_opts)
{
        return 0;
}


static inline int security_sb_kern_mount(struct super_block *sb)
{
        return 0;
}

static inline int security_sb_show_options(struct seq_file *m,
                                           struct super_block *sb)
{
        return 0;
}

static inline int security_sb_statfs(struct dentry *dentry)
{
        return 0;
}

static inline int security_sb_mount(const char *dev_name, const struct path *path,
                                    const char *type, unsigned long flags,
                                    void *data)
{
        return 0;
}

static inline int security_sb_umount(struct vfsmount *mnt, int flags)
{
        return 0;
}

static inline int security_sb_pivotroot(const struct path *old_path,
                                        const struct path *new_path)
{
        return 0;
}

static inline int security_sb_set_mnt_opts(struct super_block *sb,
                                           void *mnt_opts,
                                           unsigned long kern_flags,
                                           unsigned long *set_kern_flags)
{
        return 0;
}

static inline int security_sb_clone_mnt_opts(const struct super_block *oldsb,
                                              struct super_block *newsb,
                                              unsigned long kern_flags,
                                              unsigned long *set_kern_flags)
{
        return 0;
}

static inline int security_move_mount(const struct path *from_path,
                                      const struct path *to_path)
{
        return 0;
}

static inline int security_path_notify(const struct path *path, u64 mask,
                                unsigned int obj_type)
{
        return 0;
}

static inline int security_inode_alloc(struct inode *inode, gfp_t gfp)
{
        return 0;
}

static inline void security_inode_free(struct inode *inode)
{ }

static inline int security_dentry_init_security(struct dentry *dentry,
                                                 int mode,
                                                 const struct qstr *name,
                                                 const char **xattr_name,
                                                 void **ctx,
                                                 u32 *ctxlen)
{
        return -EOPNOTSUPP;
}

static inline int security_dentry_create_files_as(struct dentry *dentry,
                                                  int mode, struct qstr *name,
                                                  const struct cred *old,
                                                  struct cred *new)
{
        return 0;
}


static inline int security_inode_init_security(struct inode *inode,
                                                struct inode *dir,
                                                const struct qstr *qstr,
                                                const initxattrs xattrs,
                                                void *fs_data)
{
        return 0;
}

static inline int security_inode_init_security_anon(struct inode *inode,
                                                    const struct qstr *name,
                                                    const struct inode *context_inode)
{
        return 0;
}

static inline int security_inode_create(struct inode *dir,
                                         struct dentry *dentry,
                                         umode_t mode)
{
        return 0;
}

static inline void
security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode)
{ }

static inline int security_inode_link(struct dentry *old_dentry,
                                       struct inode *dir,
                                       struct dentry *new_dentry)
{
        return 0;
}

static inline int security_inode_unlink(struct inode *dir,
                                         struct dentry *dentry)
{
        return 0;
}

static inline int security_inode_symlink(struct inode *dir,
                                          struct dentry *dentry,
                                          const char *old_name)
{
        return 0;
}

static inline int security_inode_mkdir(struct inode *dir,
                                        struct dentry *dentry,
                                        int mode)
{
        return 0;
}

static inline int security_inode_rmdir(struct inode *dir,
                                        struct dentry *dentry)
{
        return 0;
}

static inline int security_inode_mknod(struct inode *dir,
                                        struct dentry *dentry,
                                        int mode, dev_t dev)
{
        return 0;
}

static inline int security_inode_rename(struct inode *old_dir,
                                         struct dentry *old_dentry,
                                         struct inode *new_dir,
                                         struct dentry *new_dentry,
                                         unsigned int flags)
{
        return 0;
}

static inline int security_inode_readlink(struct dentry *dentry)
{
        return 0;
}

static inline int security_inode_follow_link(struct dentry *dentry,
                                             struct inode *inode,
                                             bool rcu)
{
        return 0;
}

static inline int security_inode_permission(struct inode *inode, int mask)
{
        return 0;
}

static inline int security_inode_setattr(struct mnt_idmap *idmap,
                                         struct dentry *dentry,
                                         struct iattr *attr)
{
        return 0;
}

static inline void
security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                            int ia_valid)
{ }

static inline int security_inode_getattr(const struct path *path)
{
        return 0;
}

static inline int security_inode_setxattr(struct mnt_idmap *idmap,
                struct dentry *dentry, const char *name, const void *value,
                size_t size, int flags)
{
        return cap_inode_setxattr(dentry, name, value, size, flags);
}

static inline int security_inode_set_acl(struct mnt_idmap *idmap,
                                         struct dentry *dentry,
                                         const char *acl_name,
                                         struct posix_acl *kacl)
{
        return 0;
}

static inline void security_inode_post_set_acl(struct dentry *dentry,
                                               const char *acl_name,
                                               struct posix_acl *kacl)
{ }

static inline int security_inode_get_acl(struct mnt_idmap *idmap,
                                         struct dentry *dentry,
                                         const char *acl_name)
{
        return 0;
}

static inline int security_inode_remove_acl(struct mnt_idmap *idmap,
                                            struct dentry *dentry,
                                            const char *acl_name)
{
        return 0;
}

static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap,
                                                  struct dentry *dentry,
                                                  const char *acl_name)
{ }

static inline void security_inode_post_setxattr(struct dentry *dentry,
                const char *name, const void *value, size_t size, int flags)
{ }

static inline int security_inode_getxattr(struct dentry *dentry,
                        const char *name)
{
        return 0;
}

static inline int security_inode_listxattr(struct dentry *dentry)
{
        return 0;
}

static inline int security_inode_removexattr(struct mnt_idmap *idmap,
                                             struct dentry *dentry,
                                             const char *name)
{
        return cap_inode_removexattr(idmap, dentry, name);
}

static inline void security_inode_post_removexattr(struct dentry *dentry,
                                                   const char *name)
{ }

static inline int security_inode_need_killpriv(struct dentry *dentry)
{
        return cap_inode_need_killpriv(dentry);
}

static inline int security_inode_killpriv(struct mnt_idmap *idmap,
                                          struct dentry *dentry)
{
        return cap_inode_killpriv(idmap, dentry);
}

static inline int security_inode_getsecurity(struct mnt_idmap *idmap,
                                             struct inode *inode,
                                             const char *name, void **buffer,
                                             bool alloc)
{
        return cap_inode_getsecurity(idmap, inode, name, buffer, alloc);
}

static inline int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags)
{
        return -EOPNOTSUPP;
}

static inline int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size)
{
        return 0;
}

static inline void security_inode_getlsmprop(struct inode *inode,
                                             struct lsm_prop *prop)
{
        lsmprop_init(prop);
}

static inline int security_inode_copy_up(struct dentry *src, struct cred **new)
{
        return 0;
}

static inline int security_inode_setintegrity(const struct inode *inode,
                                              enum lsm_integrity_type type,
                                              const void *value, size_t size)
{
        return 0;
}

static inline int security_kernfs_init_security(struct kernfs_node *kn_dir,
                                                struct kernfs_node *kn)
{
        return 0;
}

static inline int security_inode_copy_up_xattr(struct dentry *src, const char *name)
{
        return -EOPNOTSUPP;
}

static inline int security_file_permission(struct file *file, int mask)
{
        return 0;
}

static inline int security_file_alloc(struct file *file)
{
        return 0;
}

static inline void security_file_release(struct file *file)
{ }

static inline void security_file_free(struct file *file)
{ }

static inline int security_file_ioctl(struct file *file, unsigned int cmd,
                                      unsigned long arg)
{
        return 0;
}

static inline int security_file_ioctl_compat(struct file *file,
                                             unsigned int cmd,
                                             unsigned long arg)
{
        return 0;
}

static inline int security_mmap_file(struct file *file, unsigned long prot,
                                     unsigned long flags)
{
        return 0;
}

static inline int security_mmap_addr(unsigned long addr)
{
        return cap_mmap_addr(addr);
}

static inline int security_file_mprotect(struct vm_area_struct *vma,
                                         unsigned long reqprot,
                                         unsigned long prot)
{
        return 0;
}

static inline int security_file_lock(struct file *file, unsigned int cmd)
{
        return 0;
}

static inline int security_file_fcntl(struct file *file, unsigned int cmd,
                                      unsigned long arg)
{
        return 0;
}

static inline void security_file_set_fowner(struct file *file)
{
        return;
}

static inline int security_file_send_sigiotask(struct task_struct *tsk,
                                               struct fown_struct *fown,
                                               int sig)
{
        return 0;
}

static inline int security_file_receive(struct file *file)
{
        return 0;
}

static inline int security_file_open(struct file *file)
{
        return 0;
}

static inline int security_file_post_open(struct file *file, int mask)
{
        return 0;
}

static inline int security_file_truncate(struct file *file)
{
        return 0;
}

static inline int security_task_alloc(struct task_struct *task,
                                      unsigned long clone_flags)
{
        return 0;
}

static inline void security_task_free(struct task_struct *task)
{ }

static inline int security_cred_alloc_blank(struct cred *cred, gfp_t gfp)
{
        return 0;
}

static inline void security_cred_free(struct cred *cred)
{ }

static inline int security_prepare_creds(struct cred *new,
                                         const struct cred *old,
                                         gfp_t gfp)
{
        return 0;
}

static inline void security_transfer_creds(struct cred *new,
                                           const struct cred *old)
{
}

static inline void security_cred_getsecid(const struct cred *c, u32 *secid)
{
        *secid = 0;
}

static inline void security_cred_getlsmprop(const struct cred *c,
                                            struct lsm_prop *prop)
{ }

static inline int security_kernel_act_as(struct cred *cred, u32 secid)
{
        return 0;
}

static inline int security_kernel_create_files_as(struct cred *cred,
                                                  struct inode *inode)
{
        return 0;
}

static inline int security_kernel_module_request(char *kmod_name)
{
        return 0;
}

static inline int security_kernel_load_data(enum kernel_load_data_id id, bool contents)
{
        return 0;
}

static inline int security_kernel_post_load_data(char *buf, loff_t size,
                                                 enum kernel_load_data_id id,
                                                 char *description)
{
        return 0;
}

static inline int security_kernel_read_file(struct file *file,
                                            enum kernel_read_file_id id,
                                            bool contents)
{
        return 0;
}

static inline int security_kernel_post_read_file(struct file *file,
                                                 char *buf, loff_t size,
                                                 enum kernel_read_file_id id)
{
        return 0;
}

static inline int security_task_fix_setuid(struct cred *new,
                                           const struct cred *old,
                                           int flags)
{
        return cap_task_fix_setuid(new, old, flags);
}

static inline int security_task_fix_setgid(struct cred *new,
                                           const struct cred *old,
                                           int flags)
{
        return 0;
}

static inline int security_task_fix_setgroups(struct cred *new,
                                           const struct cred *old)
{
        return 0;
}

static inline int security_task_setpgid(struct task_struct *p, pid_t pgid)
{
        return 0;
}

static inline int security_task_getpgid(struct task_struct *p)
{
        return 0;
}

static inline int security_task_getsid(struct task_struct *p)
{
        return 0;
}

static inline void security_current_getlsmprop_subj(struct lsm_prop *prop)
{
        lsmprop_init(prop);
}

static inline void security_task_getlsmprop_obj(struct task_struct *p,
                                                struct lsm_prop *prop)
{
        lsmprop_init(prop);
}

static inline int security_task_setnice(struct task_struct *p, int nice)
{
        return cap_task_setnice(p, nice);
}

static inline int security_task_setioprio(struct task_struct *p, int ioprio)
{
        return cap_task_setioprio(p, ioprio);
}

static inline int security_task_getioprio(struct task_struct *p)
{
        return 0;
}

static inline int security_task_prlimit(const struct cred *cred,
                                        const struct cred *tcred,
                                        unsigned int flags)
{
        return 0;
}

static inline int security_task_setrlimit(struct task_struct *p,
                                          unsigned int resource,
                                          struct rlimit *new_rlim)
{
        return 0;
}

static inline int security_task_setscheduler(struct task_struct *p)
{
        return cap_task_setscheduler(p);
}

static inline int security_task_getscheduler(struct task_struct *p)
{
        return 0;
}

static inline int security_task_movememory(struct task_struct *p)
{
        return 0;
}

static inline int security_task_kill(struct task_struct *p,
                                     struct kernel_siginfo *info, int sig,
                                     const struct cred *cred)
{
        return 0;
}

static inline int security_task_prctl(int option, unsigned long arg2,
                                      unsigned long arg3,
                                      unsigned long arg4,
                                      unsigned long arg5)
{
        return cap_task_prctl(option, arg2, arg3, arg4, arg5);
}

static inline void security_task_to_inode(struct task_struct *p, struct inode *inode)
{ }

static inline int security_create_user_ns(const struct cred *cred)
{
        return 0;
}

static inline int security_ipc_permission(struct kern_ipc_perm *ipcp,
                                          short flag)
{
        return 0;
}

static inline void security_ipc_getlsmprop(struct kern_ipc_perm *ipcp,
                                           struct lsm_prop *prop)
{
        lsmprop_init(prop);
}

static inline int security_msg_msg_alloc(struct msg_msg *msg)
{
        return 0;
}

static inline void security_msg_msg_free(struct msg_msg *msg)
{ }

static inline int security_msg_queue_alloc(struct kern_ipc_perm *msq)
{
        return 0;
}

static inline void security_msg_queue_free(struct kern_ipc_perm *msq)
{ }

static inline int security_msg_queue_associate(struct kern_ipc_perm *msq,
                                               int msqflg)
{
        return 0;
}

static inline int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd)
{
        return 0;
}

static inline int security_msg_queue_msgsnd(struct kern_ipc_perm *msq,
                                            struct msg_msg *msg, int msqflg)
{
        return 0;
}

static inline int security_msg_queue_msgrcv(struct kern_ipc_perm *msq,
                                            struct msg_msg *msg,
                                            struct task_struct *target,
                                            long type, int mode)
{
        return 0;
}

static inline int security_shm_alloc(struct kern_ipc_perm *shp)
{
        return 0;
}

static inline void security_shm_free(struct kern_ipc_perm *shp)
{ }

static inline int security_shm_associate(struct kern_ipc_perm *shp,
                                         int shmflg)
{
        return 0;
}

static inline int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd)
{
        return 0;
}

static inline int security_shm_shmat(struct kern_ipc_perm *shp,
                                     char __user *shmaddr, int shmflg)
{
        return 0;
}

static inline int security_sem_alloc(struct kern_ipc_perm *sma)
{
        return 0;
}

static inline void security_sem_free(struct kern_ipc_perm *sma)
{ }

static inline int security_sem_associate(struct kern_ipc_perm *sma, int semflg)
{
        return 0;
}

static inline int security_sem_semctl(struct kern_ipc_perm *sma, int cmd)
{
        return 0;
}

static inline int security_sem_semop(struct kern_ipc_perm *sma,
                                     struct sembuf *sops, unsigned nsops,
                                     int alter)
{
        return 0;
}

static inline void security_d_instantiate(struct dentry *dentry,
                                          struct inode *inode)
{ }

static inline int security_getselfattr(unsigned int attr,
                                       struct lsm_ctx __user *ctx,
                                       size_t __user *size, u32 flags)
{
        return -EOPNOTSUPP;
}

static inline int security_setselfattr(unsigned int attr,
                                       struct lsm_ctx __user *ctx,
                                       size_t size, u32 flags)
{
        return -EOPNOTSUPP;
}

static inline int security_getprocattr(struct task_struct *p, int lsmid,
                                       const char *name, char **value)
{
        return -EINVAL;
}

static inline int security_setprocattr(int lsmid, char *name, void *value,
                                       size_t size)
{
        return -EINVAL;
}

static inline int security_netlink_send(struct sock *sk, struct sk_buff *skb)
{
        return 0;
}

static inline int security_ismaclabel(const char *name)
{
        return 0;
}

static inline int security_secid_to_secctx(u32 secid, char **secdata,
                                           u32 *seclen)
{
        return -EOPNOTSUPP;
}

static inline int security_lsmprop_to_secctx(struct lsm_prop *prop,
                                             char **secdata, u32 *seclen)
{
        return -EOPNOTSUPP;
}

static inline int security_secctx_to_secid(const char *secdata,
                                           u32 seclen,
                                           u32 *secid)
{
        return -EOPNOTSUPP;
}

static inline void security_release_secctx(char *secdata, u32 seclen)
{
}

static inline void security_inode_invalidate_secctx(struct inode *inode)
{
}

static inline int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen)
{
        return -EOPNOTSUPP;
}
static inline int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen)
{
        return -EOPNOTSUPP;
}
static inline int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen)
{
        return -EOPNOTSUPP;
}
static inline int security_locked_down(enum lockdown_reason what)
{
        return 0;
}
static inline int lsm_fill_user_ctx(struct lsm_ctx __user *uctx,
                                    u32 *uctx_len, void *val, size_t val_len,
                                    u64 id, u64 flags)
{
        return -EOPNOTSUPP;
}

static inline int security_bdev_alloc(struct block_device *bdev)
{
        return 0;
}

static inline void security_bdev_free(struct block_device *bdev)
{
}

static inline int security_bdev_setintegrity(struct block_device *bdev,
                                             enum lsm_integrity_type type,
                                             const void *value, size_t size)
{
        return 0;
}

#endif        /* CONFIG_SECURITY */

#if defined(CONFIG_SECURITY) && defined(CONFIG_WATCH_QUEUE)
int security_post_notification(const struct cred *w_cred,
                               const struct cred *cred,
                               struct watch_notification *n);
#else
static inline int security_post_notification(const struct cred *w_cred,
                                             const struct cred *cred,
                                             struct watch_notification *n)
{
        return 0;
}
#endif

#if defined(CONFIG_SECURITY) && defined(CONFIG_KEY_NOTIFICATIONS)
int security_watch_key(struct key *key);
#else
static inline int security_watch_key(struct key *key)
{
        return 0;
}
#endif

#ifdef CONFIG_SECURITY_NETWORK

int security_unix_stream_connect(struct sock *sock, struct sock *other, struct sock *newsk);
int security_unix_may_send(struct socket *sock,  struct socket *other);
int security_socket_create(int family, int type, int protocol, int kern);
int security_socket_post_create(struct socket *sock, int family,
                                int type, int protocol, int kern);
int security_socket_socketpair(struct socket *socka, struct socket *sockb);
int security_socket_bind(struct socket *sock, struct sockaddr *address, int addrlen);
int security_socket_connect(struct socket *sock, struct sockaddr *address, int addrlen);
int security_socket_listen(struct socket *sock, int backlog);
int security_socket_accept(struct socket *sock, struct socket *newsock);
int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size);
int security_socket_recvmsg(struct socket *sock, struct msghdr *msg,
                            int size, int flags);
int security_socket_getsockname(struct socket *sock);
int security_socket_getpeername(struct socket *sock);
int security_socket_getsockopt(struct socket *sock, int level, int optname);
int security_socket_setsockopt(struct socket *sock, int level, int optname);
int security_socket_shutdown(struct socket *sock, int how);
int security_sock_rcv_skb(struct sock *sk, struct sk_buff *skb);
int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval,
                                      sockptr_t optlen, unsigned int len);
int security_socket_getpeersec_dgram(struct socket *sock, struct sk_buff *skb, u32 *secid);
int security_sk_alloc(struct sock *sk, int family, gfp_t priority);
void security_sk_free(struct sock *sk);
void security_sk_clone(const struct sock *sk, struct sock *newsk);
void security_sk_classify_flow(const struct sock *sk,
                               struct flowi_common *flic);
void security_req_classify_flow(const struct request_sock *req,
                                struct flowi_common *flic);
void security_sock_graft(struct sock*sk, struct socket *parent);
int security_inet_conn_request(const struct sock *sk,
                        struct sk_buff *skb, struct request_sock *req);
void security_inet_csk_clone(struct sock *newsk,
                        const struct request_sock *req);
void security_inet_conn_established(struct sock *sk,
                        struct sk_buff *skb);
int security_secmark_relabel_packet(u32 secid);
void security_secmark_refcount_inc(void);
void security_secmark_refcount_dec(void);
int security_tun_dev_alloc_security(void **security);
void security_tun_dev_free_security(void *security);
int security_tun_dev_create(void);
int security_tun_dev_attach_queue(void *security);
int security_tun_dev_attach(struct sock *sk, void *security);
int security_tun_dev_open(void *security);
int security_sctp_assoc_request(struct sctp_association *asoc, struct sk_buff *skb);
int security_sctp_bind_connect(struct sock *sk, int optname,
                               struct sockaddr *address, int addrlen);
void security_sctp_sk_clone(struct sctp_association *asoc, struct sock *sk,
                            struct sock *newsk);
int security_sctp_assoc_established(struct sctp_association *asoc,
                                    struct sk_buff *skb);
int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk);

#else        /* CONFIG_SECURITY_NETWORK */
static inline int security_unix_stream_connect(struct sock *sock,
                                               struct sock *other,
                                               struct sock *newsk)
{
        return 0;
}

static inline int security_unix_may_send(struct socket *sock,
                                         struct socket *other)
{
        return 0;
}

static inline int security_socket_create(int family, int type,
                                         int protocol, int kern)
{
        return 0;
}

static inline int security_socket_post_create(struct socket *sock,
                                              int family,
                                              int type,
                                              int protocol, int kern)
{
        return 0;
}

static inline int security_socket_socketpair(struct socket *socka,
                                             struct socket *sockb)
{
        return 0;
}

static inline int security_socket_bind(struct socket *sock,
                                       struct sockaddr *address,
                                       int addrlen)
{
        return 0;
}

static inline int security_socket_connect(struct socket *sock,
                                          struct sockaddr *address,
                                          int addrlen)
{
        return 0;
}

static inline int security_socket_listen(struct socket *sock, int backlog)
{
        return 0;
}

static inline int security_socket_accept(struct socket *sock,
                                         struct socket *newsock)
{
        return 0;
}

static inline int security_socket_sendmsg(struct socket *sock,
                                          struct msghdr *msg, int size)
{
        return 0;
}

static inline int security_socket_recvmsg(struct socket *sock,
                                          struct msghdr *msg, int size,
                                          int flags)
{
        return 0;
}

static inline int security_socket_getsockname(struct socket *sock)
{
        return 0;
}

static inline int security_socket_getpeername(struct socket *sock)
{
        return 0;
}

static inline int security_socket_getsockopt(struct socket *sock,
                                             int level, int optname)
{
        return 0;
}

static inline int security_socket_setsockopt(struct socket *sock,
                                             int level, int optname)
{
        return 0;
}

static inline int security_socket_shutdown(struct socket *sock, int how)
{
        return 0;
}
static inline int security_sock_rcv_skb(struct sock *sk,
                                        struct sk_buff *skb)
{
        return 0;
}

static inline int security_socket_getpeersec_stream(struct socket *sock,
                                                    sockptr_t optval,
                                                    sockptr_t optlen,
                                                    unsigned int len)
{
        return -ENOPROTOOPT;
}

static inline int security_socket_getpeersec_dgram(struct socket *sock, struct sk_buff *skb, u32 *secid)
{
        return -ENOPROTOOPT;
}

static inline int security_sk_alloc(struct sock *sk, int family, gfp_t priority)
{
        return 0;
}

static inline void security_sk_free(struct sock *sk)
{
}

static inline void security_sk_clone(const struct sock *sk, struct sock *newsk)
{
}

static inline void security_sk_classify_flow(const struct sock *sk,
                                             struct flowi_common *flic)
{
}

static inline void security_req_classify_flow(const struct request_sock *req,
                                              struct flowi_common *flic)
{
}

static inline void security_sock_graft(struct sock *sk, struct socket *parent)
{
}

static inline int security_inet_conn_request(const struct sock *sk,
                        struct sk_buff *skb, struct request_sock *req)
{
        return 0;
}

static inline void security_inet_csk_clone(struct sock *newsk,
                        const struct request_sock *req)
{
}

static inline void security_inet_conn_established(struct sock *sk,
                        struct sk_buff *skb)
{
}

static inline int security_secmark_relabel_packet(u32 secid)
{
        return 0;
}

static inline void security_secmark_refcount_inc(void)
{
}

static inline void security_secmark_refcount_dec(void)
{
}

static inline int security_tun_dev_alloc_security(void **security)
{
        return 0;
}

static inline void security_tun_dev_free_security(void *security)
{
}

static inline int security_tun_dev_create(void)
{
        return 0;
}

static inline int security_tun_dev_attach_queue(void *security)
{
        return 0;
}

static inline int security_tun_dev_attach(struct sock *sk, void *security)
{
        return 0;
}

static inline int security_tun_dev_open(void *security)
{
        return 0;
}

static inline int security_sctp_assoc_request(struct sctp_association *asoc,
                                              struct sk_buff *skb)
{
        return 0;
}

static inline int security_sctp_bind_connect(struct sock *sk, int optname,
                                             struct sockaddr *address,
                                             int addrlen)
{
        return 0;
}

static inline void security_sctp_sk_clone(struct sctp_association *asoc,
                                          struct sock *sk,
                                          struct sock *newsk)
{
}

static inline int security_sctp_assoc_established(struct sctp_association *asoc,
                                                  struct sk_buff *skb)
{
        return 0;
}

static inline int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk)
{
        return 0;
}
#endif        /* CONFIG_SECURITY_NETWORK */

#ifdef CONFIG_SECURITY_INFINIBAND
int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey);
int security_ib_endport_manage_subnet(void *sec, const char *name, u8 port_num);
int security_ib_alloc_security(void **sec);
void security_ib_free_security(void *sec);
#else        /* CONFIG_SECURITY_INFINIBAND */
static inline int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey)
{
        return 0;
}

static inline int security_ib_endport_manage_subnet(void *sec, const char *dev_name, u8 port_num)
{
        return 0;
}

static inline int security_ib_alloc_security(void **sec)
{
        return 0;
}

static inline void security_ib_free_security(void *sec)
{
}
#endif        /* CONFIG_SECURITY_INFINIBAND */

#ifdef CONFIG_SECURITY_NETWORK_XFRM

int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp,
                               struct xfrm_user_sec_ctx *sec_ctx, gfp_t gfp);
int security_xfrm_policy_clone(struct xfrm_sec_ctx *old_ctx, struct xfrm_sec_ctx **new_ctxp);
void security_xfrm_policy_free(struct xfrm_sec_ctx *ctx);
int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx);
int security_xfrm_state_alloc(struct xfrm_state *x, struct xfrm_user_sec_ctx *sec_ctx);
int security_xfrm_state_alloc_acquire(struct xfrm_state *x,
                                      struct xfrm_sec_ctx *polsec, u32 secid);
int security_xfrm_state_delete(struct xfrm_state *x);
void security_xfrm_state_free(struct xfrm_state *x);
int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid);
int security_xfrm_state_pol_flow_match(struct xfrm_state *x,
                                       struct xfrm_policy *xp,
                                       const struct flowi_common *flic);
int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid);
void security_skb_classify_flow(struct sk_buff *skb, struct flowi_common *flic);

#else        /* CONFIG_SECURITY_NETWORK_XFRM */

static inline int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp,
                                             struct xfrm_user_sec_ctx *sec_ctx,
                                             gfp_t gfp)
{
        return 0;
}

static inline int security_xfrm_policy_clone(struct xfrm_sec_ctx *old, struct xfrm_sec_ctx **new_ctxp)
{
        return 0;
}

static inline void security_xfrm_policy_free(struct xfrm_sec_ctx *ctx)
{
}

static inline int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx)
{
        return 0;
}

static inline int security_xfrm_state_alloc(struct xfrm_state *x,
                                        struct xfrm_user_sec_ctx *sec_ctx)
{
        return 0;
}

static inline int security_xfrm_state_alloc_acquire(struct xfrm_state *x,
                                        struct xfrm_sec_ctx *polsec, u32 secid)
{
        return 0;
}

static inline void security_xfrm_state_free(struct xfrm_state *x)
{
}

static inline int security_xfrm_state_delete(struct xfrm_state *x)
{
        return 0;
}

static inline int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid)
{
        return 0;
}

static inline int security_xfrm_state_pol_flow_match(struct xfrm_state *x,
                                                     struct xfrm_policy *xp,
                                                     const struct flowi_common *flic)
{
        return 1;
}

static inline int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid)
{
        return 0;
}

static inline void security_skb_classify_flow(struct sk_buff *skb,
                                              struct flowi_common *flic)
{
}

#endif        /* CONFIG_SECURITY_NETWORK_XFRM */

#ifdef CONFIG_SECURITY_PATH
int security_path_unlink(const struct path *dir, struct dentry *dentry);
int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t mode);
int security_path_rmdir(const struct path *dir, struct dentry *dentry);
int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode,
                        unsigned int dev);
void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry);
int security_path_truncate(const struct path *path);
int security_path_symlink(const struct path *dir, struct dentry *dentry,
                          const char *old_name);
int security_path_link(struct dentry *old_dentry, const struct path *new_dir,
                       struct dentry *new_dentry);
int security_path_rename(const struct path *old_dir, struct dentry *old_dentry,
                         const struct path *new_dir, struct dentry *new_dentry,
                         unsigned int flags);
int security_path_chmod(const struct path *path, umode_t mode);
int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid);
int security_path_chroot(const struct path *path);
#else        /* CONFIG_SECURITY_PATH */
static inline int security_path_unlink(const struct path *dir, struct dentry *dentry)
{
        return 0;
}

static inline int security_path_mkdir(const struct path *dir, struct dentry *dentry,
                                      umode_t mode)
{
        return 0;
}

static inline int security_path_rmdir(const struct path *dir, struct dentry *dentry)
{
        return 0;
}

static inline int security_path_mknod(const struct path *dir, struct dentry *dentry,
                                      umode_t mode, unsigned int dev)
{
        return 0;
}

static inline void security_path_post_mknod(struct mnt_idmap *idmap,
                                            struct dentry *dentry)
{ }

static inline int security_path_truncate(const struct path *path)
{
        return 0;
}

static inline int security_path_symlink(const struct path *dir, struct dentry *dentry,
                                        const char *old_name)
{
        return 0;
}

static inline int security_path_link(struct dentry *old_dentry,
                                     const struct path *new_dir,
                                     struct dentry *new_dentry)
{
        return 0;
}

static inline int security_path_rename(const struct path *old_dir,
                                       struct dentry *old_dentry,
                                       const struct path *new_dir,
                                       struct dentry *new_dentry,
                                       unsigned int flags)
{
        return 0;
}

static inline int security_path_chmod(const struct path *path, umode_t mode)
{
        return 0;
}

static inline int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid)
{
        return 0;
}

static inline int security_path_chroot(const struct path *path)
{
        return 0;
}
#endif        /* CONFIG_SECURITY_PATH */

#ifdef CONFIG_KEYS
#ifdef CONFIG_SECURITY

int security_key_alloc(struct key *key, const struct cred *cred, unsigned long flags);
void security_key_free(struct key *key);
int security_key_permission(key_ref_t key_ref, const struct cred *cred,
                            enum key_need_perm need_perm);
int security_key_getsecurity(struct key *key, char **_buffer);
void security_key_post_create_or_update(struct key *keyring, struct key *key,
                                        const void *payload, size_t payload_len,
                                        unsigned long flags, bool create);

#else

static inline int security_key_alloc(struct key *key,
                                     const struct cred *cred,
                                     unsigned long flags)
{
        return 0;
}

static inline void security_key_free(struct key *key)
{
}

static inline int security_key_permission(key_ref_t key_ref,
                                          const struct cred *cred,
                                          enum key_need_perm need_perm)
{
        return 0;
}

static inline int security_key_getsecurity(struct key *key, char **_buffer)
{
        *_buffer = NULL;
        return 0;
}

static inline void security_key_post_create_or_update(struct key *keyring,
                                                      struct key *key,
                                                      const void *payload,
                                                      size_t payload_len,
                                                      unsigned long flags,
                                                      bool create)
{ }

#endif
#endif /* CONFIG_KEYS */

#ifdef CONFIG_AUDIT
#ifdef CONFIG_SECURITY
int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule,
                             gfp_t gfp);
int security_audit_rule_known(struct audit_krule *krule);
int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op,
                              void *lsmrule);
void security_audit_rule_free(void *lsmrule);

#else

static inline int security_audit_rule_init(u32 field, u32 op, char *rulestr,
                                           void **lsmrule, gfp_t gfp)
{
        return 0;
}

static inline int security_audit_rule_known(struct audit_krule *krule)
{
        return 0;
}

static inline int security_audit_rule_match(struct lsm_prop *prop, u32 field,
                                            u32 op, void *lsmrule)
{
        return 0;
}

static inline void security_audit_rule_free(void *lsmrule)
{ }

#endif /* CONFIG_SECURITY */
#endif /* CONFIG_AUDIT */

#ifdef CONFIG_SECURITYFS

extern struct dentry *securityfs_create_file(const char *name, umode_t mode,
                                             struct dentry *parent, void *data,
                                             const struct file_operations *fops);
extern struct dentry *securityfs_create_dir(const char *name, struct dentry *parent);
struct dentry *securityfs_create_symlink(const char *name,
                                         struct dentry *parent,
                                         const char *target,
                                         const struct inode_operations *iops);
extern void securityfs_remove(struct dentry *dentry);
extern void securityfs_recursive_remove(struct dentry *dentry);

#else /* CONFIG_SECURITYFS */

static inline struct dentry *securityfs_create_dir(const char *name,
                                                   struct dentry *parent)
{
        return ERR_PTR(-ENODEV);
}

static inline struct dentry *securityfs_create_file(const char *name,
                                                    umode_t mode,
                                                    struct dentry *parent,
                                                    void *data,
                                                    const struct file_operations *fops)
{
        return ERR_PTR(-ENODEV);
}

static inline struct dentry *securityfs_create_symlink(const char *name,
                                        struct dentry *parent,
                                        const char *target,
                                        const struct inode_operations *iops)
{
        return ERR_PTR(-ENODEV);
}

static inline void securityfs_remove(struct dentry *dentry)
{}

#endif

#ifdef CONFIG_BPF_SYSCALL
union bpf_attr;
struct bpf_map;
struct bpf_prog;
struct bpf_token;
#ifdef CONFIG_SECURITY
extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size);
extern int security_bpf_map(struct bpf_map *map, fmode_t fmode);
extern int security_bpf_prog(struct bpf_prog *prog);
extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
                                   struct bpf_token *token);
extern void security_bpf_map_free(struct bpf_map *map);
extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
                                  struct bpf_token *token);
extern void security_bpf_prog_free(struct bpf_prog *prog);
extern int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr,
                                     const struct path *path);
extern void security_bpf_token_free(struct bpf_token *token);
extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
extern int security_bpf_token_capable(const struct bpf_token *token, int cap);
#else
static inline int security_bpf(int cmd, union bpf_attr *attr,
                                             unsigned int size)
{
        return 0;
}

static inline int security_bpf_map(struct bpf_map *map, fmode_t fmode)
{
        return 0;
}

static inline int security_bpf_prog(struct bpf_prog *prog)
{
        return 0;
}

static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
                                          struct bpf_token *token)
{
        return 0;
}

static inline void security_bpf_map_free(struct bpf_map *map)
{ }

static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
                                         struct bpf_token *token)
{
        return 0;
}

static inline void security_bpf_prog_free(struct bpf_prog *prog)
{ }

static inline int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr,
                                            const struct path *path)
{
        return 0;
}

static inline void security_bpf_token_free(struct bpf_token *token)
{ }

static inline int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd)
{
        return 0;
}

static inline int security_bpf_token_capable(const struct bpf_token *token, int cap)
{
        return 0;
}
#endif /* CONFIG_SECURITY */
#endif /* CONFIG_BPF_SYSCALL */

#ifdef CONFIG_PERF_EVENTS
struct perf_event_attr;
struct perf_event;

#ifdef CONFIG_SECURITY
extern int security_perf_event_open(struct perf_event_attr *attr, int type);
extern int security_perf_event_alloc(struct perf_event *event);
extern void security_perf_event_free(struct perf_event *event);
extern int security_perf_event_read(struct perf_event *event);
extern int security_perf_event_write(struct perf_event *event);
#else
static inline int security_perf_event_open(struct perf_event_attr *attr,
                                           int type)
{
        return 0;
}

static inline int security_perf_event_alloc(struct perf_event *event)
{
        return 0;
}

static inline void security_perf_event_free(struct perf_event *event)
{
}

static inline int security_perf_event_read(struct perf_event *event)
{
        return 0;
}

static inline int security_perf_event_write(struct perf_event *event)
{
        return 0;
}
#endif /* CONFIG_SECURITY */
#endif /* CONFIG_PERF_EVENTS */

#ifdef CONFIG_IO_URING
#ifdef CONFIG_SECURITY
extern int security_uring_override_creds(const struct cred *new);
extern int security_uring_sqpoll(void);
extern int security_uring_cmd(struct io_uring_cmd *ioucmd);
#else
static inline int security_uring_override_creds(const struct cred *new)
{
        return 0;
}
static inline int security_uring_sqpoll(void)
{
        return 0;
}
static inline int security_uring_cmd(struct io_uring_cmd *ioucmd)
{
        return 0;
}
#endif /* CONFIG_SECURITY */
#endif /* CONFIG_IO_URING */

#ifdef CONFIG_SECURITY
extern void security_initramfs_populated(void);
#else
static inline void security_initramfs_populated(void)
{
}
#endif /* CONFIG_SECURITY */

#endif /* ! __LINUX_SECURITY_H */


































    1 
    1 

    1 

    1 

    1 

    1 



























































































































































































    2 

    2 





    1 











    1 







    1 
    1 















    1 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
// SPDX-License-Identifier: GPL-2.0-only

#include "netlink.h"
#include "common.h"
#include "bitset.h"

struct fec_req_info {
        struct ethnl_req_info                base;
};

struct fec_reply_data {
        struct ethnl_reply_data                base;
        __ETHTOOL_DECLARE_LINK_MODE_MASK(fec_link_modes);
        u32 active_fec;
        u8 fec_auto;
        struct fec_stat_grp {
                u64 stats[1 + ETHTOOL_MAX_LANES];
                u8 cnt;
        } corr, uncorr, corr_bits;
};

#define FEC_REPDATA(__reply_base) \
        container_of(__reply_base, struct fec_reply_data, base)

#define ETHTOOL_FEC_MASK        ((ETHTOOL_FEC_LLRS << 1) - 1)

const struct nla_policy ethnl_fec_get_policy[ETHTOOL_A_FEC_HEADER + 1] = {
        [ETHTOOL_A_FEC_HEADER]        = NLA_POLICY_NESTED(ethnl_header_policy_stats),
};

static void
ethtool_fec_to_link_modes(u32 fec, unsigned long *link_modes, u8 *fec_auto)
{
        if (fec_auto)
                *fec_auto = !!(fec & ETHTOOL_FEC_AUTO);

        if (fec & ETHTOOL_FEC_OFF)
                __set_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, link_modes);
        if (fec & ETHTOOL_FEC_RS)
                __set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, link_modes);
        if (fec & ETHTOOL_FEC_BASER)
                __set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, link_modes);
        if (fec & ETHTOOL_FEC_LLRS)
                __set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, link_modes);
}

static int
ethtool_link_modes_to_fecparam(struct ethtool_fecparam *fec,
                               unsigned long *link_modes, u8 fec_auto)
{
        memset(fec, 0, sizeof(*fec));

        if (fec_auto)
                fec->fec |= ETHTOOL_FEC_AUTO;

        if (__test_and_clear_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, link_modes))
                fec->fec |= ETHTOOL_FEC_OFF;
        if (__test_and_clear_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, link_modes))
                fec->fec |= ETHTOOL_FEC_RS;
        if (__test_and_clear_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, link_modes))
                fec->fec |= ETHTOOL_FEC_BASER;
        if (__test_and_clear_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, link_modes))
                fec->fec |= ETHTOOL_FEC_LLRS;

        if (!bitmap_empty(link_modes, __ETHTOOL_LINK_MODE_MASK_NBITS))
                return -EINVAL;

        return 0;
}

static void
fec_stats_recalc(struct fec_stat_grp *grp, struct ethtool_fec_stat *stats)
{
        int i;

        if (stats->lanes[0] == ETHTOOL_STAT_NOT_SET) {
                grp->stats[0] = stats->total;
                grp->cnt = stats->total != ETHTOOL_STAT_NOT_SET;
                return;
        }

        grp->cnt = 1;
        grp->stats[0] = 0;
        for (i = 0; i < ETHTOOL_MAX_LANES; i++) {
                if (stats->lanes[i] == ETHTOOL_STAT_NOT_SET)
                        break;

                grp->stats[0] += stats->lanes[i];
                grp->stats[grp->cnt++] = stats->lanes[i];
        }
}

static int fec_prepare_data(const struct ethnl_req_info *req_base,
                            struct ethnl_reply_data *reply_base,
                            const struct genl_info *info)
{
        __ETHTOOL_DECLARE_LINK_MODE_MASK(active_fec_modes) = {};
        struct fec_reply_data *data = FEC_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        struct ethtool_fecparam fec = {};
        int ret;

        if (!dev->ethtool_ops->get_fecparam)
                return -EOPNOTSUPP;
        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;
        ret = dev->ethtool_ops->get_fecparam(dev, &fec);
        if (ret)
                goto out_complete;
        if (req_base->flags & ETHTOOL_FLAG_STATS &&
            dev->ethtool_ops->get_fec_stats) {
                struct ethtool_fec_stats stats;

                ethtool_stats_init((u64 *)&stats, sizeof(stats) / 8);
                dev->ethtool_ops->get_fec_stats(dev, &stats);

                fec_stats_recalc(&data->corr, &stats.corrected_blocks);
                fec_stats_recalc(&data->uncorr, &stats.uncorrectable_blocks);
                fec_stats_recalc(&data->corr_bits, &stats.corrected_bits);
        }

        WARN_ON_ONCE(fec.reserved);

        ethtool_fec_to_link_modes(fec.fec, data->fec_link_modes,
                                  &data->fec_auto);

        ethtool_fec_to_link_modes(fec.active_fec, active_fec_modes, NULL);
        data->active_fec = find_first_bit(active_fec_modes,
                                          __ETHTOOL_LINK_MODE_MASK_NBITS);
        /* Don't report attr if no FEC mode set. Note that
         * ethtool_fecparam_to_link_modes() ignores NONE and AUTO.
         */
        if (data->active_fec == __ETHTOOL_LINK_MODE_MASK_NBITS)
                data->active_fec = 0;

out_complete:
        ethnl_ops_complete(dev);
        return ret;
}

static int fec_reply_size(const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        const struct fec_reply_data *data = FEC_REPDATA(reply_base);
        int len = 0;
        int ret;

        ret = ethnl_bitset_size(data->fec_link_modes, NULL,
                                __ETHTOOL_LINK_MODE_MASK_NBITS,
                                link_mode_names, compact);
        if (ret < 0)
                return ret;
        len += ret;

        len += nla_total_size(sizeof(u8)) +        /* _FEC_AUTO */
               nla_total_size(sizeof(u32));        /* _FEC_ACTIVE */

        if (req_base->flags & ETHTOOL_FLAG_STATS)
                len += 3 * nla_total_size_64bit(sizeof(u64) *
                                                (1 + ETHTOOL_MAX_LANES));

        return len;
}

static int fec_put_stats(struct sk_buff *skb, const struct fec_reply_data *data)
{
        struct nlattr *nest;

        nest = nla_nest_start(skb, ETHTOOL_A_FEC_STATS);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_64bit(skb, ETHTOOL_A_FEC_STAT_CORRECTED,
                          sizeof(u64) * data->corr.cnt,
                          data->corr.stats, ETHTOOL_A_FEC_STAT_PAD) ||
            nla_put_64bit(skb, ETHTOOL_A_FEC_STAT_UNCORR,
                          sizeof(u64) * data->uncorr.cnt,
                          data->uncorr.stats, ETHTOOL_A_FEC_STAT_PAD) ||
            nla_put_64bit(skb, ETHTOOL_A_FEC_STAT_CORR_BITS,
                          sizeof(u64) * data->corr_bits.cnt,
                          data->corr_bits.stats, ETHTOOL_A_FEC_STAT_PAD))
                goto err_cancel;

        nla_nest_end(skb, nest);
        return 0;

err_cancel:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int fec_fill_reply(struct sk_buff *skb,
                          const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;
        const struct fec_reply_data *data = FEC_REPDATA(reply_base);
        int ret;

        ret = ethnl_put_bitset(skb, ETHTOOL_A_FEC_MODES,
                               data->fec_link_modes, NULL,
                               __ETHTOOL_LINK_MODE_MASK_NBITS,
                               link_mode_names, compact);
        if (ret < 0)
                return ret;

        if (nla_put_u8(skb, ETHTOOL_A_FEC_AUTO, data->fec_auto) ||
            (data->active_fec &&
             nla_put_u32(skb, ETHTOOL_A_FEC_ACTIVE, data->active_fec)))
                return -EMSGSIZE;

        if (req_base->flags & ETHTOOL_FLAG_STATS && fec_put_stats(skb, data))
                return -EMSGSIZE;

        return 0;
}

/* FEC_SET */

const struct nla_policy ethnl_fec_set_policy[ETHTOOL_A_FEC_AUTO + 1] = {
        [ETHTOOL_A_FEC_HEADER]        = NLA_POLICY_NESTED(ethnl_header_policy),
        [ETHTOOL_A_FEC_MODES]        = { .type = NLA_NESTED },
        [ETHTOOL_A_FEC_AUTO]        = NLA_POLICY_MAX(NLA_U8, 1),
};

static int
ethnl_set_fec_validate(struct ethnl_req_info *req_info, struct genl_info *info)
{
        const struct ethtool_ops *ops = req_info->dev->ethtool_ops;

        return ops->get_fecparam && ops->set_fecparam ? 1 : -EOPNOTSUPP;
}

static int
ethnl_set_fec(struct ethnl_req_info *req_info, struct genl_info *info)
{
        __ETHTOOL_DECLARE_LINK_MODE_MASK(fec_link_modes) = {};
        struct net_device *dev = req_info->dev;
        struct nlattr **tb = info->attrs;
        struct ethtool_fecparam fec = {};
        bool mod = false;
        u8 fec_auto;
        int ret;

        ret = dev->ethtool_ops->get_fecparam(dev, &fec);
        if (ret < 0)
                return ret;

        ethtool_fec_to_link_modes(fec.fec, fec_link_modes, &fec_auto);

        ret = ethnl_update_bitset(fec_link_modes,
                                  __ETHTOOL_LINK_MODE_MASK_NBITS,
                                  tb[ETHTOOL_A_FEC_MODES],
                                  link_mode_names, info->extack, &mod);
        if (ret < 0)
                return ret;
        ethnl_update_u8(&fec_auto, tb[ETHTOOL_A_FEC_AUTO], &mod);
        if (!mod)
                return 0;

        ret = ethtool_link_modes_to_fecparam(&fec, fec_link_modes, fec_auto);
        if (ret) {
                NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_FEC_MODES],
                                    "invalid FEC modes requested");
                return ret;
        }
        if (!fec.fec) {
                NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_FEC_MODES],
                                    "no FEC modes set");
                return -EINVAL;
        }

        ret = dev->ethtool_ops->set_fecparam(dev, &fec);
        return ret < 0 ? ret : 1;
}

const struct ethnl_request_ops ethnl_fec_request_ops = {
        .request_cmd                = ETHTOOL_MSG_FEC_GET,
        .reply_cmd                = ETHTOOL_MSG_FEC_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_FEC_HEADER,
        .req_info_size                = sizeof(struct fec_req_info),
        .reply_data_size        = sizeof(struct fec_reply_data),

        .prepare_data                = fec_prepare_data,
        .reply_size                = fec_reply_size,
        .fill_reply                = fec_fill_reply,

        .set_validate                = ethnl_set_fec_validate,
        .set                        = ethnl_set_fec,
        .set_ntf_cmd                = ETHTOOL_MSG_FEC_NTF,
};

























   40 











































   55 




    2 


    1 

    1 


   55 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *        Definitions for the UDP-Lite (RFC 3828) code.
 */
#ifndef _UDPLITE_H
#define _UDPLITE_H

#include <net/ip6_checksum.h>
#include <net/udp.h>

/* UDP-Lite socket options */
#define UDPLITE_SEND_CSCOV   10 /* sender partial coverage (as sent)      */
#define UDPLITE_RECV_CSCOV   11 /* receiver partial coverage (threshold ) */

extern struct proto                 udplite_prot;
extern struct udp_table                udplite_table;

/*
 *        Checksum computation is all in software, hence simpler getfrag.
 */
static __inline__ int udplite_getfrag(void *from, char *to, int  offset,
                                      int len, int odd, struct sk_buff *skb)
{
        struct msghdr *msg = from;
        return copy_from_iter_full(to, len, &msg->msg_iter) ? 0 : -EFAULT;
}

/*
 *         Checksumming routines
 */
static inline int udplite_checksum_init(struct sk_buff *skb, struct udphdr *uh)
{
        u16 cscov;

        /* In UDPv4 a zero checksum means that the transmitter generated no
         * checksum. UDP-Lite (like IPv6) mandates checksums, hence packets
         * with a zero checksum field are illegal.                            */
        if (uh->check == 0) {
                net_dbg_ratelimited("UDPLite: zeroed checksum field\n");
                return 1;
        }

        cscov = ntohs(uh->len);

        if (cscov == 0)                 /* Indicates that full coverage is required. */
                ;
        else if (cscov < 8  || cscov > skb->len) {
                /*
                 * Coverage length violates RFC 3828: log and discard silently.
                 */
                net_dbg_ratelimited("UDPLite: bad csum coverage %d/%d\n",
                                    cscov, skb->len);
                return 1;

        } else if (cscov < skb->len) {
                UDP_SKB_CB(skb)->partial_cov = 1;
                UDP_SKB_CB(skb)->cscov = cscov;
                if (skb->ip_summed == CHECKSUM_COMPLETE)
                        skb->ip_summed = CHECKSUM_NONE;
                skb->csum_valid = 0;
        }

        return 0;
}

/* Fast-path computation of checksum. Socket may not be locked. */
static inline __wsum udplite_csum(struct sk_buff *skb)
{
        const int off = skb_transport_offset(skb);
        const struct sock *sk = skb->sk;
        int len = skb->len - off;

        if (udp_test_bit(UDPLITE_SEND_CC, sk)) {
                u16 pcslen = READ_ONCE(udp_sk(sk)->pcslen);

                if (pcslen < len) {
                        if (pcslen > 0)
                                len = pcslen;
                        udp_hdr(skb)->len = htons(pcslen);
                }
        }
        skb->ip_summed = CHECKSUM_NONE;     /* no HW support for checksumming */

        return skb_checksum(skb, off, len, 0);
}

void udplite4_register(void);
#endif        /* _UDPLITE_H */






























































































































































































































































































    2 

















   96 

















    4 



    2 

    1 






















   97 
   96 




   48 
















    2 








































    2 
    2 



    2 


    2 










































































































    2 


    2 
    2 



    2 


    2 

























    2 
    2 


    2 
    2 





































    1 

    1 






    1 








    2 
    1 

    2 


















































































    2 


    2 

    2 
    2 




    2 




    2 

    2 
    2 
    2 


    2 

    2 

    2 






    2 

    2 






































































































  177 
  177 
  173 
  177 
  177 






  177 


















    7 





    6 

    4 



    2 


    1 




    2 



    2 
    2 





    2 

    1 



    1 




    1 



    1 





    2 





























   32 


















































    2 













    2 







   33 


















   33 

   33 










   33 


   33 

   33 
   33 


   33 




   33 
   33 

   33 




   32 




   33 





   33 
   33 




   33 




   33 




   33 




   33 

   33 

   33 





   33 
   33 


   33 

   33 

   33 



    1 


   33 

















    6 






    1 





























































    1 


    1 

    2 

























































































































    2 



















































    3 







    3 







    2 

    3 

    3 







    3 


    3 

    3 


    3 
































































   48 

   48 





   46 
    1 
   48 


    1 

   47 




   47 





   93 





   91 



   94 





























   96 


   98 


   97 


   98 


   96 










   50 




   51 



   51 


   51 














































   50 








   51 
   52 
   52 
   49 

   52 





   52 




   50 





   49 
   51 























































   96 


   97 



















   93 


















   96 

   99 




   96 

















   98 




   52 
    1 


   49 


    3 

   48 


   48 


   48 









   48 

   48 



    1 



   47 


   47 

   46 






   95 





   93 


   93 
























   93 




   93 

   92 




   95 
   95 
   94 





   93 
   92 
   95 
   91 







   95 







   95 



   95 
   94 






   93 




























   92 





















   95 



   95 





   94 





    2 
    4 


    4 
    2 


   98 




   95 




   97 








   98 


   95 

   99 







































    1 











    1 













































    1 

















    1 



    1 

    1 






    2 



    2 


    1 






































    2 
    1 



    2 

    2 
    1 



    1 



    3 



    1 











    3 








    2 


    3 


    1 
    3 

















    2 






    2 
    2 







    2 




    2 



    2 












































   16 





   24 



   24 

    7 

   24 

   12 

   25 

   25 

   25 


   24 

   25 






















   95 




   93 


   95 





























































































































































































































































































































































































































































































    6 

    5 


    4 

    2 

    4 
    2 
    4 
    2 
    4 


    4 


    5 
    1 






    6 


    4 












    3 
    1 





    3 







    2 
    2 




    2 














    1 






    2 






    2 









    2 
    2 
    2 


















    3 




    1 


    1 

    1 


    2 












































   81 














   80 
   75 
    8 


   73 

    3 




   81 

    5 
    2 
   71 
    2 

    1 


   72 




    1 













   71 

    3 



    2 


    1 


    1 




   66 


   67 

   68 


















    1 







    1 
    1 



    2 





    1 





    3 




    2 







    5 




    5 







    1 
    1 





















    1 



    6 





    7 

    7 




    1 






    2 



    3 





    3 



    2 




    1 
    1 



    3 





    2 



    2 




    1 



    4 





    3 




    2 

    1 



    4 



    4 





    2 


    2 


    2 





    1 


    1 




    1 


    1 





    1 



    2 




    3 


    3 




    2 

    1 








    8 


   48 

   72 






   80 

































    1 



    1 




    1 



    1 





    1 



















































































































    7 

































    1 
















    1 
























    1 
    1 














































  204 




  203 

   16 




































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  TUN - Universal TUN/TAP device driver.
 *  Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
 *
 *  $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
 */

/*
 *  Changes:
 *
 *  Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
 *    Add TUNSETLINK ioctl to set the link encapsulation
 *
 *  Mark Smith <markzzzsmith@yahoo.com.au>
 *    Use eth_random_addr() for tap MAC address.
 *
 *  Harald Roelle <harald.roelle@ifi.lmu.de>  2004/04/20
 *    Fixes in packet dropping, queue length setting and queue wakeup.
 *    Increased default tx queue length.
 *    Added ethtool API.
 *    Minor cleanups
 *
 *  Daniel Podlejski <underley@underley.eu.org>
 *    Modifications for 2.3.99-pre5 kernel.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define DRV_NAME        "tun"
#define DRV_VERSION        "1.6"
#define DRV_DESCRIPTION        "Universal TUN/TAP device driver"
#define DRV_COPYRIGHT        "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"

#include <linux/module.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/major.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/fcntl.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/miscdevice.h>
#include <linux/ethtool.h>
#include <linux/rtnetlink.h>
#include <linux/compat.h>
#include <linux/if.h>
#include <linux/if_arp.h>
#include <linux/if_ether.h>
#include <linux/if_tun.h>
#include <linux/if_vlan.h>
#include <linux/crc32.h>
#include <linux/math.h>
#include <linux/nsproxy.h>
#include <linux/virtio_net.h>
#include <linux/rcupdate.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
#include <net/sock.h>
#include <net/xdp.h>
#include <net/ip_tunnels.h>
#include <linux/seq_file.h>
#include <linux/uio.h>
#include <linux/skb_array.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/mutex.h>
#include <linux/ieee802154.h>
#include <uapi/linux/if_ltalk.h>
#include <uapi/linux/if_fddi.h>
#include <uapi/linux/if_hippi.h>
#include <uapi/linux/if_fc.h>
#include <net/ax25.h>
#include <net/rose.h>
#include <net/6lowpan.h>
#include <net/rps.h>

#include <linux/uaccess.h>
#include <linux/proc_fs.h>

static void tun_default_link_ksettings(struct net_device *dev,
                                       struct ethtool_link_ksettings *cmd);

#define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)

/* TUN device flags */

/* IFF_ATTACH_QUEUE is never stored in device flags,
 * overload it to mean fasync when stored there.
 */
#define TUN_FASYNC        IFF_ATTACH_QUEUE
/* High bits in flags field are unused. */
#define TUN_VNET_LE     0x80000000
#define TUN_VNET_BE     0x40000000

#define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
                      IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS)

#define GOODCOPY_LEN 128

#define FLT_EXACT_COUNT 8
struct tap_filter {
        unsigned int    count;    /* Number of addrs. Zero means disabled */
        u32             mask[2];  /* Mask of the hashed addrs */
        unsigned char        addr[FLT_EXACT_COUNT][ETH_ALEN];
};

/* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
 * to max number of VCPUs in guest. */
#define MAX_TAP_QUEUES 256
#define MAX_TAP_FLOWS  4096

#define TUN_FLOW_EXPIRE (3 * HZ)

/* A tun_file connects an open character device to a tuntap netdevice. It
 * also contains all socket related structures (except sock_fprog and tap_filter)
 * to serve as one transmit queue for tuntap device. The sock_fprog and
 * tap_filter were kept in tun_struct since they were used for filtering for the
 * netdevice not for a specific queue (at least I didn't see the requirement for
 * this).
 *
 * RCU usage:
 * The tun_file and tun_struct are loosely coupled, the pointer from one to the
 * other can only be read while rcu_read_lock or rtnl_lock is held.
 */
struct tun_file {
        struct sock sk;
        struct socket socket;
        struct tun_struct __rcu *tun;
        struct fasync_struct *fasync;
        /* only used for fasnyc */
        unsigned int flags;
        union {
                u16 queue_index;
                unsigned int ifindex;
        };
        struct napi_struct napi;
        bool napi_enabled;
        bool napi_frags_enabled;
        struct mutex napi_mutex;        /* Protects access to the above napi */
        struct list_head next;
        struct tun_struct *detached;
        struct ptr_ring tx_ring;
        struct xdp_rxq_info xdp_rxq;
};

struct tun_page {
        struct page *page;
        int count;
};

struct tun_flow_entry {
        struct hlist_node hash_link;
        struct rcu_head rcu;
        struct tun_struct *tun;

        u32 rxhash;
        u32 rps_rxhash;
        int queue_index;
        unsigned long updated ____cacheline_aligned_in_smp;
};

#define TUN_NUM_FLOW_ENTRIES 1024
#define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)

struct tun_prog {
        struct rcu_head rcu;
        struct bpf_prog *prog;
};

/* Since the socket were moved to tun_file, to preserve the behavior of persist
 * device, socket filter, sndbuf and vnet header size were restore when the
 * file were attached to a persist device.
 */
struct tun_struct {
        struct tun_file __rcu        *tfiles[MAX_TAP_QUEUES];
        unsigned int            numqueues;
        unsigned int                 flags;
        kuid_t                        owner;
        kgid_t                        group;

        struct net_device        *dev;
        netdev_features_t        set_features;
#define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
                          NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4)

        int                        align;
        int                        vnet_hdr_sz;
        int                        sndbuf;
        struct tap_filter        txflt;
        struct sock_fprog        fprog;
        /* protected by rtnl lock */
        bool                        filter_attached;
        u32                        msg_enable;
        spinlock_t lock;
        struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
        struct timer_list flow_gc_timer;
        unsigned long ageing_time;
        unsigned int numdisabled;
        struct list_head disabled;
        void *security;
        u32 flow_count;
        u32 rx_batched;
        atomic_long_t rx_frame_errors;
        struct bpf_prog __rcu *xdp_prog;
        struct tun_prog __rcu *steering_prog;
        struct tun_prog __rcu *filter_prog;
        struct ethtool_link_ksettings link_ksettings;
        /* init args */
        struct file *file;
        struct ifreq *ifr;
};

struct veth {
        __be16 h_vlan_proto;
        __be16 h_vlan_TCI;
};

static void tun_flow_init(struct tun_struct *tun);
static void tun_flow_uninit(struct tun_struct *tun);

static int tun_napi_receive(struct napi_struct *napi, int budget)
{
        struct tun_file *tfile = container_of(napi, struct tun_file, napi);
        struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
        struct sk_buff_head process_queue;
        struct sk_buff *skb;
        int received = 0;

        __skb_queue_head_init(&process_queue);

        spin_lock(&queue->lock);
        skb_queue_splice_tail_init(queue, &process_queue);
        spin_unlock(&queue->lock);

        while (received < budget && (skb = __skb_dequeue(&process_queue))) {
                napi_gro_receive(napi, skb);
                ++received;
        }

        if (!skb_queue_empty(&process_queue)) {
                spin_lock(&queue->lock);
                skb_queue_splice(&process_queue, queue);
                spin_unlock(&queue->lock);
        }

        return received;
}

static int tun_napi_poll(struct napi_struct *napi, int budget)
{
        unsigned int received;

        received = tun_napi_receive(napi, budget);

        if (received < budget)
                napi_complete_done(napi, received);

        return received;
}

static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
                          bool napi_en, bool napi_frags)
{
        tfile->napi_enabled = napi_en;
        tfile->napi_frags_enabled = napi_en && napi_frags;
        if (napi_en) {
                netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll);
                napi_enable(&tfile->napi);
        }
}

static void tun_napi_enable(struct tun_file *tfile)
{
        if (tfile->napi_enabled)
                napi_enable(&tfile->napi);
}

static void tun_napi_disable(struct tun_file *tfile)
{
        if (tfile->napi_enabled)
                napi_disable(&tfile->napi);
}

static void tun_napi_del(struct tun_file *tfile)
{
        if (tfile->napi_enabled)
                netif_napi_del(&tfile->napi);
}

static bool tun_napi_frags_enabled(const struct tun_file *tfile)
{
        return tfile->napi_frags_enabled;
}

#ifdef CONFIG_TUN_VNET_CROSS_LE
static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
{
        return tun->flags & TUN_VNET_BE ? false :
                virtio_legacy_is_little_endian();
}

static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
{
        int be = !!(tun->flags & TUN_VNET_BE);

        if (put_user(be, argp))
                return -EFAULT;

        return 0;
}

static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
{
        int be;

        if (get_user(be, argp))
                return -EFAULT;

        if (be)
                tun->flags |= TUN_VNET_BE;
        else
                tun->flags &= ~TUN_VNET_BE;

        return 0;
}
#else
static inline bool tun_legacy_is_little_endian(struct tun_struct *tun)
{
        return virtio_legacy_is_little_endian();
}

static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp)
{
        return -EINVAL;
}

static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp)
{
        return -EINVAL;
}
#endif /* CONFIG_TUN_VNET_CROSS_LE */

static inline bool tun_is_little_endian(struct tun_struct *tun)
{
        return tun->flags & TUN_VNET_LE ||
                tun_legacy_is_little_endian(tun);
}

static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val)
{
        return __virtio16_to_cpu(tun_is_little_endian(tun), val);
}

static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val)
{
        return __cpu_to_virtio16(tun_is_little_endian(tun), val);
}

static inline u32 tun_hashfn(u32 rxhash)
{
        return rxhash & TUN_MASK_FLOW_ENTRIES;
}

static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
{
        struct tun_flow_entry *e;

        hlist_for_each_entry_rcu(e, head, hash_link) {
                if (e->rxhash == rxhash)
                        return e;
        }
        return NULL;
}

static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
                                              struct hlist_head *head,
                                              u32 rxhash, u16 queue_index)
{
        struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC);

        if (e) {
                netif_info(tun, tx_queued, tun->dev,
                           "create flow: hash %u index %u\n",
                           rxhash, queue_index);
                e->updated = jiffies;
                e->rxhash = rxhash;
                e->rps_rxhash = 0;
                e->queue_index = queue_index;
                e->tun = tun;
                hlist_add_head_rcu(&e->hash_link, head);
                ++tun->flow_count;
        }
        return e;
}

static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
{
        netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n",
                   e->rxhash, e->queue_index);
        hlist_del_rcu(&e->hash_link);
        kfree_rcu(e, rcu);
        --tun->flow_count;
}

static void tun_flow_flush(struct tun_struct *tun)
{
        int i;

        spin_lock_bh(&tun->lock);
        for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
                struct tun_flow_entry *e;
                struct hlist_node *n;

                hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
                        tun_flow_delete(tun, e);
        }
        spin_unlock_bh(&tun->lock);
}

static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
{
        int i;

        spin_lock_bh(&tun->lock);
        for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
                struct tun_flow_entry *e;
                struct hlist_node *n;

                hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
                        if (e->queue_index == queue_index)
                                tun_flow_delete(tun, e);
                }
        }
        spin_unlock_bh(&tun->lock);
}

static void tun_flow_cleanup(struct timer_list *t)
{
        struct tun_struct *tun = from_timer(tun, t, flow_gc_timer);
        unsigned long delay = tun->ageing_time;
        unsigned long next_timer = jiffies + delay;
        unsigned long count = 0;
        int i;

        spin_lock(&tun->lock);
        for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
                struct tun_flow_entry *e;
                struct hlist_node *n;

                hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
                        unsigned long this_timer;

                        this_timer = e->updated + delay;
                        if (time_before_eq(this_timer, jiffies)) {
                                tun_flow_delete(tun, e);
                                continue;
                        }
                        count++;
                        if (time_before(this_timer, next_timer))
                                next_timer = this_timer;
                }
        }

        if (count)
                mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
        spin_unlock(&tun->lock);
}

static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
                            struct tun_file *tfile)
{
        struct hlist_head *head;
        struct tun_flow_entry *e;
        unsigned long delay = tun->ageing_time;
        u16 queue_index = tfile->queue_index;

        head = &tun->flows[tun_hashfn(rxhash)];

        rcu_read_lock();

        e = tun_flow_find(head, rxhash);
        if (likely(e)) {
                /* TODO: keep queueing to old queue until it's empty? */
                if (READ_ONCE(e->queue_index) != queue_index)
                        WRITE_ONCE(e->queue_index, queue_index);
                if (e->updated != jiffies)
                        e->updated = jiffies;
                sock_rps_record_flow_hash(e->rps_rxhash);
        } else {
                spin_lock_bh(&tun->lock);
                if (!tun_flow_find(head, rxhash) &&
                    tun->flow_count < MAX_TAP_FLOWS)
                        tun_flow_create(tun, head, rxhash, queue_index);

                if (!timer_pending(&tun->flow_gc_timer))
                        mod_timer(&tun->flow_gc_timer,
                                  round_jiffies_up(jiffies + delay));
                spin_unlock_bh(&tun->lock);
        }

        rcu_read_unlock();
}

/* Save the hash received in the stack receive path and update the
 * flow_hash table accordingly.
 */
static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
{
        if (unlikely(e->rps_rxhash != hash))
                e->rps_rxhash = hash;
}

/* We try to identify a flow through its rxhash. The reason that
 * we do not check rxq no. is because some cards(e.g 82599), chooses
 * the rxq based on the txq where the last packet of the flow comes. As
 * the userspace application move between processors, we may get a
 * different rxq no. here.
 */
static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
{
        struct tun_flow_entry *e;
        u32 txq, numqueues;

        numqueues = READ_ONCE(tun->numqueues);

        txq = __skb_get_hash_symmetric(skb);
        e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
        if (e) {
                tun_flow_save_rps_rxhash(e, txq);
                txq = e->queue_index;
        } else {
                txq = reciprocal_scale(txq, numqueues);
        }

        return txq;
}

static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
{
        struct tun_prog *prog;
        u32 numqueues;
        u16 ret = 0;

        numqueues = READ_ONCE(tun->numqueues);
        if (!numqueues)
                return 0;

        prog = rcu_dereference(tun->steering_prog);
        if (prog)
                ret = bpf_prog_run_clear_cb(prog->prog, skb);

        return ret % numqueues;
}

static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
                            struct net_device *sb_dev)
{
        struct tun_struct *tun = netdev_priv(dev);
        u16 ret;

        rcu_read_lock();
        if (rcu_dereference(tun->steering_prog))
                ret = tun_ebpf_select_queue(tun, skb);
        else
                ret = tun_automq_select_queue(tun, skb);
        rcu_read_unlock();

        return ret;
}

static inline bool tun_not_capable(struct tun_struct *tun)
{
        const struct cred *cred = current_cred();
        struct net *net = dev_net(tun->dev);

        return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) ||
                  (gid_valid(tun->group) && !in_egroup_p(tun->group))) &&
                !ns_capable(net->user_ns, CAP_NET_ADMIN);
}

static void tun_set_real_num_queues(struct tun_struct *tun)
{
        netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
        netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
}

static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
{
        tfile->detached = tun;
        list_add_tail(&tfile->next, &tun->disabled);
        ++tun->numdisabled;
}

static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
{
        struct tun_struct *tun = tfile->detached;

        tfile->detached = NULL;
        list_del_init(&tfile->next);
        --tun->numdisabled;
        return tun;
}

void tun_ptr_free(void *ptr)
{
        if (!ptr)
                return;
        if (tun_is_xdp_frame(ptr)) {
                struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);

                xdp_return_frame(xdpf);
        } else {
                __skb_array_destroy_skb(ptr);
        }
}
EXPORT_SYMBOL_GPL(tun_ptr_free);

static void tun_queue_purge(struct tun_file *tfile)
{
        void *ptr;

        while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
                tun_ptr_free(ptr);

        skb_queue_purge(&tfile->sk.sk_write_queue);
        skb_queue_purge(&tfile->sk.sk_error_queue);
}

static void __tun_detach(struct tun_file *tfile, bool clean)
{
        struct tun_file *ntfile;
        struct tun_struct *tun;

        tun = rtnl_dereference(tfile->tun);

        if (tun && clean) {
                if (!tfile->detached)
                        tun_napi_disable(tfile);
                tun_napi_del(tfile);
        }

        if (tun && !tfile->detached) {
                u16 index = tfile->queue_index;
                BUG_ON(index >= tun->numqueues);

                rcu_assign_pointer(tun->tfiles[index],
                                   tun->tfiles[tun->numqueues - 1]);
                ntfile = rtnl_dereference(tun->tfiles[index]);
                ntfile->queue_index = index;
                ntfile->xdp_rxq.queue_index = index;
                rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
                                   NULL);

                --tun->numqueues;
                if (clean) {
                        RCU_INIT_POINTER(tfile->tun, NULL);
                        sock_put(&tfile->sk);
                } else {
                        tun_disable_queue(tun, tfile);
                        tun_napi_disable(tfile);
                }

                synchronize_net();
                tun_flow_delete_by_queue(tun, tun->numqueues + 1);
                /* Drop read queue */
                tun_queue_purge(tfile);
                tun_set_real_num_queues(tun);
        } else if (tfile->detached && clean) {
                tun = tun_enable_queue(tfile);
                sock_put(&tfile->sk);
        }

        if (clean) {
                if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
                        netif_carrier_off(tun->dev);

                        if (!(tun->flags & IFF_PERSIST) &&
                            tun->dev->reg_state == NETREG_REGISTERED)
                                unregister_netdevice(tun->dev);
                }
                if (tun)
                        xdp_rxq_info_unreg(&tfile->xdp_rxq);
                ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
        }
}

static void tun_detach(struct tun_file *tfile, bool clean)
{
        struct tun_struct *tun;
        struct net_device *dev;

        rtnl_lock();
        tun = rtnl_dereference(tfile->tun);
        dev = tun ? tun->dev : NULL;
        __tun_detach(tfile, clean);
        if (dev)
                netdev_state_change(dev);
        rtnl_unlock();

        if (clean)
                sock_put(&tfile->sk);
}

static void tun_detach_all(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);
        struct tun_file *tfile, *tmp;
        int i, n = tun->numqueues;

        for (i = 0; i < n; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                BUG_ON(!tfile);
                tun_napi_disable(tfile);
                tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
                tfile->socket.sk->sk_data_ready(tfile->socket.sk);
                RCU_INIT_POINTER(tfile->tun, NULL);
                --tun->numqueues;
        }
        list_for_each_entry(tfile, &tun->disabled, next) {
                tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
                tfile->socket.sk->sk_data_ready(tfile->socket.sk);
                RCU_INIT_POINTER(tfile->tun, NULL);
        }
        BUG_ON(tun->numqueues != 0);

        synchronize_net();
        for (i = 0; i < n; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                tun_napi_del(tfile);
                /* Drop read queue */
                tun_queue_purge(tfile);
                xdp_rxq_info_unreg(&tfile->xdp_rxq);
                sock_put(&tfile->sk);
        }
        list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
                tun_napi_del(tfile);
                tun_enable_queue(tfile);
                tun_queue_purge(tfile);
                xdp_rxq_info_unreg(&tfile->xdp_rxq);
                sock_put(&tfile->sk);
        }
        BUG_ON(tun->numdisabled != 0);

        if (tun->flags & IFF_PERSIST)
                module_put(THIS_MODULE);
}

static int tun_attach(struct tun_struct *tun, struct file *file,
                      bool skip_filter, bool napi, bool napi_frags,
                      bool publish_tun)
{
        struct tun_file *tfile = file->private_data;
        struct net_device *dev = tun->dev;
        int err;

        err = security_tun_dev_attach(tfile->socket.sk, tun->security);
        if (err < 0)
                goto out;

        err = -EINVAL;
        if (rtnl_dereference(tfile->tun) && !tfile->detached)
                goto out;

        err = -EBUSY;
        if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
                goto out;

        err = -E2BIG;
        if (!tfile->detached &&
            tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
                goto out;

        err = 0;

        /* Re-attach the filter to persist device */
        if (!skip_filter && (tun->filter_attached == true)) {
                lock_sock(tfile->socket.sk);
                err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
                release_sock(tfile->socket.sk);
                if (!err)
                        goto out;
        }

        if (!tfile->detached &&
            ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
                            GFP_KERNEL, tun_ptr_free)) {
                err = -ENOMEM;
                goto out;
        }

        tfile->queue_index = tun->numqueues;
        tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;

        if (tfile->detached) {
                /* Re-attach detached tfile, updating XDP queue_index */
                WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));

                if (tfile->xdp_rxq.queue_index    != tfile->queue_index)
                        tfile->xdp_rxq.queue_index = tfile->queue_index;
        } else {
                /* Setup XDP RX-queue info, for new tfile getting attached */
                err = xdp_rxq_info_reg(&tfile->xdp_rxq,
                                       tun->dev, tfile->queue_index, 0);
                if (err < 0)
                        goto out;
                err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
                                                 MEM_TYPE_PAGE_SHARED, NULL);
                if (err < 0) {
                        xdp_rxq_info_unreg(&tfile->xdp_rxq);
                        goto out;
                }
                err = 0;
        }

        if (tfile->detached) {
                tun_enable_queue(tfile);
                tun_napi_enable(tfile);
        } else {
                sock_hold(&tfile->sk);
                tun_napi_init(tun, tfile, napi, napi_frags);
        }

        if (rtnl_dereference(tun->xdp_prog))
                sock_set_flag(&tfile->sk, SOCK_XDP);

        /* device is allowed to go away first, so no need to hold extra
         * refcnt.
         */

        /* Publish tfile->tun and tun->tfiles only after we've fully
         * initialized tfile; otherwise we risk using half-initialized
         * object.
         */
        if (publish_tun)
                rcu_assign_pointer(tfile->tun, tun);
        rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
        tun->numqueues++;
        tun_set_real_num_queues(tun);
out:
        return err;
}

static struct tun_struct *tun_get(struct tun_file *tfile)
{
        struct tun_struct *tun;

        rcu_read_lock();
        tun = rcu_dereference(tfile->tun);
        if (tun)
                dev_hold(tun->dev);
        rcu_read_unlock();

        return tun;
}

static void tun_put(struct tun_struct *tun)
{
        dev_put(tun->dev);
}

/* TAP filtering */
static void addr_hash_set(u32 *mask, const u8 *addr)
{
        int n = ether_crc(ETH_ALEN, addr) >> 26;
        mask[n >> 5] |= (1 << (n & 31));
}

static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
{
        int n = ether_crc(ETH_ALEN, addr) >> 26;
        return mask[n >> 5] & (1 << (n & 31));
}

static int update_filter(struct tap_filter *filter, void __user *arg)
{
        struct { u8 u[ETH_ALEN]; } *addr;
        struct tun_filter uf;
        int err, alen, n, nexact;

        if (copy_from_user(&uf, arg, sizeof(uf)))
                return -EFAULT;

        if (!uf.count) {
                /* Disabled */
                filter->count = 0;
                return 0;
        }

        alen = ETH_ALEN * uf.count;
        addr = memdup_user(arg + sizeof(uf), alen);
        if (IS_ERR(addr))
                return PTR_ERR(addr);

        /* The filter is updated without holding any locks. Which is
         * perfectly safe. We disable it first and in the worst
         * case we'll accept a few undesired packets. */
        filter->count = 0;
        wmb();

        /* Use first set of addresses as an exact filter */
        for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
                memcpy(filter->addr[n], addr[n].u, ETH_ALEN);

        nexact = n;

        /* Remaining multicast addresses are hashed,
         * unicast will leave the filter disabled. */
        memset(filter->mask, 0, sizeof(filter->mask));
        for (; n < uf.count; n++) {
                if (!is_multicast_ether_addr(addr[n].u)) {
                        err = 0; /* no filter */
                        goto free_addr;
                }
                addr_hash_set(filter->mask, addr[n].u);
        }

        /* For ALLMULTI just set the mask to all ones.
         * This overrides the mask populated above. */
        if ((uf.flags & TUN_FLT_ALLMULTI))
                memset(filter->mask, ~0, sizeof(filter->mask));

        /* Now enable the filter */
        wmb();
        filter->count = nexact;

        /* Return the number of exact filters */
        err = nexact;
free_addr:
        kfree(addr);
        return err;
}

/* Returns: 0 - drop, !=0 - accept */
static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
{
        /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
         * at this point. */
        struct ethhdr *eh = (struct ethhdr *) skb->data;
        int i;

        /* Exact match */
        for (i = 0; i < filter->count; i++)
                if (ether_addr_equal(eh->h_dest, filter->addr[i]))
                        return 1;

        /* Inexact match (multicast only) */
        if (is_multicast_ether_addr(eh->h_dest))
                return addr_hash_test(filter->mask, eh->h_dest);

        return 0;
}

/*
 * Checks whether the packet is accepted or not.
 * Returns: 0 - drop, !=0 - accept
 */
static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
{
        if (!filter->count)
                return 1;

        return run_filter(filter, skb);
}

/* Network device part of the driver */

static const struct ethtool_ops tun_ethtool_ops;

static int tun_net_init(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);
        struct ifreq *ifr = tun->ifr;
        int err;

        spin_lock_init(&tun->lock);

        err = security_tun_dev_alloc_security(&tun->security);
        if (err < 0)
                return err;

        tun_flow_init(tun);

        dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
        dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
                           TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
                           NETIF_F_HW_VLAN_STAG_TX;
        dev->features = dev->hw_features;
        dev->vlan_features = dev->features &
                             ~(NETIF_F_HW_VLAN_CTAG_TX |
                               NETIF_F_HW_VLAN_STAG_TX);
        dev->lltx = true;

        tun->flags = (tun->flags & ~TUN_FEATURES) |
                      (ifr->ifr_flags & TUN_FEATURES);

        INIT_LIST_HEAD(&tun->disabled);
        err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI,
                         ifr->ifr_flags & IFF_NAPI_FRAGS, false);
        if (err < 0) {
                tun_flow_uninit(tun);
                security_tun_dev_free_security(tun->security);
                return err;
        }
        return 0;
}

/* Net device detach from fd. */
static void tun_net_uninit(struct net_device *dev)
{
        tun_detach_all(dev);
}

/* Net device open. */
static int tun_net_open(struct net_device *dev)
{
        netif_tx_start_all_queues(dev);

        return 0;
}

/* Net device close. */
static int tun_net_close(struct net_device *dev)
{
        netif_tx_stop_all_queues(dev);
        return 0;
}

/* Net device start xmit */
static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
{
#ifdef CONFIG_RPS
        if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {
                /* Select queue was not called for the skbuff, so we extract the
                 * RPS hash and save it into the flow_table here.
                 */
                struct tun_flow_entry *e;
                __u32 rxhash;

                rxhash = __skb_get_hash_symmetric(skb);
                e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
                if (e)
                        tun_flow_save_rps_rxhash(e, rxhash);
        }
#endif
}

static unsigned int run_ebpf_filter(struct tun_struct *tun,
                                    struct sk_buff *skb,
                                    int len)
{
        struct tun_prog *prog = rcu_dereference(tun->filter_prog);

        if (prog)
                len = bpf_prog_run_clear_cb(prog->prog, skb);

        return len;
}

/* Net device start xmit */
static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);
        enum skb_drop_reason drop_reason;
        int txq = skb->queue_mapping;
        struct netdev_queue *queue;
        struct tun_file *tfile;
        int len = skb->len;

        rcu_read_lock();
        tfile = rcu_dereference(tun->tfiles[txq]);

        /* Drop packet if interface is not attached */
        if (!tfile) {
                drop_reason = SKB_DROP_REASON_DEV_READY;
                goto drop;
        }

        if (!rcu_dereference(tun->steering_prog))
                tun_automq_xmit(tun, skb);

        netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len);

        /* Drop if the filter does not like it.
         * This is a noop if the filter is disabled.
         * Filter can be enabled only for the TAP devices. */
        if (!check_filter(&tun->txflt, skb)) {
                drop_reason = SKB_DROP_REASON_TAP_TXFILTER;
                goto drop;
        }

        if (tfile->socket.sk->sk_filter &&
            sk_filter(tfile->socket.sk, skb)) {
                drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
                goto drop;
        }

        len = run_ebpf_filter(tun, skb, len);
        if (len == 0) {
                drop_reason = SKB_DROP_REASON_TAP_FILTER;
                goto drop;
        }

        if (pskb_trim(skb, len)) {
                drop_reason = SKB_DROP_REASON_NOMEM;
                goto drop;
        }

        if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) {
                drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
                goto drop;
        }

        skb_tx_timestamp(skb);

        /* Orphan the skb - required as we might hang on to it
         * for indefinite time.
         */
        skb_orphan(skb);

        nf_reset_ct(skb);

        if (ptr_ring_produce(&tfile->tx_ring, skb)) {
                drop_reason = SKB_DROP_REASON_FULL_RING;
                goto drop;
        }

        /* dev->lltx requires to do our own update of trans_start */
        queue = netdev_get_tx_queue(dev, txq);
        txq_trans_cond_update(queue);

        /* Notify and wake up reader process */
        if (tfile->flags & TUN_FASYNC)
                kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
        tfile->socket.sk->sk_data_ready(tfile->socket.sk);

        rcu_read_unlock();
        return NETDEV_TX_OK;

drop:
        dev_core_stats_tx_dropped_inc(dev);
        skb_tx_error(skb);
        kfree_skb_reason(skb, drop_reason);
        rcu_read_unlock();
        return NET_XMIT_DROP;
}

static void tun_net_mclist(struct net_device *dev)
{
        /*
         * This callback is supposed to deal with mc filter in
         * _rx_ path and has nothing to do with the _tx_ path.
         * In rx path we always accept everything userspace gives us.
         */
}

static netdev_features_t tun_net_fix_features(struct net_device *dev,
        netdev_features_t features)
{
        struct tun_struct *tun = netdev_priv(dev);

        return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
}

static void tun_set_headroom(struct net_device *dev, int new_hr)
{
        struct tun_struct *tun = netdev_priv(dev);

        if (new_hr < NET_SKB_PAD)
                new_hr = NET_SKB_PAD;

        tun->align = new_hr;
}

static void
tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
        struct tun_struct *tun = netdev_priv(dev);

        dev_get_tstats64(dev, stats);

        stats->rx_frame_errors +=
                (unsigned long)atomic_long_read(&tun->rx_frame_errors);
}

static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
                       struct netlink_ext_ack *extack)
{
        struct tun_struct *tun = netdev_priv(dev);
        struct tun_file *tfile;
        struct bpf_prog *old_prog;
        int i;

        old_prog = rtnl_dereference(tun->xdp_prog);
        rcu_assign_pointer(tun->xdp_prog, prog);
        if (old_prog)
                bpf_prog_put(old_prog);

        for (i = 0; i < tun->numqueues; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                if (prog)
                        sock_set_flag(&tfile->sk, SOCK_XDP);
                else
                        sock_reset_flag(&tfile->sk, SOCK_XDP);
        }
        list_for_each_entry(tfile, &tun->disabled, next) {
                if (prog)
                        sock_set_flag(&tfile->sk, SOCK_XDP);
                else
                        sock_reset_flag(&tfile->sk, SOCK_XDP);
        }

        return 0;
}

static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
        switch (xdp->command) {
        case XDP_SETUP_PROG:
                return tun_xdp_set(dev, xdp->prog, xdp->extack);
        default:
                return -EINVAL;
        }
}

static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
{
        if (new_carrier) {
                struct tun_struct *tun = netdev_priv(dev);

                if (!tun->numqueues)
                        return -EPERM;

                netif_carrier_on(dev);
        } else {
                netif_carrier_off(dev);
        }
        return 0;
}

static const struct net_device_ops tun_netdev_ops = {
        .ndo_init                = tun_net_init,
        .ndo_uninit                = tun_net_uninit,
        .ndo_open                = tun_net_open,
        .ndo_stop                = tun_net_close,
        .ndo_start_xmit                = tun_net_xmit,
        .ndo_fix_features        = tun_net_fix_features,
        .ndo_select_queue        = tun_select_queue,
        .ndo_set_rx_headroom        = tun_set_headroom,
        .ndo_get_stats64        = tun_net_get_stats64,
        .ndo_change_carrier        = tun_net_change_carrier,
};

static void __tun_xdp_flush_tfile(struct tun_file *tfile)
{
        /* Notify and wake up reader process */
        if (tfile->flags & TUN_FASYNC)
                kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
        tfile->socket.sk->sk_data_ready(tfile->socket.sk);
}

static int tun_xdp_xmit(struct net_device *dev, int n,
                        struct xdp_frame **frames, u32 flags)
{
        struct tun_struct *tun = netdev_priv(dev);
        struct tun_file *tfile;
        u32 numqueues;
        int nxmit = 0;
        int i;

        if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
                return -EINVAL;

        rcu_read_lock();

resample:
        numqueues = READ_ONCE(tun->numqueues);
        if (!numqueues) {
                rcu_read_unlock();
                return -ENXIO; /* Caller will free/return all frames */
        }

        tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
                                            numqueues]);
        if (unlikely(!tfile))
                goto resample;

        spin_lock(&tfile->tx_ring.producer_lock);
        for (i = 0; i < n; i++) {
                struct xdp_frame *xdp = frames[i];
                /* Encode the XDP flag into lowest bit for consumer to differ
                 * XDP buffer from sk_buff.
                 */
                void *frame = tun_xdp_to_ptr(xdp);

                if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
                        dev_core_stats_tx_dropped_inc(dev);
                        break;
                }
                nxmit++;
        }
        spin_unlock(&tfile->tx_ring.producer_lock);

        if (flags & XDP_XMIT_FLUSH)
                __tun_xdp_flush_tfile(tfile);

        rcu_read_unlock();
        return nxmit;
}

static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
{
        struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
        int nxmit;

        if (unlikely(!frame))
                return -EOVERFLOW;

        nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH);
        if (!nxmit)
                xdp_return_frame_rx_napi(frame);
        return nxmit;
}

static const struct net_device_ops tap_netdev_ops = {
        .ndo_init                = tun_net_init,
        .ndo_uninit                = tun_net_uninit,
        .ndo_open                = tun_net_open,
        .ndo_stop                = tun_net_close,
        .ndo_start_xmit                = tun_net_xmit,
        .ndo_fix_features        = tun_net_fix_features,
        .ndo_set_rx_mode        = tun_net_mclist,
        .ndo_set_mac_address        = eth_mac_addr,
        .ndo_validate_addr        = eth_validate_addr,
        .ndo_select_queue        = tun_select_queue,
        .ndo_features_check        = passthru_features_check,
        .ndo_set_rx_headroom        = tun_set_headroom,
        .ndo_bpf                = tun_xdp,
        .ndo_xdp_xmit                = tun_xdp_xmit,
        .ndo_change_carrier        = tun_net_change_carrier,
};

static void tun_flow_init(struct tun_struct *tun)
{
        int i;

        for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
                INIT_HLIST_HEAD(&tun->flows[i]);

        tun->ageing_time = TUN_FLOW_EXPIRE;
        timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
        mod_timer(&tun->flow_gc_timer,
                  round_jiffies_up(jiffies + tun->ageing_time));
}

static void tun_flow_uninit(struct tun_struct *tun)
{
        del_timer_sync(&tun->flow_gc_timer);
        tun_flow_flush(tun);
}

#define MIN_MTU 68
#define MAX_MTU 65535

/* Initialize net device. */
static void tun_net_initialize(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);

        switch (tun->flags & TUN_TYPE_MASK) {
        case IFF_TUN:
                dev->netdev_ops = &tun_netdev_ops;
                dev->header_ops = &ip_tunnel_header_ops;

                /* Point-to-Point TUN Device */
                dev->hard_header_len = 0;
                dev->addr_len = 0;
                dev->mtu = 1500;

                /* Zero header length */
                dev->type = ARPHRD_NONE;
                dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
                break;

        case IFF_TAP:
                dev->netdev_ops = &tap_netdev_ops;
                /* Ethernet TAP Device */
                ether_setup(dev);
                dev->priv_flags &= ~IFF_TX_SKB_SHARING;
                dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;

                eth_hw_addr_random(dev);

                /* Currently tun does not support XDP, only tap does. */
                dev->xdp_features = NETDEV_XDP_ACT_BASIC |
                                    NETDEV_XDP_ACT_REDIRECT |
                                    NETDEV_XDP_ACT_NDO_XMIT;

                break;
        }

        dev->min_mtu = MIN_MTU;
        dev->max_mtu = MAX_MTU - dev->hard_header_len;
}

static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
{
        struct sock *sk = tfile->socket.sk;

        return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
}

/* Character device part */

/* Poll */
static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
{
        struct tun_file *tfile = file->private_data;
        struct tun_struct *tun = tun_get(tfile);
        struct sock *sk;
        __poll_t mask = 0;

        if (!tun)
                return EPOLLERR;

        sk = tfile->socket.sk;

        poll_wait(file, sk_sleep(sk), wait);

        if (!ptr_ring_empty(&tfile->tx_ring))
                mask |= EPOLLIN | EPOLLRDNORM;

        /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to
         * guarantee EPOLLOUT to be raised by either here or
         * tun_sock_write_space(). Then process could get notification
         * after it writes to a down device and meets -EIO.
         */
        if (tun_sock_writeable(tun, tfile) ||
            (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
             tun_sock_writeable(tun, tfile)))
                mask |= EPOLLOUT | EPOLLWRNORM;

        if (tun->dev->reg_state != NETREG_REGISTERED)
                mask = EPOLLERR;

        tun_put(tun);
        return mask;
}

static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
                                            size_t len,
                                            const struct iov_iter *it)
{
        struct sk_buff *skb;
        size_t linear;
        int err;
        int i;

        if (it->nr_segs > MAX_SKB_FRAGS + 1 ||
            len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN))
                return ERR_PTR(-EMSGSIZE);

        local_bh_disable();
        skb = napi_get_frags(&tfile->napi);
        local_bh_enable();
        if (!skb)
                return ERR_PTR(-ENOMEM);

        linear = iov_iter_single_seg_count(it);
        err = __skb_grow(skb, linear);
        if (err)
                goto free;

        skb->len = len;
        skb->data_len = len - linear;
        skb->truesize += skb->data_len;

        for (i = 1; i < it->nr_segs; i++) {
                const struct iovec *iov = iter_iov(it);
                size_t fragsz = iov->iov_len;
                struct page *page;
                void *frag;

                if (fragsz == 0 || fragsz > PAGE_SIZE) {
                        err = -EINVAL;
                        goto free;
                }
                frag = netdev_alloc_frag(fragsz);
                if (!frag) {
                        err = -ENOMEM;
                        goto free;
                }
                page = virt_to_head_page(frag);
                skb_fill_page_desc(skb, i - 1, page,
                                   frag - page_address(page), fragsz);
        }

        return skb;
free:
        /* frees skb and all frags allocated with napi_alloc_frag() */
        napi_free_frags(&tfile->napi);
        return ERR_PTR(err);
}

/* prepad is the amount to reserve at front.  len is length after that.
 * linear is a hint as to how much to copy (usually headers). */
static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
                                     size_t prepad, size_t len,
                                     size_t linear, int noblock)
{
        struct sock *sk = tfile->socket.sk;
        struct sk_buff *skb;
        int err;

        /* Under a page?  Don't bother with paged skb. */
        if (prepad + len < PAGE_SIZE)
                linear = len;

        if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
                linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
        skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
                                   &err, PAGE_ALLOC_COSTLY_ORDER);
        if (!skb)
                return ERR_PTR(err);

        skb_reserve(skb, prepad);
        skb_put(skb, linear);
        skb->data_len = len - linear;
        skb->len += len - linear;

        return skb;
}

static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
                           struct sk_buff *skb, int more)
{
        struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
        struct sk_buff_head process_queue;
        u32 rx_batched = tun->rx_batched;
        bool rcv = false;

        if (!rx_batched || (!more && skb_queue_empty(queue))) {
                local_bh_disable();
                skb_record_rx_queue(skb, tfile->queue_index);
                netif_receive_skb(skb);
                local_bh_enable();
                return;
        }

        spin_lock(&queue->lock);
        if (!more || skb_queue_len(queue) == rx_batched) {
                __skb_queue_head_init(&process_queue);
                skb_queue_splice_tail_init(queue, &process_queue);
                rcv = true;
        } else {
                __skb_queue_tail(queue, skb);
        }
        spin_unlock(&queue->lock);

        if (rcv) {
                struct sk_buff *nskb;

                local_bh_disable();
                while ((nskb = __skb_dequeue(&process_queue))) {
                        skb_record_rx_queue(nskb, tfile->queue_index);
                        netif_receive_skb(nskb);
                }
                skb_record_rx_queue(skb, tfile->queue_index);
                netif_receive_skb(skb);
                local_bh_enable();
        }
}

static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
                              int len, int noblock, bool zerocopy)
{
        if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
                return false;

        if (tfile->socket.sk->sk_sndbuf != INT_MAX)
                return false;

        if (!noblock)
                return false;

        if (zerocopy)
                return false;

        if (SKB_DATA_ALIGN(len + TUN_RX_PAD + XDP_PACKET_HEADROOM) +
            SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
                return false;

        return true;
}

static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
                                       struct page_frag *alloc_frag, char *buf,
                                       int buflen, int len, int pad)
{
        struct sk_buff *skb = build_skb(buf, buflen);

        if (!skb)
                return ERR_PTR(-ENOMEM);

        skb_reserve(skb, pad);
        skb_put(skb, len);
        skb_set_owner_w(skb, tfile->socket.sk);

        get_page(alloc_frag->page);
        alloc_frag->offset += buflen;

        return skb;
}

static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,
                       struct xdp_buff *xdp, u32 act)
{
        int err;

        switch (act) {
        case XDP_REDIRECT:
                err = xdp_do_redirect(tun->dev, xdp, xdp_prog);
                if (err) {
                        dev_core_stats_rx_dropped_inc(tun->dev);
                        return err;
                }
                dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
                break;
        case XDP_TX:
                err = tun_xdp_tx(tun->dev, xdp);
                if (err < 0) {
                        dev_core_stats_rx_dropped_inc(tun->dev);
                        return err;
                }
                dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
                break;
        case XDP_PASS:
                break;
        default:
                bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act);
                fallthrough;
        case XDP_ABORTED:
                trace_xdp_exception(tun->dev, xdp_prog, act);
                fallthrough;
        case XDP_DROP:
                dev_core_stats_rx_dropped_inc(tun->dev);
                break;
        }

        return act;
}

static struct sk_buff *tun_build_skb(struct tun_struct *tun,
                                     struct tun_file *tfile,
                                     struct iov_iter *from,
                                     struct virtio_net_hdr *hdr,
                                     int len, int *skb_xdp)
{
        struct page_frag *alloc_frag = &current->task_frag;
        struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
        struct bpf_prog *xdp_prog;
        int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
        char *buf;
        size_t copied;
        int pad = TUN_RX_PAD;
        int err = 0;

        rcu_read_lock();
        xdp_prog = rcu_dereference(tun->xdp_prog);
        if (xdp_prog)
                pad += XDP_PACKET_HEADROOM;
        buflen += SKB_DATA_ALIGN(len + pad);
        rcu_read_unlock();

        alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
        if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
                return ERR_PTR(-ENOMEM);

        buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
        copied = copy_page_from_iter(alloc_frag->page,
                                     alloc_frag->offset + pad,
                                     len, from);
        if (copied != len)
                return ERR_PTR(-EFAULT);

        /* There's a small window that XDP may be set after the check
         * of xdp_prog above, this should be rare and for simplicity
         * we do XDP on skb in case the headroom is not enough.
         */
        if (hdr->gso_type || !xdp_prog) {
                *skb_xdp = 1;
                return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
                                       pad);
        }

        *skb_xdp = 0;

        local_bh_disable();
        rcu_read_lock();
        bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
        xdp_prog = rcu_dereference(tun->xdp_prog);
        if (xdp_prog) {
                struct xdp_buff xdp;
                u32 act;

                xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq);
                xdp_prepare_buff(&xdp, buf, pad, len, false);

                act = bpf_prog_run_xdp(xdp_prog, &xdp);
                if (act == XDP_REDIRECT || act == XDP_TX) {
                        get_page(alloc_frag->page);
                        alloc_frag->offset += buflen;
                }
                err = tun_xdp_act(tun, xdp_prog, &xdp, act);
                if (err < 0) {
                        if (act == XDP_REDIRECT || act == XDP_TX)
                                put_page(alloc_frag->page);
                        goto out;
                }

                if (err == XDP_REDIRECT)
                        xdp_do_flush();
                if (err != XDP_PASS)
                        goto out;

                pad = xdp.data - xdp.data_hard_start;
                len = xdp.data_end - xdp.data;
        }
        bpf_net_ctx_clear(bpf_net_ctx);
        rcu_read_unlock();
        local_bh_enable();

        return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad);

out:
        bpf_net_ctx_clear(bpf_net_ctx);
        rcu_read_unlock();
        local_bh_enable();
        return NULL;
}

/* Get packet from user space buffer */
static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
                            void *msg_control, struct iov_iter *from,
                            int noblock, bool more)
{
        struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
        struct sk_buff *skb;
        size_t total_len = iov_iter_count(from);
        size_t len = total_len, align = tun->align, linear;
        struct virtio_net_hdr gso = { 0 };
        int good_linear;
        int copylen;
        bool zerocopy = false;
        int err;
        u32 rxhash = 0;
        int skb_xdp = 1;
        bool frags = tun_napi_frags_enabled(tfile);
        enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;

        if (!(tun->flags & IFF_NO_PI)) {
                if (len < sizeof(pi))
                        return -EINVAL;
                len -= sizeof(pi);

                if (!copy_from_iter_full(&pi, sizeof(pi), from))
                        return -EFAULT;
        }

        if (tun->flags & IFF_VNET_HDR) {
                int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);

                if (len < vnet_hdr_sz)
                        return -EINVAL;
                len -= vnet_hdr_sz;

                if (!copy_from_iter_full(&gso, sizeof(gso), from))
                        return -EFAULT;

                if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
                    tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len))
                        gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2);

                if (tun16_to_cpu(tun, gso.hdr_len) > len)
                        return -EINVAL;
                iov_iter_advance(from, vnet_hdr_sz - sizeof(gso));
        }

        if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
                align += NET_IP_ALIGN;
                if (unlikely(len < ETH_HLEN ||
                             (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN)))
                        return -EINVAL;
        }

        good_linear = SKB_MAX_HEAD(align);

        if (msg_control) {
                struct iov_iter i = *from;

                /* There are 256 bytes to be copied in skb, so there is
                 * enough room for skb expand head in case it is used.
                 * The rest of the buffer is mapped from userspace.
                 */
                copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN;
                if (copylen > good_linear)
                        copylen = good_linear;
                linear = copylen;
                iov_iter_advance(&i, copylen);
                if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
                        zerocopy = true;
        }

        if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
                /* For the packet that is not easy to be processed
                 * (e.g gso or jumbo packet), we will do it at after
                 * skb was created with generic XDP routine.
                 */
                skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
                err = PTR_ERR_OR_ZERO(skb);
                if (err)
                        goto drop;
                if (!skb)
                        return total_len;
        } else {
                if (!zerocopy) {
                        copylen = len;
                        if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
                                linear = good_linear;
                        else
                                linear = tun16_to_cpu(tun, gso.hdr_len);
                }

                if (frags) {
                        mutex_lock(&tfile->napi_mutex);
                        skb = tun_napi_alloc_frags(tfile, copylen, from);
                        /* tun_napi_alloc_frags() enforces a layout for the skb.
                         * If zerocopy is enabled, then this layout will be
                         * overwritten by zerocopy_sg_from_iter().
                         */
                        zerocopy = false;
                } else {
                        if (!linear)
                                linear = min_t(size_t, good_linear, copylen);

                        skb = tun_alloc_skb(tfile, align, copylen, linear,
                                            noblock);
                }

                err = PTR_ERR_OR_ZERO(skb);
                if (err)
                        goto drop;

                if (zerocopy)
                        err = zerocopy_sg_from_iter(skb, from);
                else
                        err = skb_copy_datagram_from_iter(skb, 0, from, len);

                if (err) {
                        err = -EFAULT;
                        drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
                        goto drop;
                }
        }

        if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {
                atomic_long_inc(&tun->rx_frame_errors);
                err = -EINVAL;
                goto free_skb;
        }

        switch (tun->flags & TUN_TYPE_MASK) {
        case IFF_TUN:
                if (tun->flags & IFF_NO_PI) {
                        u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0;

                        switch (ip_version) {
                        case 4:
                                pi.proto = htons(ETH_P_IP);
                                break;
                        case 6:
                                pi.proto = htons(ETH_P_IPV6);
                                break;
                        default:
                                err = -EINVAL;
                                goto drop;
                        }
                }

                skb_reset_mac_header(skb);
                skb->protocol = pi.proto;
                skb->dev = tun->dev;
                break;
        case IFF_TAP:
                if (frags && !pskb_may_pull(skb, ETH_HLEN)) {
                        err = -ENOMEM;
                        drop_reason = SKB_DROP_REASON_HDR_TRUNC;
                        goto drop;
                }
                skb->protocol = eth_type_trans(skb, tun->dev);
                break;
        }

        /* copy skb_ubuf_info for callback when skb has no error */
        if (zerocopy) {
                skb_zcopy_init(skb, msg_control);
        } else if (msg_control) {
                struct ubuf_info *uarg = msg_control;
                uarg->ops->complete(NULL, uarg, false);
        }

        skb_reset_network_header(skb);
        skb_probe_transport_header(skb);
        skb_record_rx_queue(skb, tfile->queue_index);

        if (skb_xdp) {
                struct bpf_prog *xdp_prog;
                int ret;

                local_bh_disable();
                rcu_read_lock();
                xdp_prog = rcu_dereference(tun->xdp_prog);
                if (xdp_prog) {
                        ret = do_xdp_generic(xdp_prog, &skb);
                        if (ret != XDP_PASS) {
                                rcu_read_unlock();
                                local_bh_enable();
                                goto unlock_frags;
                        }
                }
                rcu_read_unlock();
                local_bh_enable();
        }

        /* Compute the costly rx hash only if needed for flow updates.
         * We may get a very small possibility of OOO during switching, not
         * worth to optimize.
         */
        if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
            !tfile->detached)
                rxhash = __skb_get_hash_symmetric(skb);

        rcu_read_lock();
        if (unlikely(!(tun->dev->flags & IFF_UP))) {
                err = -EIO;
                rcu_read_unlock();
                drop_reason = SKB_DROP_REASON_DEV_READY;
                goto drop;
        }

        if (frags) {
                u32 headlen;

                /* Exercise flow dissector code path. */
                skb_push(skb, ETH_HLEN);
                headlen = eth_get_headlen(tun->dev, skb->data,
                                          skb_headlen(skb));

                if (unlikely(headlen > skb_headlen(skb))) {
                        WARN_ON_ONCE(1);
                        err = -ENOMEM;
                        dev_core_stats_rx_dropped_inc(tun->dev);
napi_busy:
                        napi_free_frags(&tfile->napi);
                        rcu_read_unlock();
                        mutex_unlock(&tfile->napi_mutex);
                        return err;
                }

                if (likely(napi_schedule_prep(&tfile->napi))) {
                        local_bh_disable();
                        napi_gro_frags(&tfile->napi);
                        napi_complete(&tfile->napi);
                        local_bh_enable();
                } else {
                        err = -EBUSY;
                        goto napi_busy;
                }
                mutex_unlock(&tfile->napi_mutex);
        } else if (tfile->napi_enabled) {
                struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
                int queue_len;

                spin_lock_bh(&queue->lock);

                if (unlikely(tfile->detached)) {
                        spin_unlock_bh(&queue->lock);
                        rcu_read_unlock();
                        err = -EBUSY;
                        goto free_skb;
                }

                __skb_queue_tail(queue, skb);
                queue_len = skb_queue_len(queue);
                spin_unlock(&queue->lock);

                if (!more || queue_len > NAPI_POLL_WEIGHT)
                        napi_schedule(&tfile->napi);

                local_bh_enable();
        } else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
                tun_rx_batched(tun, tfile, skb, more);
        } else {
                netif_rx(skb);
        }
        rcu_read_unlock();

        preempt_disable();
        dev_sw_netstats_rx_add(tun->dev, len);
        preempt_enable();

        if (rxhash)
                tun_flow_update(tun, rxhash, tfile);

        return total_len;

drop:
        if (err != -EAGAIN)
                dev_core_stats_rx_dropped_inc(tun->dev);

free_skb:
        if (!IS_ERR_OR_NULL(skb))
                kfree_skb_reason(skb, drop_reason);

unlock_frags:
        if (frags) {
                tfile->napi.skb = NULL;
                mutex_unlock(&tfile->napi_mutex);
        }

        return err ?: total_len;
}

static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
        struct file *file = iocb->ki_filp;
        struct tun_file *tfile = file->private_data;
        struct tun_struct *tun = tun_get(tfile);
        ssize_t result;
        int noblock = 0;

        if (!tun)
                return -EBADFD;

        if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
                noblock = 1;

        result = tun_get_user(tun, tfile, NULL, from, noblock, false);

        tun_put(tun);
        return result;
}

static ssize_t tun_put_user_xdp(struct tun_struct *tun,
                                struct tun_file *tfile,
                                struct xdp_frame *xdp_frame,
                                struct iov_iter *iter)
{
        int vnet_hdr_sz = 0;
        size_t size = xdp_frame->len;
        size_t ret;

        if (tun->flags & IFF_VNET_HDR) {
                struct virtio_net_hdr gso = { 0 };

                vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
                if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
                        return -EINVAL;
                if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
                             sizeof(gso)))
                        return -EFAULT;
                iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
        }

        ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;

        preempt_disable();
        dev_sw_netstats_tx_add(tun->dev, 1, ret);
        preempt_enable();

        return ret;
}

/* Put packet to the user space buffer */
static ssize_t tun_put_user(struct tun_struct *tun,
                            struct tun_file *tfile,
                            struct sk_buff *skb,
                            struct iov_iter *iter)
{
        struct tun_pi pi = { 0, skb->protocol };
        ssize_t total;
        int vlan_offset = 0;
        int vlan_hlen = 0;
        int vnet_hdr_sz = 0;

        if (skb_vlan_tag_present(skb))
                vlan_hlen = VLAN_HLEN;

        if (tun->flags & IFF_VNET_HDR)
                vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);

        total = skb->len + vlan_hlen + vnet_hdr_sz;

        if (!(tun->flags & IFF_NO_PI)) {
                if (iov_iter_count(iter) < sizeof(pi))
                        return -EINVAL;

                total += sizeof(pi);
                if (iov_iter_count(iter) < total) {
                        /* Packet will be striped */
                        pi.flags |= TUN_PKT_STRIP;
                }

                if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
                        return -EFAULT;
        }

        if (vnet_hdr_sz) {
                struct virtio_net_hdr gso;

                if (iov_iter_count(iter) < vnet_hdr_sz)
                        return -EINVAL;

                if (virtio_net_hdr_from_skb(skb, &gso,
                                            tun_is_little_endian(tun), true,
                                            vlan_hlen)) {
                        struct skb_shared_info *sinfo = skb_shinfo(skb);

                        if (net_ratelimit()) {
                                netdev_err(tun->dev, "unexpected GSO type: 0x%x, gso_size %d, hdr_len %d\n",
                                           sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size),
                                           tun16_to_cpu(tun, gso.hdr_len));
                                print_hex_dump(KERN_ERR, "tun: ",
                                               DUMP_PREFIX_NONE,
                                               16, 1, skb->head,
                                               min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true);
                        }
                        WARN_ON_ONCE(1);
                        return -EINVAL;
                }

                if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso))
                        return -EFAULT;

                iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
        }

        if (vlan_hlen) {
                int ret;
                struct veth veth;

                veth.h_vlan_proto = skb->vlan_proto;
                veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));

                vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);

                ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
                if (ret || !iov_iter_count(iter))
                        goto done;

                ret = copy_to_iter(&veth, sizeof(veth), iter);
                if (ret != sizeof(veth) || !iov_iter_count(iter))
                        goto done;
        }

        skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);

done:
        /* caller is in process context, */
        preempt_disable();
        dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen);
        preempt_enable();

        return total;
}

static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
{
        DECLARE_WAITQUEUE(wait, current);
        void *ptr = NULL;
        int error = 0;

        ptr = ptr_ring_consume(&tfile->tx_ring);
        if (ptr)
                goto out;
        if (noblock) {
                error = -EAGAIN;
                goto out;
        }

        add_wait_queue(&tfile->socket.wq.wait, &wait);

        while (1) {
                set_current_state(TASK_INTERRUPTIBLE);
                ptr = ptr_ring_consume(&tfile->tx_ring);
                if (ptr)
                        break;
                if (signal_pending(current)) {
                        error = -ERESTARTSYS;
                        break;
                }
                if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
                        error = -EFAULT;
                        break;
                }

                schedule();
        }

        __set_current_state(TASK_RUNNING);
        remove_wait_queue(&tfile->socket.wq.wait, &wait);

out:
        *err = error;
        return ptr;
}

static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
                           struct iov_iter *to,
                           int noblock, void *ptr)
{
        ssize_t ret;
        int err;

        if (!iov_iter_count(to)) {
                tun_ptr_free(ptr);
                return 0;
        }

        if (!ptr) {
                /* Read frames from ring */
                ptr = tun_ring_recv(tfile, noblock, &err);
                if (!ptr)
                        return err;
        }

        if (tun_is_xdp_frame(ptr)) {
                struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);

                ret = tun_put_user_xdp(tun, tfile, xdpf, to);
                xdp_return_frame(xdpf);
        } else {
                struct sk_buff *skb = ptr;

                ret = tun_put_user(tun, tfile, skb, to);
                if (unlikely(ret < 0))
                        kfree_skb(skb);
                else
                        consume_skb(skb);
        }

        return ret;
}

static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct file *file = iocb->ki_filp;
        struct tun_file *tfile = file->private_data;
        struct tun_struct *tun = tun_get(tfile);
        ssize_t len = iov_iter_count(to), ret;
        int noblock = 0;

        if (!tun)
                return -EBADFD;

        if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
                noblock = 1;

        ret = tun_do_read(tun, tfile, to, noblock, NULL);
        ret = min_t(ssize_t, ret, len);
        if (ret > 0)
                iocb->ki_pos = ret;
        tun_put(tun);
        return ret;
}

static void tun_prog_free(struct rcu_head *rcu)
{
        struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);

        bpf_prog_destroy(prog->prog);
        kfree(prog);
}

static int __tun_set_ebpf(struct tun_struct *tun,
                          struct tun_prog __rcu **prog_p,
                          struct bpf_prog *prog)
{
        struct tun_prog *old, *new = NULL;

        if (prog) {
                new = kmalloc(sizeof(*new), GFP_KERNEL);
                if (!new)
                        return -ENOMEM;
                new->prog = prog;
        }

        spin_lock_bh(&tun->lock);
        old = rcu_dereference_protected(*prog_p,
                                        lockdep_is_held(&tun->lock));
        rcu_assign_pointer(*prog_p, new);
        spin_unlock_bh(&tun->lock);

        if (old)
                call_rcu(&old->rcu, tun_prog_free);

        return 0;
}

static void tun_free_netdev(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);

        BUG_ON(!(list_empty(&tun->disabled)));

        tun_flow_uninit(tun);
        security_tun_dev_free_security(tun->security);
        __tun_set_ebpf(tun, &tun->steering_prog, NULL);
        __tun_set_ebpf(tun, &tun->filter_prog, NULL);
}

static void tun_setup(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);

        tun->owner = INVALID_UID;
        tun->group = INVALID_GID;
        tun_default_link_ksettings(dev, &tun->link_ksettings);

        dev->ethtool_ops = &tun_ethtool_ops;
        dev->needs_free_netdev = true;
        dev->priv_destructor = tun_free_netdev;
        /* We prefer our own queue length */
        dev->tx_queue_len = TUN_READQ_SIZE;
}

/* Trivial set of netlink ops to allow deleting tun or tap
 * device with netlink.
 */
static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
                        struct netlink_ext_ack *extack)
{
        NL_SET_ERR_MSG(extack,
                       "tun/tap creation via rtnetlink is not supported.");
        return -EOPNOTSUPP;
}

static size_t tun_get_size(const struct net_device *dev)
{
        BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
        BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));

        return nla_total_size(sizeof(uid_t)) + /* OWNER */
               nla_total_size(sizeof(gid_t)) + /* GROUP */
               nla_total_size(sizeof(u8)) + /* TYPE */
               nla_total_size(sizeof(u8)) + /* PI */
               nla_total_size(sizeof(u8)) + /* VNET_HDR */
               nla_total_size(sizeof(u8)) + /* PERSIST */
               nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */
               nla_total_size(sizeof(u32)) + /* NUM_QUEUES */
               nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */
               0;
}

static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);

        if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK))
                goto nla_put_failure;
        if (uid_valid(tun->owner) &&
            nla_put_u32(skb, IFLA_TUN_OWNER,
                        from_kuid_munged(current_user_ns(), tun->owner)))
                goto nla_put_failure;
        if (gid_valid(tun->group) &&
            nla_put_u32(skb, IFLA_TUN_GROUP,
                        from_kgid_munged(current_user_ns(), tun->group)))
                goto nla_put_failure;
        if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI)))
                goto nla_put_failure;
        if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR)))
                goto nla_put_failure;
        if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST)))
                goto nla_put_failure;
        if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
                       !!(tun->flags & IFF_MULTI_QUEUE)))
                goto nla_put_failure;
        if (tun->flags & IFF_MULTI_QUEUE) {
                if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues))
                        goto nla_put_failure;
                if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
                                tun->numdisabled))
                        goto nla_put_failure;
        }

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static struct rtnl_link_ops tun_link_ops __read_mostly = {
        .kind                = DRV_NAME,
        .priv_size        = sizeof(struct tun_struct),
        .setup                = tun_setup,
        .validate        = tun_validate,
        .get_size       = tun_get_size,
        .fill_info      = tun_fill_info,
};

static void tun_sock_write_space(struct sock *sk)
{
        struct tun_file *tfile;
        wait_queue_head_t *wqueue;

        if (!sock_writeable(sk))
                return;

        if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
                return;

        wqueue = sk_sleep(sk);
        if (wqueue && waitqueue_active(wqueue))
                wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
                                                EPOLLWRNORM | EPOLLWRBAND);

        tfile = container_of(sk, struct tun_file, sk);
        kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
}

static void tun_put_page(struct tun_page *tpage)
{
        if (tpage->page)
                __page_frag_cache_drain(tpage->page, tpage->count);
}

static int tun_xdp_one(struct tun_struct *tun,
                       struct tun_file *tfile,
                       struct xdp_buff *xdp, int *flush,
                       struct tun_page *tpage)
{
        unsigned int datasize = xdp->data_end - xdp->data;
        struct tun_xdp_hdr *hdr = xdp->data_hard_start;
        struct virtio_net_hdr *gso = &hdr->gso;
        struct bpf_prog *xdp_prog;
        struct sk_buff *skb = NULL;
        struct sk_buff_head *queue;
        u32 rxhash = 0, act;
        int buflen = hdr->buflen;
        int ret = 0;
        bool skb_xdp = false;
        struct page *page;

        if (unlikely(datasize < ETH_HLEN))
                return -EINVAL;

        xdp_prog = rcu_dereference(tun->xdp_prog);
        if (xdp_prog) {
                if (gso->gso_type) {
                        skb_xdp = true;
                        goto build;
                }

                xdp_init_buff(xdp, buflen, &tfile->xdp_rxq);
                xdp_set_data_meta_invalid(xdp);

                act = bpf_prog_run_xdp(xdp_prog, xdp);
                ret = tun_xdp_act(tun, xdp_prog, xdp, act);
                if (ret < 0) {
                        put_page(virt_to_head_page(xdp->data));
                        return ret;
                }

                switch (ret) {
                case XDP_REDIRECT:
                        *flush = true;
                        fallthrough;
                case XDP_TX:
                        return 0;
                case XDP_PASS:
                        break;
                default:
                        page = virt_to_head_page(xdp->data);
                        if (tpage->page == page) {
                                ++tpage->count;
                        } else {
                                tun_put_page(tpage);
                                tpage->page = page;
                                tpage->count = 1;
                        }
                        return 0;
                }
        }

build:
        skb = build_skb(xdp->data_hard_start, buflen);
        if (!skb) {
                ret = -ENOMEM;
                goto out;
        }

        skb_reserve(skb, xdp->data - xdp->data_hard_start);
        skb_put(skb, xdp->data_end - xdp->data);

        if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) {
                atomic_long_inc(&tun->rx_frame_errors);
                kfree_skb(skb);
                ret = -EINVAL;
                goto out;
        }

        skb->protocol = eth_type_trans(skb, tun->dev);
        skb_reset_network_header(skb);
        skb_probe_transport_header(skb);
        skb_record_rx_queue(skb, tfile->queue_index);

        if (skb_xdp) {
                ret = do_xdp_generic(xdp_prog, &skb);
                if (ret != XDP_PASS) {
                        ret = 0;
                        goto out;
                }
        }

        if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
            !tfile->detached)
                rxhash = __skb_get_hash_symmetric(skb);

        if (tfile->napi_enabled) {
                queue = &tfile->sk.sk_write_queue;
                spin_lock(&queue->lock);

                if (unlikely(tfile->detached)) {
                        spin_unlock(&queue->lock);
                        kfree_skb(skb);
                        return -EBUSY;
                }

                __skb_queue_tail(queue, skb);
                spin_unlock(&queue->lock);
                ret = 1;
        } else {
                netif_receive_skb(skb);
                ret = 0;
        }

        /* No need to disable preemption here since this function is
         * always called with bh disabled
         */
        dev_sw_netstats_rx_add(tun->dev, datasize);

        if (rxhash)
                tun_flow_update(tun, rxhash, tfile);

out:
        return ret;
}

static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
{
        int ret, i;
        struct tun_file *tfile = container_of(sock, struct tun_file, socket);
        struct tun_struct *tun = tun_get(tfile);
        struct tun_msg_ctl *ctl = m->msg_control;
        struct xdp_buff *xdp;

        if (!tun)
                return -EBADFD;

        if (m->msg_controllen == sizeof(struct tun_msg_ctl) &&
            ctl && ctl->type == TUN_MSG_PTR) {
                struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
                struct tun_page tpage;
                int n = ctl->num;
                int flush = 0, queued = 0;

                memset(&tpage, 0, sizeof(tpage));

                local_bh_disable();
                rcu_read_lock();
                bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);

                for (i = 0; i < n; i++) {
                        xdp = &((struct xdp_buff *)ctl->ptr)[i];
                        ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
                        if (ret > 0)
                                queued += ret;
                }

                if (flush)
                        xdp_do_flush();

                if (tfile->napi_enabled && queued > 0)
                        napi_schedule(&tfile->napi);

                bpf_net_ctx_clear(bpf_net_ctx);
                rcu_read_unlock();
                local_bh_enable();

                tun_put_page(&tpage);

                ret = total_len;
                goto out;
        }

        ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
                           m->msg_flags & MSG_DONTWAIT,
                           m->msg_flags & MSG_MORE);
out:
        tun_put(tun);
        return ret;
}

static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
                       int flags)
{
        struct tun_file *tfile = container_of(sock, struct tun_file, socket);
        struct tun_struct *tun = tun_get(tfile);
        void *ptr = m->msg_control;
        int ret;

        if (!tun) {
                ret = -EBADFD;
                goto out_free;
        }

        if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
                ret = -EINVAL;
                goto out_put_tun;
        }
        if (flags & MSG_ERRQUEUE) {
                ret = sock_recv_errqueue(sock->sk, m, total_len,
                                         SOL_PACKET, TUN_TX_TIMESTAMP);
                goto out;
        }
        ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
        if (ret > (ssize_t)total_len) {
                m->msg_flags |= MSG_TRUNC;
                ret = flags & MSG_TRUNC ? ret : total_len;
        }
out:
        tun_put(tun);
        return ret;

out_put_tun:
        tun_put(tun);
out_free:
        tun_ptr_free(ptr);
        return ret;
}

static int tun_ptr_peek_len(void *ptr)
{
        if (likely(ptr)) {
                if (tun_is_xdp_frame(ptr)) {
                        struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);

                        return xdpf->len;
                }
                return __skb_array_len_with_tag(ptr);
        } else {
                return 0;
        }
}

static int tun_peek_len(struct socket *sock)
{
        struct tun_file *tfile = container_of(sock, struct tun_file, socket);
        struct tun_struct *tun;
        int ret = 0;

        tun = tun_get(tfile);
        if (!tun)
                return 0;

        ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
        tun_put(tun);

        return ret;
}

/* Ops structure to mimic raw sockets with tun */
static const struct proto_ops tun_socket_ops = {
        .peek_len = tun_peek_len,
        .sendmsg = tun_sendmsg,
        .recvmsg = tun_recvmsg,
};

static struct proto tun_proto = {
        .name                = "tun",
        .owner                = THIS_MODULE,
        .obj_size        = sizeof(struct tun_file),
};

static int tun_flags(struct tun_struct *tun)
{
        return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
}

static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr,
                              char *buf)
{
        struct tun_struct *tun = netdev_priv(to_net_dev(dev));
        return sysfs_emit(buf, "0x%x\n", tun_flags(tun));
}

static ssize_t owner_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        struct tun_struct *tun = netdev_priv(to_net_dev(dev));
        return uid_valid(tun->owner)?
                sysfs_emit(buf, "%u\n",
                           from_kuid_munged(current_user_ns(), tun->owner)) :
                sysfs_emit(buf, "-1\n");
}

static ssize_t group_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        struct tun_struct *tun = netdev_priv(to_net_dev(dev));
        return gid_valid(tun->group) ?
                sysfs_emit(buf, "%u\n",
                           from_kgid_munged(current_user_ns(), tun->group)) :
                sysfs_emit(buf, "-1\n");
}

static DEVICE_ATTR_RO(tun_flags);
static DEVICE_ATTR_RO(owner);
static DEVICE_ATTR_RO(group);

static struct attribute *tun_dev_attrs[] = {
        &dev_attr_tun_flags.attr,
        &dev_attr_owner.attr,
        &dev_attr_group.attr,
        NULL
};

static const struct attribute_group tun_attr_group = {
        .attrs = tun_dev_attrs
};

static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
{
        struct tun_struct *tun;
        struct tun_file *tfile = file->private_data;
        struct net_device *dev;
        int err;

        if (tfile->detached)
                return -EINVAL;

        if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
                if (!capable(CAP_NET_ADMIN))
                        return -EPERM;

                if (!(ifr->ifr_flags & IFF_NAPI) ||
                    (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
                        return -EINVAL;
        }

        dev = __dev_get_by_name(net, ifr->ifr_name);
        if (dev) {
                if (ifr->ifr_flags & IFF_TUN_EXCL)
                        return -EBUSY;
                if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
                        tun = netdev_priv(dev);
                else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
                        tun = netdev_priv(dev);
                else
                        return -EINVAL;

                if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
                    !!(tun->flags & IFF_MULTI_QUEUE))
                        return -EINVAL;

                if (tun_not_capable(tun))
                        return -EPERM;
                err = security_tun_dev_open(tun->security);
                if (err < 0)
                        return err;

                err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
                                 ifr->ifr_flags & IFF_NAPI,
                                 ifr->ifr_flags & IFF_NAPI_FRAGS, true);
                if (err < 0)
                        return err;

                if (tun->flags & IFF_MULTI_QUEUE &&
                    (tun->numqueues + tun->numdisabled > 1)) {
                        /* One or more queue has already been attached, no need
                         * to initialize the device again.
                         */
                        netdev_state_change(dev);
                        return 0;
                }

                tun->flags = (tun->flags & ~TUN_FEATURES) |
                              (ifr->ifr_flags & TUN_FEATURES);

                netdev_state_change(dev);
        } else {
                char *name;
                unsigned long flags = 0;
                int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
                             MAX_TAP_QUEUES : 1;

                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                err = security_tun_dev_create();
                if (err < 0)
                        return err;

                /* Set dev type */
                if (ifr->ifr_flags & IFF_TUN) {
                        /* TUN device */
                        flags |= IFF_TUN;
                        name = "tun%d";
                } else if (ifr->ifr_flags & IFF_TAP) {
                        /* TAP device */
                        flags |= IFF_TAP;
                        name = "tap%d";
                } else
                        return -EINVAL;

                if (*ifr->ifr_name)
                        name = ifr->ifr_name;

                dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
                                       NET_NAME_UNKNOWN, tun_setup, queues,
                                       queues);

                if (!dev)
                        return -ENOMEM;

                dev_net_set(dev, net);
                dev->rtnl_link_ops = &tun_link_ops;
                dev->ifindex = tfile->ifindex;
                dev->sysfs_groups[0] = &tun_attr_group;

                tun = netdev_priv(dev);
                tun->dev = dev;
                tun->flags = flags;
                tun->txflt.count = 0;
                tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);

                tun->align = NET_SKB_PAD;
                tun->filter_attached = false;
                tun->sndbuf = tfile->socket.sk->sk_sndbuf;
                tun->rx_batched = 0;
                RCU_INIT_POINTER(tun->steering_prog, NULL);

                tun->ifr = ifr;
                tun->file = file;

                tun_net_initialize(dev);

                err = register_netdevice(tun->dev);
                if (err < 0) {
                        free_netdev(dev);
                        return err;
                }
                /* free_netdev() won't check refcnt, to avoid race
                 * with dev_put() we need publish tun after registration.
                 */
                rcu_assign_pointer(tfile->tun, tun);
        }

        if (ifr->ifr_flags & IFF_NO_CARRIER)
                netif_carrier_off(tun->dev);
        else
                netif_carrier_on(tun->dev);

        /* Make sure persistent devices do not get stuck in
         * xoff state.
         */
        if (netif_running(tun->dev))
                netif_tx_wake_all_queues(tun->dev);

        strcpy(ifr->ifr_name, tun->dev->name);
        return 0;
}

static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)
{
        strcpy(ifr->ifr_name, tun->dev->name);

        ifr->ifr_flags = tun_flags(tun);

}

/* This is like a cut-down ethtool ops, except done via tun fd so no
 * privs required. */
static int set_offload(struct tun_struct *tun, unsigned long arg)
{
        netdev_features_t features = 0;

        if (arg & TUN_F_CSUM) {
                features |= NETIF_F_HW_CSUM;
                arg &= ~TUN_F_CSUM;

                if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
                        if (arg & TUN_F_TSO_ECN) {
                                features |= NETIF_F_TSO_ECN;
                                arg &= ~TUN_F_TSO_ECN;
                        }
                        if (arg & TUN_F_TSO4)
                                features |= NETIF_F_TSO;
                        if (arg & TUN_F_TSO6)
                                features |= NETIF_F_TSO6;
                        arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
                }

                arg &= ~TUN_F_UFO;

                /* TODO: for now USO4 and USO6 should work simultaneously */
                if (arg & TUN_F_USO4 && arg & TUN_F_USO6) {
                        features |= NETIF_F_GSO_UDP_L4;
                        arg &= ~(TUN_F_USO4 | TUN_F_USO6);
                }
        }

        /* This gives the user a way to test for new features in future by
         * trying to set them. */
        if (arg)
                return -EINVAL;

        tun->set_features = features;
        tun->dev->wanted_features &= ~TUN_USER_FEATURES;
        tun->dev->wanted_features |= features;
        netdev_update_features(tun->dev);

        return 0;
}

static void tun_detach_filter(struct tun_struct *tun, int n)
{
        int i;
        struct tun_file *tfile;

        for (i = 0; i < n; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                lock_sock(tfile->socket.sk);
                sk_detach_filter(tfile->socket.sk);
                release_sock(tfile->socket.sk);
        }

        tun->filter_attached = false;
}

static int tun_attach_filter(struct tun_struct *tun)
{
        int i, ret = 0;
        struct tun_file *tfile;

        for (i = 0; i < tun->numqueues; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                lock_sock(tfile->socket.sk);
                ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
                release_sock(tfile->socket.sk);
                if (ret) {
                        tun_detach_filter(tun, i);
                        return ret;
                }
        }

        tun->filter_attached = true;
        return ret;
}

static void tun_set_sndbuf(struct tun_struct *tun)
{
        struct tun_file *tfile;
        int i;

        for (i = 0; i < tun->numqueues; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                tfile->socket.sk->sk_sndbuf = tun->sndbuf;
        }
}

static int tun_set_queue(struct file *file, struct ifreq *ifr)
{
        struct tun_file *tfile = file->private_data;
        struct tun_struct *tun;
        int ret = 0;

        rtnl_lock();

        if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
                tun = tfile->detached;
                if (!tun) {
                        ret = -EINVAL;
                        goto unlock;
                }
                ret = security_tun_dev_attach_queue(tun->security);
                if (ret < 0)
                        goto unlock;
                ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,
                                 tun->flags & IFF_NAPI_FRAGS, true);
        } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
                tun = rtnl_dereference(tfile->tun);
                if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
                        ret = -EINVAL;
                else
                        __tun_detach(tfile, false);
        } else
                ret = -EINVAL;

        if (ret >= 0)
                netdev_state_change(tun->dev);

unlock:
        rtnl_unlock();
        return ret;
}

static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p,
                        void __user *data)
{
        struct bpf_prog *prog;
        int fd;

        if (copy_from_user(&fd, data, sizeof(fd)))
                return -EFAULT;

        if (fd == -1) {
                prog = NULL;
        } else {
                prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
                if (IS_ERR(prog))
                        return PTR_ERR(prog);
        }

        return __tun_set_ebpf(tun, prog_p, prog);
}

/* Return correct value for tun->dev->addr_len based on tun->dev->type. */
static unsigned char tun_get_addr_len(unsigned short type)
{
        switch (type) {
        case ARPHRD_IP6GRE:
        case ARPHRD_TUNNEL6:
                return sizeof(struct in6_addr);
        case ARPHRD_IPGRE:
        case ARPHRD_TUNNEL:
        case ARPHRD_SIT:
                return 4;
        case ARPHRD_ETHER:
                return ETH_ALEN;
        case ARPHRD_IEEE802154:
        case ARPHRD_IEEE802154_MONITOR:
                return IEEE802154_EXTENDED_ADDR_LEN;
        case ARPHRD_PHONET_PIPE:
        case ARPHRD_PPP:
        case ARPHRD_NONE:
                return 0;
        case ARPHRD_6LOWPAN:
                return EUI64_ADDR_LEN;
        case ARPHRD_FDDI:
                return FDDI_K_ALEN;
        case ARPHRD_HIPPI:
                return HIPPI_ALEN;
        case ARPHRD_IEEE802:
                return FC_ALEN;
        case ARPHRD_ROSE:
                return ROSE_ADDR_LEN;
        case ARPHRD_NETROM:
                return AX25_ADDR_LEN;
        case ARPHRD_LOCALTLK:
                return LTALK_ALEN;
        default:
                return 0;
        }
}

static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
                            unsigned long arg, int ifreq_len)
{
        struct tun_file *tfile = file->private_data;
        struct net *net = sock_net(&tfile->sk);
        struct tun_struct *tun;
        void __user* argp = (void __user*)arg;
        unsigned int carrier;
        struct ifreq ifr;
        kuid_t owner;
        kgid_t group;
        int ifindex;
        int sndbuf;
        int vnet_hdr_sz;
        int le;
        int ret;
        bool do_notify = false;

        if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
            (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
                if (copy_from_user(&ifr, argp, ifreq_len))
                        return -EFAULT;
        } else {
                memset(&ifr, 0, sizeof(ifr));
        }
        if (cmd == TUNGETFEATURES) {
                /* Currently this just means: "what IFF flags are valid?".
                 * This is needed because we never checked for invalid flags on
                 * TUNSETIFF.
                 */
                return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER |
                                TUN_FEATURES, (unsigned int __user*)argp);
        } else if (cmd == TUNSETQUEUE) {
                return tun_set_queue(file, &ifr);
        } else if (cmd == SIOCGSKNS) {
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                return open_related_ns(&net->ns, get_net_ns);
        }

        rtnl_lock();

        tun = tun_get(tfile);
        if (cmd == TUNSETIFF) {
                ret = -EEXIST;
                if (tun)
                        goto unlock;

                ifr.ifr_name[IFNAMSIZ-1] = '\0';

                ret = tun_set_iff(net, file, &ifr);

                if (ret)
                        goto unlock;

                if (copy_to_user(argp, &ifr, ifreq_len))
                        ret = -EFAULT;
                goto unlock;
        }
        if (cmd == TUNSETIFINDEX) {
                ret = -EPERM;
                if (tun)
                        goto unlock;

                ret = -EFAULT;
                if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
                        goto unlock;
                ret = -EINVAL;
                if (ifindex < 0)
                        goto unlock;
                ret = 0;
                tfile->ifindex = ifindex;
                goto unlock;
        }

        ret = -EBADFD;
        if (!tun)
                goto unlock;

        netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd);

        net = dev_net(tun->dev);
        ret = 0;
        switch (cmd) {
        case TUNGETIFF:
                tun_get_iff(tun, &ifr);

                if (tfile->detached)
                        ifr.ifr_flags |= IFF_DETACH_QUEUE;
                if (!tfile->socket.sk->sk_filter)
                        ifr.ifr_flags |= IFF_NOFILTER;

                if (copy_to_user(argp, &ifr, ifreq_len))
                        ret = -EFAULT;
                break;

        case TUNSETNOCSUM:
                /* Disable/Enable checksum */

                /* [unimplemented] */
                netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n",
                           arg ? "disabled" : "enabled");
                break;

        case TUNSETPERSIST:
                /* Disable/Enable persist mode. Keep an extra reference to the
                 * module to prevent the module being unprobed.
                 */
                if (arg && !(tun->flags & IFF_PERSIST)) {
                        tun->flags |= IFF_PERSIST;
                        __module_get(THIS_MODULE);
                        do_notify = true;
                }
                if (!arg && (tun->flags & IFF_PERSIST)) {
                        tun->flags &= ~IFF_PERSIST;
                        module_put(THIS_MODULE);
                        do_notify = true;
                }

                netif_info(tun, drv, tun->dev, "persist %s\n",
                           arg ? "enabled" : "disabled");
                break;

        case TUNSETOWNER:
                /* Set owner of the device */
                owner = make_kuid(current_user_ns(), arg);
                if (!uid_valid(owner)) {
                        ret = -EINVAL;
                        break;
                }
                tun->owner = owner;
                do_notify = true;
                netif_info(tun, drv, tun->dev, "owner set to %u\n",
                           from_kuid(&init_user_ns, tun->owner));
                break;

        case TUNSETGROUP:
                /* Set group of the device */
                group = make_kgid(current_user_ns(), arg);
                if (!gid_valid(group)) {
                        ret = -EINVAL;
                        break;
                }
                tun->group = group;
                do_notify = true;
                netif_info(tun, drv, tun->dev, "group set to %u\n",
                           from_kgid(&init_user_ns, tun->group));
                break;

        case TUNSETLINK:
                /* Only allow setting the type when the interface is down */
                if (tun->dev->flags & IFF_UP) {
                        netif_info(tun, drv, tun->dev,
                                   "Linktype set failed because interface is up\n");
                        ret = -EBUSY;
                } else {
                        ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE,
                                                       tun->dev);
                        ret = notifier_to_errno(ret);
                        if (ret) {
                                netif_info(tun, drv, tun->dev,
                                           "Refused to change device type\n");
                                break;
                        }
                        tun->dev->type = (int) arg;
                        tun->dev->addr_len = tun_get_addr_len(tun->dev->type);
                        netif_info(tun, drv, tun->dev, "linktype set to %d\n",
                                   tun->dev->type);
                        call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE,
                                                 tun->dev);
                }
                break;

        case TUNSETDEBUG:
                tun->msg_enable = (u32)arg;
                break;

        case TUNSETOFFLOAD:
                ret = set_offload(tun, arg);
                break;

        case TUNSETTXFILTER:
                /* Can be set only for TAPs */
                ret = -EINVAL;
                if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
                        break;
                ret = update_filter(&tun->txflt, (void __user *)arg);
                break;

        case SIOCGIFHWADDR:
                /* Get hw address */
                dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name);
                if (copy_to_user(argp, &ifr, ifreq_len))
                        ret = -EFAULT;
                break;

        case SIOCSIFHWADDR:
                /* Set hw address */
                ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL);
                break;

        case TUNGETSNDBUF:
                sndbuf = tfile->socket.sk->sk_sndbuf;
                if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
                        ret = -EFAULT;
                break;

        case TUNSETSNDBUF:
                if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
                        ret = -EFAULT;
                        break;
                }
                if (sndbuf <= 0) {
                        ret = -EINVAL;
                        break;
                }

                tun->sndbuf = sndbuf;
                tun_set_sndbuf(tun);
                break;

        case TUNGETVNETHDRSZ:
                vnet_hdr_sz = tun->vnet_hdr_sz;
                if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz)))
                        ret = -EFAULT;
                break;

        case TUNSETVNETHDRSZ:
                if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) {
                        ret = -EFAULT;
                        break;
                }
                if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) {
                        ret = -EINVAL;
                        break;
                }

                tun->vnet_hdr_sz = vnet_hdr_sz;
                break;

        case TUNGETVNETLE:
                le = !!(tun->flags & TUN_VNET_LE);
                if (put_user(le, (int __user *)argp))
                        ret = -EFAULT;
                break;

        case TUNSETVNETLE:
                if (get_user(le, (int __user *)argp)) {
                        ret = -EFAULT;
                        break;
                }
                if (le)
                        tun->flags |= TUN_VNET_LE;
                else
                        tun->flags &= ~TUN_VNET_LE;
                break;

        case TUNGETVNETBE:
                ret = tun_get_vnet_be(tun, argp);
                break;

        case TUNSETVNETBE:
                ret = tun_set_vnet_be(tun, argp);
                break;

        case TUNATTACHFILTER:
                /* Can be set only for TAPs */
                ret = -EINVAL;
                if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
                        break;
                ret = -EFAULT;
                if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
                        break;

                ret = tun_attach_filter(tun);
                break;

        case TUNDETACHFILTER:
                /* Can be set only for TAPs */
                ret = -EINVAL;
                if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
                        break;
                ret = 0;
                tun_detach_filter(tun, tun->numqueues);
                break;

        case TUNGETFILTER:
                ret = -EINVAL;
                if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
                        break;
                ret = -EFAULT;
                if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
                        break;
                ret = 0;
                break;

        case TUNSETSTEERINGEBPF:
                ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
                break;

        case TUNSETFILTEREBPF:
                ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
                break;

        case TUNSETCARRIER:
                ret = -EFAULT;
                if (copy_from_user(&carrier, argp, sizeof(carrier)))
                        goto unlock;

                ret = tun_net_change_carrier(tun->dev, (bool)carrier);
                break;

        case TUNGETDEVNETNS:
                ret = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto unlock;
                ret = open_related_ns(&net->ns, get_net_ns);
                break;

        default:
                ret = -EINVAL;
                break;
        }

        if (do_notify)
                netdev_state_change(tun->dev);

unlock:
        rtnl_unlock();
        if (tun)
                tun_put(tun);
        return ret;
}

static long tun_chr_ioctl(struct file *file,
                          unsigned int cmd, unsigned long arg)
{
        return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
}

#ifdef CONFIG_COMPAT
static long tun_chr_compat_ioctl(struct file *file,
                         unsigned int cmd, unsigned long arg)
{
        switch (cmd) {
        case TUNSETIFF:
        case TUNGETIFF:
        case TUNSETTXFILTER:
        case TUNGETSNDBUF:
        case TUNSETSNDBUF:
        case SIOCGIFHWADDR:
        case SIOCSIFHWADDR:
                arg = (unsigned long)compat_ptr(arg);
                break;
        default:
                arg = (compat_ulong_t)arg;
                break;
        }

        /*
         * compat_ifreq is shorter than ifreq, so we must not access beyond
         * the end of that structure. All fields that are used in this
         * driver are compatible though, we don't need to convert the
         * contents.
         */
        return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
}
#endif /* CONFIG_COMPAT */

static int tun_chr_fasync(int fd, struct file *file, int on)
{
        struct tun_file *tfile = file->private_data;
        int ret;

        if (on) {
                ret = file_f_owner_allocate(file);
                if (ret)
                        goto out;
        }

        if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
                goto out;

        if (on) {
                __f_setown(file, task_pid(current), PIDTYPE_TGID, 0);
                tfile->flags |= TUN_FASYNC;
        } else
                tfile->flags &= ~TUN_FASYNC;
        ret = 0;
out:
        return ret;
}

static int tun_chr_open(struct inode *inode, struct file * file)
{
        struct net *net = current->nsproxy->net_ns;
        struct tun_file *tfile;

        tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
                                            &tun_proto, 0);
        if (!tfile)
                return -ENOMEM;
        if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
                sk_free(&tfile->sk);
                return -ENOMEM;
        }

        mutex_init(&tfile->napi_mutex);
        RCU_INIT_POINTER(tfile->tun, NULL);
        tfile->flags = 0;
        tfile->ifindex = 0;

        init_waitqueue_head(&tfile->socket.wq.wait);

        tfile->socket.file = file;
        tfile->socket.ops = &tun_socket_ops;

        sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid());

        tfile->sk.sk_write_space = tun_sock_write_space;
        tfile->sk.sk_sndbuf = INT_MAX;

        file->private_data = tfile;
        INIT_LIST_HEAD(&tfile->next);

        sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);

        /* tun groks IOCB_NOWAIT just fine, mark it as such */
        file->f_mode |= FMODE_NOWAIT;
        return 0;
}

static int tun_chr_close(struct inode *inode, struct file *file)
{
        struct tun_file *tfile = file->private_data;

        tun_detach(tfile, true);

        return 0;
}

#ifdef CONFIG_PROC_FS
static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
{
        struct tun_file *tfile = file->private_data;
        struct tun_struct *tun;
        struct ifreq ifr;

        memset(&ifr, 0, sizeof(ifr));

        rtnl_lock();
        tun = tun_get(tfile);
        if (tun)
                tun_get_iff(tun, &ifr);
        rtnl_unlock();

        if (tun)
                tun_put(tun);

        seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
}
#endif

static const struct file_operations tun_fops = {
        .owner        = THIS_MODULE,
        .read_iter  = tun_chr_read_iter,
        .write_iter = tun_chr_write_iter,
        .poll        = tun_chr_poll,
        .unlocked_ioctl        = tun_chr_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = tun_chr_compat_ioctl,
#endif
        .open        = tun_chr_open,
        .release = tun_chr_close,
        .fasync = tun_chr_fasync,
#ifdef CONFIG_PROC_FS
        .show_fdinfo = tun_chr_show_fdinfo,
#endif
};

static struct miscdevice tun_miscdev = {
        .minor = TUN_MINOR,
        .name = "tun",
        .nodename = "net/tun",
        .fops = &tun_fops,
};

/* ethtool interface */

static void tun_default_link_ksettings(struct net_device *dev,
                                       struct ethtool_link_ksettings *cmd)
{
        ethtool_link_ksettings_zero_link_mode(cmd, supported);
        ethtool_link_ksettings_zero_link_mode(cmd, advertising);
        cmd->base.speed                = SPEED_10000;
        cmd->base.duplex        = DUPLEX_FULL;
        cmd->base.port                = PORT_TP;
        cmd->base.phy_address        = 0;
        cmd->base.autoneg        = AUTONEG_DISABLE;
}

static int tun_get_link_ksettings(struct net_device *dev,
                                  struct ethtool_link_ksettings *cmd)
{
        struct tun_struct *tun = netdev_priv(dev);

        memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));
        return 0;
}

static int tun_set_link_ksettings(struct net_device *dev,
                                  const struct ethtool_link_ksettings *cmd)
{
        struct tun_struct *tun = netdev_priv(dev);

        memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));
        return 0;
}

static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
{
        struct tun_struct *tun = netdev_priv(dev);

        strscpy(info->driver, DRV_NAME, sizeof(info->driver));
        strscpy(info->version, DRV_VERSION, sizeof(info->version));

        switch (tun->flags & TUN_TYPE_MASK) {
        case IFF_TUN:
                strscpy(info->bus_info, "tun", sizeof(info->bus_info));
                break;
        case IFF_TAP:
                strscpy(info->bus_info, "tap", sizeof(info->bus_info));
                break;
        }
}

static u32 tun_get_msglevel(struct net_device *dev)
{
        struct tun_struct *tun = netdev_priv(dev);

        return tun->msg_enable;
}

static void tun_set_msglevel(struct net_device *dev, u32 value)
{
        struct tun_struct *tun = netdev_priv(dev);

        tun->msg_enable = value;
}

static int tun_get_coalesce(struct net_device *dev,
                            struct ethtool_coalesce *ec,
                            struct kernel_ethtool_coalesce *kernel_coal,
                            struct netlink_ext_ack *extack)
{
        struct tun_struct *tun = netdev_priv(dev);

        ec->rx_max_coalesced_frames = tun->rx_batched;

        return 0;
}

static int tun_set_coalesce(struct net_device *dev,
                            struct ethtool_coalesce *ec,
                            struct kernel_ethtool_coalesce *kernel_coal,
                            struct netlink_ext_ack *extack)
{
        struct tun_struct *tun = netdev_priv(dev);

        if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
                tun->rx_batched = NAPI_POLL_WEIGHT;
        else
                tun->rx_batched = ec->rx_max_coalesced_frames;

        return 0;
}

static void tun_get_channels(struct net_device *dev,
                             struct ethtool_channels *channels)
{
        struct tun_struct *tun = netdev_priv(dev);

        channels->combined_count = tun->numqueues;
        channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1;
}

static const struct ethtool_ops tun_ethtool_ops = {
        .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES,
        .get_drvinfo        = tun_get_drvinfo,
        .get_msglevel        = tun_get_msglevel,
        .set_msglevel        = tun_set_msglevel,
        .get_link        = ethtool_op_get_link,
        .get_channels   = tun_get_channels,
        .get_ts_info        = ethtool_op_get_ts_info,
        .get_coalesce   = tun_get_coalesce,
        .set_coalesce   = tun_set_coalesce,
        .get_link_ksettings = tun_get_link_ksettings,
        .set_link_ksettings = tun_set_link_ksettings,
};

static int tun_queue_resize(struct tun_struct *tun)
{
        struct net_device *dev = tun->dev;
        struct tun_file *tfile;
        struct ptr_ring **rings;
        int n = tun->numqueues + tun->numdisabled;
        int ret, i;

        rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
        if (!rings)
                return -ENOMEM;

        for (i = 0; i < tun->numqueues; i++) {
                tfile = rtnl_dereference(tun->tfiles[i]);
                rings[i] = &tfile->tx_ring;
        }
        list_for_each_entry(tfile, &tun->disabled, next)
                rings[i++] = &tfile->tx_ring;

        ret = ptr_ring_resize_multiple(rings, n,
                                       dev->tx_queue_len, GFP_KERNEL,
                                       tun_ptr_free);

        kfree(rings);
        return ret;
}

static int tun_device_event(struct notifier_block *unused,
                            unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct tun_struct *tun = netdev_priv(dev);
        int i;

        if (dev->rtnl_link_ops != &tun_link_ops)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_CHANGE_TX_QUEUE_LEN:
                if (tun_queue_resize(tun))
                        return NOTIFY_BAD;
                break;
        case NETDEV_UP:
                for (i = 0; i < tun->numqueues; i++) {
                        struct tun_file *tfile;

                        tfile = rtnl_dereference(tun->tfiles[i]);
                        tfile->socket.sk->sk_write_space(tfile->socket.sk);
                }
                break;
        default:
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block tun_notifier_block __read_mostly = {
        .notifier_call        = tun_device_event,
};

static int __init tun_init(void)
{
        int ret = 0;

        pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);

        ret = rtnl_link_register(&tun_link_ops);
        if (ret) {
                pr_err("Can't register link_ops\n");
                goto err_linkops;
        }

        ret = misc_register(&tun_miscdev);
        if (ret) {
                pr_err("Can't register misc device %d\n", TUN_MINOR);
                goto err_misc;
        }

        ret = register_netdevice_notifier(&tun_notifier_block);
        if (ret) {
                pr_err("Can't register netdevice notifier\n");
                goto err_notifier;
        }

        return  0;

err_notifier:
        misc_deregister(&tun_miscdev);
err_misc:
        rtnl_link_unregister(&tun_link_ops);
err_linkops:
        return ret;
}

static void __exit tun_cleanup(void)
{
        misc_deregister(&tun_miscdev);
        rtnl_link_unregister(&tun_link_ops);
        unregister_netdevice_notifier(&tun_notifier_block);
}

/* Get an underlying socket object from tun file.  Returns error unless file is
 * attached to a device.  The returned object works like a packet socket, it
 * can be used for sock_sendmsg/sock_recvmsg.  The caller is responsible for
 * holding a reference to the file for as long as the socket is in use. */
struct socket *tun_get_socket(struct file *file)
{
        struct tun_file *tfile;
        if (file->f_op != &tun_fops)
                return ERR_PTR(-EINVAL);
        tfile = file->private_data;
        if (!tfile)
                return ERR_PTR(-EBADFD);
        return &tfile->socket;
}
EXPORT_SYMBOL_GPL(tun_get_socket);

struct ptr_ring *tun_get_tx_ring(struct file *file)
{
        struct tun_file *tfile;

        if (file->f_op != &tun_fops)
                return ERR_PTR(-EINVAL);
        tfile = file->private_data;
        if (!tfile)
                return ERR_PTR(-EBADFD);
        return &tfile->tx_ring;
}
EXPORT_SYMBOL_GPL(tun_get_tx_ring);

module_init(tun_init);
module_exit(tun_cleanup);
MODULE_DESCRIPTION(DRV_DESCRIPTION);
MODULE_AUTHOR(DRV_COPYRIGHT);
MODULE_LICENSE("GPL");
MODULE_ALIAS_MISCDEV(TUN_MINOR);
MODULE_ALIAS("devname:net/tun");



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __IEEE802154_CORE_H
#define __IEEE802154_CORE_H

#include <net/cfg802154.h>

struct cfg802154_registered_device {
        const struct cfg802154_ops *ops;
        struct list_head list;

        /* wpan_phy index, internal only */
        int wpan_phy_idx;

        /* also protected by devlist_mtx */
        int opencount;
        wait_queue_head_t dev_wait;

        /* protected by RTNL only */
        int num_running_ifaces;

        /* associated wpan interfaces, protected by rtnl or RCU */
        struct list_head wpan_dev_list;
        int devlist_generation, wpan_dev_id;

        /* must be last because of the way we do wpan_phy_priv(),
         * and it should at least be aligned to NETDEV_ALIGN
         */
        struct wpan_phy wpan_phy __aligned(NETDEV_ALIGN);
};

static inline struct cfg802154_registered_device *
wpan_phy_to_rdev(struct wpan_phy *wpan_phy)
{
        BUG_ON(!wpan_phy);
        return container_of(wpan_phy, struct cfg802154_registered_device,
                            wpan_phy);
}

extern struct list_head cfg802154_rdev_list;
extern int cfg802154_rdev_list_generation;

int cfg802154_switch_netns(struct cfg802154_registered_device *rdev,
                           struct net *net);
/* free object */
void cfg802154_dev_free(struct cfg802154_registered_device *rdev);
struct cfg802154_registered_device *
cfg802154_rdev_by_wpan_phy_idx(int wpan_phy_idx);
struct wpan_phy *wpan_phy_idx_to_wpan_phy(int wpan_phy_idx);

#endif /* __IEEE802154_CORE_H */









































































































































































































    3 










































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (c) 2020, Nikolay Aleksandrov <nikolay@nvidia.com>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/if_ether.h>
#include <linux/igmp.h>
#include <linux/in.h>
#include <linux/jhash.h>
#include <linux/kernel.h>
#include <linux/log2.h>
#include <linux/netdevice.h>
#include <linux/netfilter_bridge.h>
#include <linux/random.h>
#include <linux/rculist.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/inetdevice.h>
#include <linux/mroute.h>
#include <net/ip.h>
#include <net/switchdev.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <linux/icmpv6.h>
#include <net/ipv6.h>
#include <net/mld.h>
#include <net/ip6_checksum.h>
#include <net/addrconf.h>
#endif

#include "br_private.h"
#include "br_private_mcast_eht.h"

static bool br_multicast_del_eht_set_entry(struct net_bridge_port_group *pg,
                                           union net_bridge_eht_addr *src_addr,
                                           union net_bridge_eht_addr *h_addr);
static void br_multicast_create_eht_set_entry(const struct net_bridge_mcast *brmctx,
                                              struct net_bridge_port_group *pg,
                                              union net_bridge_eht_addr *src_addr,
                                              union net_bridge_eht_addr *h_addr,
                                              int filter_mode,
                                              bool allow_zero_src);

static struct net_bridge_group_eht_host *
br_multicast_eht_host_lookup(struct net_bridge_port_group *pg,
                             union net_bridge_eht_addr *h_addr)
{
        struct rb_node *node = pg->eht_host_tree.rb_node;

        while (node) {
                struct net_bridge_group_eht_host *this;
                int result;

                this = rb_entry(node, struct net_bridge_group_eht_host,
                                rb_node);
                result = memcmp(h_addr, &this->h_addr, sizeof(*h_addr));
                if (result < 0)
                        node = node->rb_left;
                else if (result > 0)
                        node = node->rb_right;
                else
                        return this;
        }

        return NULL;
}

static int br_multicast_eht_host_filter_mode(struct net_bridge_port_group *pg,
                                             union net_bridge_eht_addr *h_addr)
{
        struct net_bridge_group_eht_host *eht_host;

        eht_host = br_multicast_eht_host_lookup(pg, h_addr);
        if (!eht_host)
                return MCAST_INCLUDE;

        return eht_host->filter_mode;
}

static struct net_bridge_group_eht_set_entry *
br_multicast_eht_set_entry_lookup(struct net_bridge_group_eht_set *eht_set,
                                  union net_bridge_eht_addr *h_addr)
{
        struct rb_node *node = eht_set->entry_tree.rb_node;

        while (node) {
                struct net_bridge_group_eht_set_entry *this;
                int result;

                this = rb_entry(node, struct net_bridge_group_eht_set_entry,
                                rb_node);
                result = memcmp(h_addr, &this->h_addr, sizeof(*h_addr));
                if (result < 0)
                        node = node->rb_left;
                else if (result > 0)
                        node = node->rb_right;
                else
                        return this;
        }

        return NULL;
}

static struct net_bridge_group_eht_set *
br_multicast_eht_set_lookup(struct net_bridge_port_group *pg,
                            union net_bridge_eht_addr *src_addr)
{
        struct rb_node *node = pg->eht_set_tree.rb_node;

        while (node) {
                struct net_bridge_group_eht_set *this;
                int result;

                this = rb_entry(node, struct net_bridge_group_eht_set,
                                rb_node);
                result = memcmp(src_addr, &this->src_addr, sizeof(*src_addr));
                if (result < 0)
                        node = node->rb_left;
                else if (result > 0)
                        node = node->rb_right;
                else
                        return this;
        }

        return NULL;
}

static void __eht_destroy_host(struct net_bridge_group_eht_host *eht_host)
{
        WARN_ON(!hlist_empty(&eht_host->set_entries));

        br_multicast_eht_hosts_dec(eht_host->pg);

        rb_erase(&eht_host->rb_node, &eht_host->pg->eht_host_tree);
        RB_CLEAR_NODE(&eht_host->rb_node);
        kfree(eht_host);
}

static void br_multicast_destroy_eht_set_entry(struct net_bridge_mcast_gc *gc)
{
        struct net_bridge_group_eht_set_entry *set_h;

        set_h = container_of(gc, struct net_bridge_group_eht_set_entry, mcast_gc);
        WARN_ON(!RB_EMPTY_NODE(&set_h->rb_node));

        timer_shutdown_sync(&set_h->timer);
        kfree(set_h);
}

static void br_multicast_destroy_eht_set(struct net_bridge_mcast_gc *gc)
{
        struct net_bridge_group_eht_set *eht_set;

        eht_set = container_of(gc, struct net_bridge_group_eht_set, mcast_gc);
        WARN_ON(!RB_EMPTY_NODE(&eht_set->rb_node));
        WARN_ON(!RB_EMPTY_ROOT(&eht_set->entry_tree));

        timer_shutdown_sync(&eht_set->timer);
        kfree(eht_set);
}

static void __eht_del_set_entry(struct net_bridge_group_eht_set_entry *set_h)
{
        struct net_bridge_group_eht_host *eht_host = set_h->h_parent;
        union net_bridge_eht_addr zero_addr;

        rb_erase(&set_h->rb_node, &set_h->eht_set->entry_tree);
        RB_CLEAR_NODE(&set_h->rb_node);
        hlist_del_init(&set_h->host_list);
        memset(&zero_addr, 0, sizeof(zero_addr));
        if (memcmp(&set_h->h_addr, &zero_addr, sizeof(zero_addr)))
                eht_host->num_entries--;
        hlist_add_head(&set_h->mcast_gc.gc_node, &set_h->br->mcast_gc_list);
        queue_work(system_long_wq, &set_h->br->mcast_gc_work);

        if (hlist_empty(&eht_host->set_entries))
                __eht_destroy_host(eht_host);
}

static void br_multicast_del_eht_set(struct net_bridge_group_eht_set *eht_set)
{
        struct net_bridge_group_eht_set_entry *set_h;
        struct rb_node *node;

        while ((node = rb_first(&eht_set->entry_tree))) {
                set_h = rb_entry(node, struct net_bridge_group_eht_set_entry,
                                 rb_node);
                __eht_del_set_entry(set_h);
        }

        rb_erase(&eht_set->rb_node, &eht_set->pg->eht_set_tree);
        RB_CLEAR_NODE(&eht_set->rb_node);
        hlist_add_head(&eht_set->mcast_gc.gc_node, &eht_set->br->mcast_gc_list);
        queue_work(system_long_wq, &eht_set->br->mcast_gc_work);
}

void br_multicast_eht_clean_sets(struct net_bridge_port_group *pg)
{
        struct net_bridge_group_eht_set *eht_set;
        struct rb_node *node;

        while ((node = rb_first(&pg->eht_set_tree))) {
                eht_set = rb_entry(node, struct net_bridge_group_eht_set,
                                   rb_node);
                br_multicast_del_eht_set(eht_set);
        }
}

static void br_multicast_eht_set_entry_expired(struct timer_list *t)
{
        struct net_bridge_group_eht_set_entry *set_h = from_timer(set_h, t, timer);
        struct net_bridge *br = set_h->br;

        spin_lock(&br->multicast_lock);
        if (RB_EMPTY_NODE(&set_h->rb_node) || timer_pending(&set_h->timer))
                goto out;

        br_multicast_del_eht_set_entry(set_h->eht_set->pg,
                                       &set_h->eht_set->src_addr,
                                       &set_h->h_addr);
out:
        spin_unlock(&br->multicast_lock);
}

static void br_multicast_eht_set_expired(struct timer_list *t)
{
        struct net_bridge_group_eht_set *eht_set = from_timer(eht_set, t,
                                                              timer);
        struct net_bridge *br = eht_set->br;

        spin_lock(&br->multicast_lock);
        if (RB_EMPTY_NODE(&eht_set->rb_node) || timer_pending(&eht_set->timer))
                goto out;

        br_multicast_del_eht_set(eht_set);
out:
        spin_unlock(&br->multicast_lock);
}

static struct net_bridge_group_eht_host *
__eht_lookup_create_host(struct net_bridge_port_group *pg,
                         union net_bridge_eht_addr *h_addr,
                         unsigned char filter_mode)
{
        struct rb_node **link = &pg->eht_host_tree.rb_node, *parent = NULL;
        struct net_bridge_group_eht_host *eht_host;

        while (*link) {
                struct net_bridge_group_eht_host *this;
                int result;

                this = rb_entry(*link, struct net_bridge_group_eht_host,
                                rb_node);
                result = memcmp(h_addr, &this->h_addr, sizeof(*h_addr));
                parent = *link;
                if (result < 0)
                        link = &((*link)->rb_left);
                else if (result > 0)
                        link = &((*link)->rb_right);
                else
                        return this;
        }

        if (br_multicast_eht_hosts_over_limit(pg))
                return NULL;

        eht_host = kzalloc(sizeof(*eht_host), GFP_ATOMIC);
        if (!eht_host)
                return NULL;

        memcpy(&eht_host->h_addr, h_addr, sizeof(*h_addr));
        INIT_HLIST_HEAD(&eht_host->set_entries);
        eht_host->pg = pg;
        eht_host->filter_mode = filter_mode;

        rb_link_node(&eht_host->rb_node, parent, link);
        rb_insert_color(&eht_host->rb_node, &pg->eht_host_tree);

        br_multicast_eht_hosts_inc(pg);

        return eht_host;
}

static struct net_bridge_group_eht_set_entry *
__eht_lookup_create_set_entry(struct net_bridge *br,
                              struct net_bridge_group_eht_set *eht_set,
                              struct net_bridge_group_eht_host *eht_host,
                              bool allow_zero_src)
{
        struct rb_node **link = &eht_set->entry_tree.rb_node, *parent = NULL;
        struct net_bridge_group_eht_set_entry *set_h;

        while (*link) {
                struct net_bridge_group_eht_set_entry *this;
                int result;

                this = rb_entry(*link, struct net_bridge_group_eht_set_entry,
                                rb_node);
                result = memcmp(&eht_host->h_addr, &this->h_addr,
                                sizeof(union net_bridge_eht_addr));
                parent = *link;
                if (result < 0)
                        link = &((*link)->rb_left);
                else if (result > 0)
                        link = &((*link)->rb_right);
                else
                        return this;
        }

        /* always allow auto-created zero entry */
        if (!allow_zero_src && eht_host->num_entries >= PG_SRC_ENT_LIMIT)
                return NULL;

        set_h = kzalloc(sizeof(*set_h), GFP_ATOMIC);
        if (!set_h)
                return NULL;

        memcpy(&set_h->h_addr, &eht_host->h_addr,
               sizeof(union net_bridge_eht_addr));
        set_h->mcast_gc.destroy = br_multicast_destroy_eht_set_entry;
        set_h->eht_set = eht_set;
        set_h->h_parent = eht_host;
        set_h->br = br;
        timer_setup(&set_h->timer, br_multicast_eht_set_entry_expired, 0);

        hlist_add_head(&set_h->host_list, &eht_host->set_entries);
        rb_link_node(&set_h->rb_node, parent, link);
        rb_insert_color(&set_h->rb_node, &eht_set->entry_tree);
        /* we must not count the auto-created zero entry otherwise we won't be
         * able to track the full list of PG_SRC_ENT_LIMIT entries
         */
        if (!allow_zero_src)
                eht_host->num_entries++;

        return set_h;
}

static struct net_bridge_group_eht_set *
__eht_lookup_create_set(struct net_bridge_port_group *pg,
                        union net_bridge_eht_addr *src_addr)
{
        struct rb_node **link = &pg->eht_set_tree.rb_node, *parent = NULL;
        struct net_bridge_group_eht_set *eht_set;

        while (*link) {
                struct net_bridge_group_eht_set *this;
                int result;

                this = rb_entry(*link, struct net_bridge_group_eht_set,
                                rb_node);
                result = memcmp(src_addr, &this->src_addr, sizeof(*src_addr));
                parent = *link;
                if (result < 0)
                        link = &((*link)->rb_left);
                else if (result > 0)
                        link = &((*link)->rb_right);
                else
                        return this;
        }

        eht_set = kzalloc(sizeof(*eht_set), GFP_ATOMIC);
        if (!eht_set)
                return NULL;

        memcpy(&eht_set->src_addr, src_addr, sizeof(*src_addr));
        eht_set->mcast_gc.destroy = br_multicast_destroy_eht_set;
        eht_set->pg = pg;
        eht_set->br = pg->key.port->br;
        eht_set->entry_tree = RB_ROOT;
        timer_setup(&eht_set->timer, br_multicast_eht_set_expired, 0);

        rb_link_node(&eht_set->rb_node, parent, link);
        rb_insert_color(&eht_set->rb_node, &pg->eht_set_tree);

        return eht_set;
}

static void br_multicast_ip_src_to_eht_addr(const struct br_ip *src,
                                            union net_bridge_eht_addr *dest)
{
        switch (src->proto) {
        case htons(ETH_P_IP):
                dest->ip4 = src->src.ip4;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                memcpy(&dest->ip6, &src->src.ip6, sizeof(struct in6_addr));
                break;
#endif
        }
}

static void br_eht_convert_host_filter_mode(const struct net_bridge_mcast *brmctx,
                                            struct net_bridge_port_group *pg,
                                            union net_bridge_eht_addr *h_addr,
                                            int filter_mode)
{
        struct net_bridge_group_eht_host *eht_host;
        union net_bridge_eht_addr zero_addr;

        eht_host = br_multicast_eht_host_lookup(pg, h_addr);
        if (eht_host)
                eht_host->filter_mode = filter_mode;

        memset(&zero_addr, 0, sizeof(zero_addr));
        switch (filter_mode) {
        case MCAST_INCLUDE:
                br_multicast_del_eht_set_entry(pg, &zero_addr, h_addr);
                break;
        case MCAST_EXCLUDE:
                br_multicast_create_eht_set_entry(brmctx, pg, &zero_addr,
                                                  h_addr, MCAST_EXCLUDE,
                                                  true);
                break;
        }
}

static void br_multicast_create_eht_set_entry(const struct net_bridge_mcast *brmctx,
                                              struct net_bridge_port_group *pg,
                                              union net_bridge_eht_addr *src_addr,
                                              union net_bridge_eht_addr *h_addr,
                                              int filter_mode,
                                              bool allow_zero_src)
{
        struct net_bridge_group_eht_set_entry *set_h;
        struct net_bridge_group_eht_host *eht_host;
        struct net_bridge *br = pg->key.port->br;
        struct net_bridge_group_eht_set *eht_set;
        union net_bridge_eht_addr zero_addr;

        memset(&zero_addr, 0, sizeof(zero_addr));
        if (!allow_zero_src && !memcmp(src_addr, &zero_addr, sizeof(zero_addr)))
                return;

        eht_set = __eht_lookup_create_set(pg, src_addr);
        if (!eht_set)
                return;

        eht_host = __eht_lookup_create_host(pg, h_addr, filter_mode);
        if (!eht_host)
                goto fail_host;

        set_h = __eht_lookup_create_set_entry(br, eht_set, eht_host,
                                              allow_zero_src);
        if (!set_h)
                goto fail_set_entry;

        mod_timer(&set_h->timer, jiffies + br_multicast_gmi(brmctx));
        mod_timer(&eht_set->timer, jiffies + br_multicast_gmi(brmctx));

        return;

fail_set_entry:
        if (hlist_empty(&eht_host->set_entries))
                __eht_destroy_host(eht_host);
fail_host:
        if (RB_EMPTY_ROOT(&eht_set->entry_tree))
                br_multicast_del_eht_set(eht_set);
}

static bool br_multicast_del_eht_set_entry(struct net_bridge_port_group *pg,
                                           union net_bridge_eht_addr *src_addr,
                                           union net_bridge_eht_addr *h_addr)
{
        struct net_bridge_group_eht_set_entry *set_h;
        struct net_bridge_group_eht_set *eht_set;
        bool set_deleted = false;

        eht_set = br_multicast_eht_set_lookup(pg, src_addr);
        if (!eht_set)
                goto out;

        set_h = br_multicast_eht_set_entry_lookup(eht_set, h_addr);
        if (!set_h)
                goto out;

        __eht_del_set_entry(set_h);

        if (RB_EMPTY_ROOT(&eht_set->entry_tree)) {
                br_multicast_del_eht_set(eht_set);
                set_deleted = true;
        }

out:
        return set_deleted;
}

static void br_multicast_del_eht_host(struct net_bridge_port_group *pg,
                                      union net_bridge_eht_addr *h_addr)
{
        struct net_bridge_group_eht_set_entry *set_h;
        struct net_bridge_group_eht_host *eht_host;
        struct hlist_node *tmp;

        eht_host = br_multicast_eht_host_lookup(pg, h_addr);
        if (!eht_host)
                return;

        hlist_for_each_entry_safe(set_h, tmp, &eht_host->set_entries, host_list)
                br_multicast_del_eht_set_entry(set_h->eht_set->pg,
                                               &set_h->eht_set->src_addr,
                                               &set_h->h_addr);
}

/* create new set entries from reports */
static void __eht_create_set_entries(const struct net_bridge_mcast *brmctx,
                                     struct net_bridge_port_group *pg,
                                     union net_bridge_eht_addr *h_addr,
                                     void *srcs,
                                     u32 nsrcs,
                                     size_t addr_size,
                                     int filter_mode)
{
        union net_bridge_eht_addr eht_src_addr;
        u32 src_idx;

        memset(&eht_src_addr, 0, sizeof(eht_src_addr));
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&eht_src_addr, srcs + (src_idx * addr_size), addr_size);
                br_multicast_create_eht_set_entry(brmctx, pg, &eht_src_addr,
                                                  h_addr, filter_mode,
                                                  false);
        }
}

/* delete existing set entries and their (S,G) entries if they were the last */
static bool __eht_del_set_entries(struct net_bridge_port_group *pg,
                                  union net_bridge_eht_addr *h_addr,
                                  void *srcs,
                                  u32 nsrcs,
                                  size_t addr_size)
{
        union net_bridge_eht_addr eht_src_addr;
        struct net_bridge_group_src *src_ent;
        bool changed = false;
        struct br_ip src_ip;
        u32 src_idx;

        memset(&eht_src_addr, 0, sizeof(eht_src_addr));
        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&eht_src_addr, srcs + (src_idx * addr_size), addr_size);
                if (!br_multicast_del_eht_set_entry(pg, &eht_src_addr, h_addr))
                        continue;
                memcpy(&src_ip, srcs + (src_idx * addr_size), addr_size);
                src_ent = br_multicast_find_group_src(pg, &src_ip);
                if (!src_ent)
                        continue;
                br_multicast_del_group_src(src_ent, true);
                changed = true;
        }

        return changed;
}

static bool br_multicast_eht_allow(const struct net_bridge_mcast *brmctx,
                                   struct net_bridge_port_group *pg,
                                   union net_bridge_eht_addr *h_addr,
                                   void *srcs,
                                   u32 nsrcs,
                                   size_t addr_size)
{
        bool changed = false;

        switch (br_multicast_eht_host_filter_mode(pg, h_addr)) {
        case MCAST_INCLUDE:
                __eht_create_set_entries(brmctx, pg, h_addr, srcs, nsrcs,
                                         addr_size, MCAST_INCLUDE);
                break;
        case MCAST_EXCLUDE:
                changed = __eht_del_set_entries(pg, h_addr, srcs, nsrcs,
                                                addr_size);
                break;
        }

        return changed;
}

static bool br_multicast_eht_block(const struct net_bridge_mcast *brmctx,
                                   struct net_bridge_port_group *pg,
                                   union net_bridge_eht_addr *h_addr,
                                   void *srcs,
                                   u32 nsrcs,
                                   size_t addr_size)
{
        bool changed = false;

        switch (br_multicast_eht_host_filter_mode(pg, h_addr)) {
        case MCAST_INCLUDE:
                changed = __eht_del_set_entries(pg, h_addr, srcs, nsrcs,
                                                addr_size);
                break;
        case MCAST_EXCLUDE:
                __eht_create_set_entries(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                         MCAST_EXCLUDE);
                break;
        }

        return changed;
}

/* flush_entries is true when changing mode */
static bool __eht_inc_exc(const struct net_bridge_mcast *brmctx,
                          struct net_bridge_port_group *pg,
                          union net_bridge_eht_addr *h_addr,
                          void *srcs,
                          u32 nsrcs,
                          size_t addr_size,
                          unsigned char filter_mode,
                          bool to_report)
{
        bool changed = false, flush_entries = to_report;
        union net_bridge_eht_addr eht_src_addr;

        if (br_multicast_eht_host_filter_mode(pg, h_addr) != filter_mode)
                flush_entries = true;

        memset(&eht_src_addr, 0, sizeof(eht_src_addr));
        /* if we're changing mode del host and its entries */
        if (flush_entries)
                br_multicast_del_eht_host(pg, h_addr);
        __eht_create_set_entries(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                 filter_mode);
        /* we can be missing sets only if we've deleted some entries */
        if (flush_entries) {
                struct net_bridge_group_eht_set *eht_set;
                struct net_bridge_group_src *src_ent;
                struct hlist_node *tmp;

                hlist_for_each_entry_safe(src_ent, tmp, &pg->src_list, node) {
                        br_multicast_ip_src_to_eht_addr(&src_ent->addr,
                                                        &eht_src_addr);
                        if (!br_multicast_eht_set_lookup(pg, &eht_src_addr)) {
                                br_multicast_del_group_src(src_ent, true);
                                changed = true;
                                continue;
                        }
                        /* this is an optimization for TO_INCLUDE where we lower
                         * the set's timeout to LMQT to catch timeout hosts:
                         * - host A (timing out): set entries X, Y
                         * - host B: set entry Z (new from current TO_INCLUDE)
                         *           sends BLOCK Z after LMQT but host A's EHT
                         *           entries still exist (unless lowered to LMQT
                         *           so they can timeout with the S,Gs)
                         * => we wait another LMQT, when we can just delete the
                         *    group immediately
                         */
                        if (!(src_ent->flags & BR_SGRP_F_SEND) ||
                            filter_mode != MCAST_INCLUDE ||
                            !to_report)
                                continue;
                        eht_set = br_multicast_eht_set_lookup(pg,
                                                              &eht_src_addr);
                        if (!eht_set)
                                continue;
                        mod_timer(&eht_set->timer, jiffies + br_multicast_lmqt(brmctx));
                }
        }

        return changed;
}

static bool br_multicast_eht_inc(const struct net_bridge_mcast *brmctx,
                                 struct net_bridge_port_group *pg,
                                 union net_bridge_eht_addr *h_addr,
                                 void *srcs,
                                 u32 nsrcs,
                                 size_t addr_size,
                                 bool to_report)
{
        bool changed;

        changed = __eht_inc_exc(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                MCAST_INCLUDE, to_report);
        br_eht_convert_host_filter_mode(brmctx, pg, h_addr, MCAST_INCLUDE);

        return changed;
}

static bool br_multicast_eht_exc(const struct net_bridge_mcast *brmctx,
                                 struct net_bridge_port_group *pg,
                                 union net_bridge_eht_addr *h_addr,
                                 void *srcs,
                                 u32 nsrcs,
                                 size_t addr_size,
                                 bool to_report)
{
        bool changed;

        changed = __eht_inc_exc(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                MCAST_EXCLUDE, to_report);
        br_eht_convert_host_filter_mode(brmctx, pg, h_addr, MCAST_EXCLUDE);

        return changed;
}

static bool __eht_ip4_handle(const struct net_bridge_mcast *brmctx,
                             struct net_bridge_port_group *pg,
                             union net_bridge_eht_addr *h_addr,
                             void *srcs,
                             u32 nsrcs,
                             int grec_type)
{
        bool changed = false, to_report = false;

        switch (grec_type) {
        case IGMPV3_ALLOW_NEW_SOURCES:
                br_multicast_eht_allow(brmctx, pg, h_addr, srcs, nsrcs,
                                       sizeof(__be32));
                break;
        case IGMPV3_BLOCK_OLD_SOURCES:
                changed = br_multicast_eht_block(brmctx, pg, h_addr, srcs, nsrcs,
                                                 sizeof(__be32));
                break;
        case IGMPV3_CHANGE_TO_INCLUDE:
                to_report = true;
                fallthrough;
        case IGMPV3_MODE_IS_INCLUDE:
                changed = br_multicast_eht_inc(brmctx, pg, h_addr, srcs, nsrcs,
                                               sizeof(__be32), to_report);
                break;
        case IGMPV3_CHANGE_TO_EXCLUDE:
                to_report = true;
                fallthrough;
        case IGMPV3_MODE_IS_EXCLUDE:
                changed = br_multicast_eht_exc(brmctx, pg, h_addr, srcs, nsrcs,
                                               sizeof(__be32), to_report);
                break;
        }

        return changed;
}

#if IS_ENABLED(CONFIG_IPV6)
static bool __eht_ip6_handle(const struct net_bridge_mcast *brmctx,
                             struct net_bridge_port_group *pg,
                             union net_bridge_eht_addr *h_addr,
                             void *srcs,
                             u32 nsrcs,
                             int grec_type)
{
        bool changed = false, to_report = false;

        switch (grec_type) {
        case MLD2_ALLOW_NEW_SOURCES:
                br_multicast_eht_allow(brmctx, pg, h_addr, srcs, nsrcs,
                                       sizeof(struct in6_addr));
                break;
        case MLD2_BLOCK_OLD_SOURCES:
                changed = br_multicast_eht_block(brmctx, pg, h_addr, srcs, nsrcs,
                                                 sizeof(struct in6_addr));
                break;
        case MLD2_CHANGE_TO_INCLUDE:
                to_report = true;
                fallthrough;
        case MLD2_MODE_IS_INCLUDE:
                changed = br_multicast_eht_inc(brmctx, pg, h_addr, srcs, nsrcs,
                                               sizeof(struct in6_addr),
                                               to_report);
                break;
        case MLD2_CHANGE_TO_EXCLUDE:
                to_report = true;
                fallthrough;
        case MLD2_MODE_IS_EXCLUDE:
                changed = br_multicast_eht_exc(brmctx, pg, h_addr, srcs, nsrcs,
                                               sizeof(struct in6_addr),
                                               to_report);
                break;
        }

        return changed;
}
#endif

/* true means an entry was deleted */
bool br_multicast_eht_handle(const struct net_bridge_mcast *brmctx,
                             struct net_bridge_port_group *pg,
                             void *h_addr,
                             void *srcs,
                             u32 nsrcs,
                             size_t addr_size,
                             int grec_type)
{
        bool eht_enabled = !!(pg->key.port->flags & BR_MULTICAST_FAST_LEAVE);
        union net_bridge_eht_addr eht_host_addr;
        bool changed = false;

        if (!eht_enabled)
                goto out;

        memset(&eht_host_addr, 0, sizeof(eht_host_addr));
        memcpy(&eht_host_addr, h_addr, addr_size);
        if (addr_size == sizeof(__be32))
                changed = __eht_ip4_handle(brmctx, pg, &eht_host_addr, srcs,
                                           nsrcs, grec_type);
#if IS_ENABLED(CONFIG_IPV6)
        else
                changed = __eht_ip6_handle(brmctx, pg, &eht_host_addr, srcs,
                                           nsrcs, grec_type);
#endif

out:
        return changed;
}

int br_multicast_eht_set_hosts_limit(struct net_bridge_port *p,
                                     u32 eht_hosts_limit)
{
        struct net_bridge *br = p->br;

        if (!eht_hosts_limit)
                return -EINVAL;

        spin_lock_bh(&br->multicast_lock);
        p->multicast_eht_hosts_limit = eht_hosts_limit;
        spin_unlock_bh(&br->multicast_lock);

        return 0;
}



























































































































































  150 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Released under the GPLv2 only.
 */

#include <linux/pm.h>
#include <linux/acpi.h>

struct usb_hub_descriptor;
struct usb_dev_state;

/* Functions local to drivers/usb/core/ */

extern int usb_create_sysfs_dev_files(struct usb_device *dev);
extern void usb_remove_sysfs_dev_files(struct usb_device *dev);
extern void usb_create_sysfs_intf_files(struct usb_interface *intf);
extern void usb_remove_sysfs_intf_files(struct usb_interface *intf);
extern int usb_update_wireless_status_attr(struct usb_interface *intf);
extern int usb_create_ep_devs(struct device *parent,
                                struct usb_host_endpoint *endpoint,
                                struct usb_device *udev);
extern void usb_remove_ep_devs(struct usb_host_endpoint *endpoint);

extern void usb_enable_endpoint(struct usb_device *dev,
                struct usb_host_endpoint *ep, bool reset_toggle);
extern void usb_enable_interface(struct usb_device *dev,
                struct usb_interface *intf, bool reset_toggles);
extern void usb_disable_endpoint(struct usb_device *dev, unsigned int epaddr,
                bool reset_hardware);
extern void usb_disable_interface(struct usb_device *dev,
                struct usb_interface *intf, bool reset_hardware);
extern void usb_release_interface_cache(struct kref *ref);
extern void usb_disable_device(struct usb_device *dev, int skip_ep0);
extern int usb_deauthorize_device(struct usb_device *);
extern int usb_authorize_device(struct usb_device *);
extern void usb_deauthorize_interface(struct usb_interface *);
extern void usb_authorize_interface(struct usb_interface *);
extern void usb_detect_quirks(struct usb_device *udev);
extern void usb_detect_interface_quirks(struct usb_device *udev);
extern void usb_release_quirk_list(void);
extern bool usb_endpoint_is_ignored(struct usb_device *udev,
                struct usb_host_interface *intf,
                struct usb_endpoint_descriptor *epd);
extern int usb_remove_device(struct usb_device *udev);

extern struct usb_device_descriptor *usb_get_device_descriptor(
                struct usb_device *udev);
extern int usb_set_isoch_delay(struct usb_device *dev);
extern int usb_get_bos_descriptor(struct usb_device *dev);
extern void usb_release_bos_descriptor(struct usb_device *dev);
extern int usb_set_configuration(struct usb_device *dev, int configuration);
extern int usb_choose_configuration(struct usb_device *udev);
extern int usb_generic_driver_probe(struct usb_device *udev);
extern void usb_generic_driver_disconnect(struct usb_device *udev);
extern int usb_generic_driver_suspend(struct usb_device *udev,
                pm_message_t msg);
extern int usb_generic_driver_resume(struct usb_device *udev,
                pm_message_t msg);

static inline unsigned usb_get_max_power(struct usb_device *udev,
                struct usb_host_config *c)
{
        /* SuperSpeed power is in 8 mA units; others are in 2 mA units */
        unsigned mul = (udev->speed >= USB_SPEED_SUPER ? 8 : 2);

        return c->desc.bMaxPower * mul;
}

extern void usb_kick_hub_wq(struct usb_device *dev);
extern int usb_match_one_id_intf(struct usb_device *dev,
                                 struct usb_host_interface *intf,
                                 const struct usb_device_id *id);
extern int usb_match_device(struct usb_device *dev,
                            const struct usb_device_id *id);
extern const struct usb_device_id *usb_device_match_id(struct usb_device *udev,
                                const struct usb_device_id *id);
extern bool usb_driver_applicable(struct usb_device *udev,
                                  struct usb_device_driver *udrv);
extern void usb_forced_unbind_intf(struct usb_interface *intf);
extern void usb_unbind_and_rebind_marked_interfaces(struct usb_device *udev);

extern void usb_hub_release_all_ports(struct usb_device *hdev,
                struct usb_dev_state *owner);
extern bool usb_device_is_owned(struct usb_device *udev);

extern int  usb_hub_init(void);
extern void usb_hub_cleanup(void);
extern int usb_major_init(void);
extern void usb_major_cleanup(void);
extern int usb_device_supports_lpm(struct usb_device *udev);
extern int usb_port_disable(struct usb_device *udev);

#ifdef        CONFIG_PM

extern int usb_suspend(struct device *dev, pm_message_t msg);
extern int usb_resume(struct device *dev, pm_message_t msg);
extern int usb_resume_complete(struct device *dev);

extern int usb_port_suspend(struct usb_device *dev, pm_message_t msg);
extern int usb_port_resume(struct usb_device *dev, pm_message_t msg);

extern void usb_autosuspend_device(struct usb_device *udev);
extern int usb_autoresume_device(struct usb_device *udev);
extern int usb_remote_wakeup(struct usb_device *dev);
extern int usb_runtime_suspend(struct device *dev);
extern int usb_runtime_resume(struct device *dev);
extern int usb_runtime_idle(struct device *dev);
extern int usb_enable_usb2_hardware_lpm(struct usb_device *udev);
extern int usb_disable_usb2_hardware_lpm(struct usb_device *udev);

extern void usbfs_notify_suspend(struct usb_device *udev);
extern void usbfs_notify_resume(struct usb_device *udev);

#else

static inline int usb_port_suspend(struct usb_device *udev, pm_message_t msg)
{
        return 0;
}

static inline int usb_port_resume(struct usb_device *udev, pm_message_t msg)
{
        return 0;
}

#define usb_autosuspend_device(udev)                do {} while (0)
static inline int usb_autoresume_device(struct usb_device *udev)
{
        return 0;
}

static inline int usb_enable_usb2_hardware_lpm(struct usb_device *udev)
{
        return 0;
}

static inline int usb_disable_usb2_hardware_lpm(struct usb_device *udev)
{
        return 0;
}

#endif

extern const struct class usbmisc_class;
extern const struct bus_type usb_bus_type;
extern struct mutex usb_port_peer_mutex;
extern const struct device_type usb_device_type;
extern const struct device_type usb_if_device_type;
extern const struct device_type usb_ep_device_type;
extern const struct device_type usb_port_device_type;
extern struct usb_device_driver usb_generic_driver;

static inline int is_usb_device(const struct device *dev)
{
        return dev->type == &usb_device_type;
}

static inline int is_usb_interface(const struct device *dev)
{
        return dev->type == &usb_if_device_type;
}

static inline int is_usb_endpoint(const struct device *dev)
{
        return dev->type == &usb_ep_device_type;
}

static inline int is_usb_port(const struct device *dev)
{
        return dev->type == &usb_port_device_type;
}

static inline int is_root_hub(struct usb_device *udev)
{
        return (udev->parent == NULL);
}

extern bool is_usb_device_driver(const struct device_driver *drv);

/* for labeling diagnostics */
extern const char *usbcore_name;

/* sysfs stuff */
extern const struct attribute_group *usb_device_groups[];
extern const struct attribute_group *usb_interface_groups[];

/* usbfs stuff */
extern struct usb_driver usbfs_driver;
extern const struct file_operations usbfs_devices_fops;
extern const struct file_operations usbdev_file_operations;

extern int usb_devio_init(void);
extern void usb_devio_cleanup(void);

/*
 * Firmware specific cookie identifying a port's location. '0' == no location
 * data available
 */
typedef u32 usb_port_location_t;

/* internal notify stuff */
extern void usb_notify_add_device(struct usb_device *udev);
extern void usb_notify_remove_device(struct usb_device *udev);
extern void usb_notify_add_bus(struct usb_bus *ubus);
extern void usb_notify_remove_bus(struct usb_bus *ubus);
extern void usb_hub_adjust_deviceremovable(struct usb_device *hdev,
                struct usb_hub_descriptor *desc);

#ifdef CONFIG_ACPI
extern int usb_acpi_register(void);
extern void usb_acpi_unregister(void);
extern acpi_handle usb_get_hub_port_acpi_handle(struct usb_device *hdev,
        int port1);
#else
static inline int usb_acpi_register(void) { return 0; };
static inline void usb_acpi_unregister(void) { };
#endif
























































































































































































































































































































































































































































































































































































































































































































































    3 




    2 



    1 
    3 


















  123 










  118 






  118 
  118 

  118 







  118 




  118 










   13 









   10 

   13 





    5 



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  129 






  129 
  129 
  129 

  129 











  111 



  111 
    2 





    2 







  110 

  128 





    5 

    5 

    2 
    5 












  106 





  106 












  106 


  106 




























   35 





   34 

    5 
   35 

   35 





    3 







    2 
    3 

    2 







    2 

    3 










  117 
















  116 
  115 


  116 


  115 



  116 














  113 


























   11 







   10 



   10 

   10 

   10 







   11 




   11 



































































































































































































































































































































































































































































































































































   98 




   98 
   94 
   98 
   95 
   97 
   98 
   98 


   98 







  107 









   98 
  106 

  103 
   94 



  106 







   10 





   10 


    3 


    5 
    5 


    4 


   10 
   11 



















































































































































































































































































































































































































































































































































































































































































    5 











    3 


    2 




  100 
































   14 







    3 




    3 







    3 










    2 























    2 










    3 




    3 
    3 


    3 

    3 


    3 


    3 

    3 








    4 







    3 
    3 


    3 


    3 


    3 



    3 


    3 



    3 
    3 

    3 




    4 


    3 






















































































































































    6 





   31 




















   32 




   32 







   32 





   34 








   34 



   33 

   33 
   33 

   31 
   33 





   30 

   31 

   34 

   34 























    3 
    3 




    3 






    3 





    3 
    3 






    3 

    3 












    2 
    2 
    2 

    2 

    2 
    2 






    5 







    5 
    5 




    3 


    3 

    2 
    3 




    2 
    3 




    1 
    1 
    1 










    1 




    1 
    1 


    1 











    7 




    7 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/proc/base.c
 *
 *  Copyright (C) 1991, 1992 Linus Torvalds
 *
 *  proc base directory handling functions
 *
 *  1999, Al Viro. Rewritten. Now it covers the whole per-process part.
 *  Instead of using magical inumbers to determine the kind of object
 *  we allocate and fill in-core inodes upon lookup. They don't even
 *  go into icache. We cache the reference to task_struct upon lookup too.
 *  Eventually it should become a filesystem in its own. We don't use the
 *  rest of procfs anymore.
 *
 *
 *  Changelog:
 *  17-Jan-2005
 *  Allan Bezerra
 *  Bruna Moreira <bruna.moreira@indt.org.br>
 *  Edjard Mota <edjard.mota@indt.org.br>
 *  Ilias Biris <ilias.biris@indt.org.br>
 *  Mauricio Lin <mauricio.lin@indt.org.br>
 *
 *  Embedded Linux Lab - 10LE Instituto Nokia de Tecnologia - INdT
 *
 *  A new process specific entry (smaps) included in /proc. It shows the
 *  size of rss for each memory area. The maps entry lacks information
 *  about physical memory size (rss) for each mapped file, i.e.,
 *  rss information for executables and library files.
 *  This additional information is useful for any tools that need to know
 *  about physical memory consumption for a process specific library.
 *
 *  Changelog:
 *  21-Feb-2005
 *  Embedded Linux Lab - 10LE Instituto Nokia de Tecnologia - INdT
 *  Pud inclusion in the page table walking.
 *
 *  ChangeLog:
 *  10-Mar-2005
 *  10LE Instituto Nokia de Tecnologia - INdT:
 *  A better way to walks through the page table as suggested by Hugh Dickins.
 *
 *  Simo Piiroinen <simo.piiroinen@nokia.com>:
 *  Smaps information related to shared, private, clean and dirty pages.
 *
 *  Paul Mundt <paul.mundt@nokia.com>:
 *  Overall revision about smaps.
 */

#include <linux/uaccess.h>

#include <linux/errno.h>
#include <linux/time.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/init.h>
#include <linux/capability.h>
#include <linux/file.h>
#include <linux/generic-radix-tree.h>
#include <linux/string.h>
#include <linux/seq_file.h>
#include <linux/namei.h>
#include <linux/mnt_namespace.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/rcupdate.h>
#include <linux/kallsyms.h>
#include <linux/stacktrace.h>
#include <linux/resource.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/security.h>
#include <linux/ptrace.h>
#include <linux/printk.h>
#include <linux/cache.h>
#include <linux/cgroup.h>
#include <linux/cpuset.h>
#include <linux/audit.h>
#include <linux/poll.h>
#include <linux/nsproxy.h>
#include <linux/oom.h>
#include <linux/elf.h>
#include <linux/pid_namespace.h>
#include <linux/user_namespace.h>
#include <linux/fs_parser.h>
#include <linux/fs_struct.h>
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/debug.h>
#include <linux/sched/stat.h>
#include <linux/posix-timers.h>
#include <linux/time_namespace.h>
#include <linux/resctrl.h>
#include <linux/cn_proc.h>
#include <linux/ksm.h>
#include <uapi/linux/lsm.h>
#include <trace/events/oom.h>
#include "internal.h"
#include "fd.h"

#include "../../lib/kstrtox.h"

/* NOTE:
 *        Implementing inode permission operations in /proc is almost
 *        certainly an error.  Permission checks need to happen during
 *        each system call not at open time.  The reason is that most of
 *        what we wish to check for permissions in /proc varies at runtime.
 *
 *        The classic example of a problem is opening file descriptors
 *        in /proc for a task before it execs a suid executable.
 */

static u8 nlink_tid __ro_after_init;
static u8 nlink_tgid __ro_after_init;

enum proc_mem_force {
        PROC_MEM_FORCE_ALWAYS,
        PROC_MEM_FORCE_PTRACE,
        PROC_MEM_FORCE_NEVER
};

static enum proc_mem_force proc_mem_force_override __ro_after_init =
        IS_ENABLED(CONFIG_PROC_MEM_NO_FORCE) ? PROC_MEM_FORCE_NEVER :
        IS_ENABLED(CONFIG_PROC_MEM_FORCE_PTRACE) ? PROC_MEM_FORCE_PTRACE :
        PROC_MEM_FORCE_ALWAYS;

static const struct constant_table proc_mem_force_table[] __initconst = {
        { "always", PROC_MEM_FORCE_ALWAYS },
        { "ptrace", PROC_MEM_FORCE_PTRACE },
        { "never", PROC_MEM_FORCE_NEVER },
        { }
};

static int __init early_proc_mem_force_override(char *buf)
{
        if (!buf)
                return -EINVAL;

        /*
         * lookup_constant() defaults to proc_mem_force_override to preseve
         * the initial Kconfig choice in case an invalid param gets passed.
         */
        proc_mem_force_override = lookup_constant(proc_mem_force_table,
                                                  buf, proc_mem_force_override);

        return 0;
}
early_param("proc_mem.force_override", early_proc_mem_force_override);

struct pid_entry {
        const char *name;
        unsigned int len;
        umode_t mode;
        const struct inode_operations *iop;
        const struct file_operations *fop;
        union proc_op op;
};

#define NOD(NAME, MODE, IOP, FOP, OP) {                        \
        .name = (NAME),                                        \
        .len  = sizeof(NAME) - 1,                        \
        .mode = MODE,                                        \
        .iop  = IOP,                                        \
        .fop  = FOP,                                        \
        .op   = OP,                                        \
}

#define DIR(NAME, MODE, iops, fops)        \
        NOD(NAME, (S_IFDIR|(MODE)), &iops, &fops, {} )
#define LNK(NAME, get_link)                                        \
        NOD(NAME, (S_IFLNK|S_IRWXUGO),                                \
                &proc_pid_link_inode_operations, NULL,                \
                { .proc_get_link = get_link } )
#define REG(NAME, MODE, fops)                                \
        NOD(NAME, (S_IFREG|(MODE)), NULL, &fops, {})
#define ONE(NAME, MODE, show)                                \
        NOD(NAME, (S_IFREG|(MODE)),                        \
                NULL, &proc_single_file_operations,        \
                { .proc_show = show } )
#define ATTR(LSMID, NAME, MODE)                                \
        NOD(NAME, (S_IFREG|(MODE)),                        \
                NULL, &proc_pid_attr_operations,        \
                { .lsmid = LSMID })

/*
 * Count the number of hardlinks for the pid_entry table, excluding the .
 * and .. links.
 */
static unsigned int __init pid_entry_nlink(const struct pid_entry *entries,
        unsigned int n)
{
        unsigned int i;
        unsigned int count;

        count = 2;
        for (i = 0; i < n; ++i) {
                if (S_ISDIR(entries[i].mode))
                        ++count;
        }

        return count;
}

static int get_task_root(struct task_struct *task, struct path *root)
{
        int result = -ENOENT;

        task_lock(task);
        if (task->fs) {
                get_fs_root(task->fs, root);
                result = 0;
        }
        task_unlock(task);
        return result;
}

static int proc_cwd_link(struct dentry *dentry, struct path *path)
{
        struct task_struct *task = get_proc_task(d_inode(dentry));
        int result = -ENOENT;

        if (task) {
                task_lock(task);
                if (task->fs) {
                        get_fs_pwd(task->fs, path);
                        result = 0;
                }
                task_unlock(task);
                put_task_struct(task);
        }
        return result;
}

static int proc_root_link(struct dentry *dentry, struct path *path)
{
        struct task_struct *task = get_proc_task(d_inode(dentry));
        int result = -ENOENT;

        if (task) {
                result = get_task_root(task, path);
                put_task_struct(task);
        }
        return result;
}

/*
 * If the user used setproctitle(), we just get the string from
 * user space at arg_start, and limit it to a maximum of one page.
 */
static ssize_t get_mm_proctitle(struct mm_struct *mm, char __user *buf,
                                size_t count, unsigned long pos,
                                unsigned long arg_start)
{
        char *page;
        int ret, got;

        if (pos >= PAGE_SIZE)
                return 0;

        page = (char *)__get_free_page(GFP_KERNEL);
        if (!page)
                return -ENOMEM;

        ret = 0;
        got = access_remote_vm(mm, arg_start, page, PAGE_SIZE, FOLL_ANON);
        if (got > 0) {
                int len = strnlen(page, got);

                /* Include the NUL character if it was found */
                if (len < got)
                        len++;

                if (len > pos) {
                        len -= pos;
                        if (len > count)
                                len = count;
                        len -= copy_to_user(buf, page+pos, len);
                        if (!len)
                                len = -EFAULT;
                        ret = len;
                }
        }
        free_page((unsigned long)page);
        return ret;
}

static ssize_t get_mm_cmdline(struct mm_struct *mm, char __user *buf,
                              size_t count, loff_t *ppos)
{
        unsigned long arg_start, arg_end, env_start, env_end;
        unsigned long pos, len;
        char *page, c;

        /* Check if process spawned far enough to have cmdline. */
        if (!mm->env_end)
                return 0;

        spin_lock(&mm->arg_lock);
        arg_start = mm->arg_start;
        arg_end = mm->arg_end;
        env_start = mm->env_start;
        env_end = mm->env_end;
        spin_unlock(&mm->arg_lock);

        if (arg_start >= arg_end)
                return 0;

        /*
         * We allow setproctitle() to overwrite the argument
         * strings, and overflow past the original end. But
         * only when it overflows into the environment area.
         */
        if (env_start != arg_end || env_end < env_start)
                env_start = env_end = arg_end;
        len = env_end - arg_start;

        /* We're not going to care if "*ppos" has high bits set */
        pos = *ppos;
        if (pos >= len)
                return 0;
        if (count > len - pos)
                count = len - pos;
        if (!count)
                return 0;

        /*
         * Magical special case: if the argv[] end byte is not
         * zero, the user has overwritten it with setproctitle(3).
         *
         * Possible future enhancement: do this only once when
         * pos is 0, and set a flag in the 'struct file'.
         */
        if (access_remote_vm(mm, arg_end-1, &c, 1, FOLL_ANON) == 1 && c)
                return get_mm_proctitle(mm, buf, count, pos, arg_start);

        /*
         * For the non-setproctitle() case we limit things strictly
         * to the [arg_start, arg_end[ range.
         */
        pos += arg_start;
        if (pos < arg_start || pos >= arg_end)
                return 0;
        if (count > arg_end - pos)
                count = arg_end - pos;

        page = (char *)__get_free_page(GFP_KERNEL);
        if (!page)
                return -ENOMEM;

        len = 0;
        while (count) {
                int got;
                size_t size = min_t(size_t, PAGE_SIZE, count);

                got = access_remote_vm(mm, pos, page, size, FOLL_ANON);
                if (got <= 0)
                        break;
                got -= copy_to_user(buf, page, got);
                if (unlikely(!got)) {
                        if (!len)
                                len = -EFAULT;
                        break;
                }
                pos += got;
                buf += got;
                len += got;
                count -= got;
        }

        free_page((unsigned long)page);
        return len;
}

static ssize_t get_task_cmdline(struct task_struct *tsk, char __user *buf,
                                size_t count, loff_t *pos)
{
        struct mm_struct *mm;
        ssize_t ret;

        mm = get_task_mm(tsk);
        if (!mm)
                return 0;

        ret = get_mm_cmdline(mm, buf, count, pos);
        mmput(mm);
        return ret;
}

static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf,
                                     size_t count, loff_t *pos)
{
        struct task_struct *tsk;
        ssize_t ret;

        BUG_ON(*pos < 0);

        tsk = get_proc_task(file_inode(file));
        if (!tsk)
                return -ESRCH;
        ret = get_task_cmdline(tsk, buf, count, pos);
        put_task_struct(tsk);
        if (ret > 0)
                *pos += ret;
        return ret;
}

static const struct file_operations proc_pid_cmdline_ops = {
        .read        = proc_pid_cmdline_read,
        .llseek        = generic_file_llseek,
};

#ifdef CONFIG_KALLSYMS
/*
 * Provides a wchan file via kallsyms in a proper one-value-per-file format.
 * Returns the resolved symbol.  If that fails, simply return the address.
 */
static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns,
                          struct pid *pid, struct task_struct *task)
{
        unsigned long wchan;
        char symname[KSYM_NAME_LEN];

        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
                goto print0;

        wchan = get_wchan(task);
        if (wchan && !lookup_symbol_name(wchan, symname)) {
                seq_puts(m, symname);
                return 0;
        }

print0:
        seq_putc(m, '0');
        return 0;
}
#endif /* CONFIG_KALLSYMS */

static int lock_trace(struct task_struct *task)
{
        int err = down_read_killable(&task->signal->exec_update_lock);
        if (err)
                return err;
        if (!ptrace_may_access(task, PTRACE_MODE_ATTACH_FSCREDS)) {
                up_read(&task->signal->exec_update_lock);
                return -EPERM;
        }
        return 0;
}

static void unlock_trace(struct task_struct *task)
{
        up_read(&task->signal->exec_update_lock);
}

#ifdef CONFIG_STACKTRACE

#define MAX_STACK_TRACE_DEPTH        64

static int proc_pid_stack(struct seq_file *m, struct pid_namespace *ns,
                          struct pid *pid, struct task_struct *task)
{
        unsigned long *entries;
        int err;

        /*
         * The ability to racily run the kernel stack unwinder on a running task
         * and then observe the unwinder output is scary; while it is useful for
         * debugging kernel issues, it can also allow an attacker to leak kernel
         * stack contents.
         * Doing this in a manner that is at least safe from races would require
         * some work to ensure that the remote task can not be scheduled; and
         * even then, this would still expose the unwinder as local attack
         * surface.
         * Therefore, this interface is restricted to root.
         */
        if (!file_ns_capable(m->file, &init_user_ns, CAP_SYS_ADMIN))
                return -EACCES;

        entries = kmalloc_array(MAX_STACK_TRACE_DEPTH, sizeof(*entries),
                                GFP_KERNEL);
        if (!entries)
                return -ENOMEM;

        err = lock_trace(task);
        if (!err) {
                unsigned int i, nr_entries;

                nr_entries = stack_trace_save_tsk(task, entries,
                                                  MAX_STACK_TRACE_DEPTH, 0);

                for (i = 0; i < nr_entries; i++) {
                        seq_printf(m, "[<0>] %pB\n", (void *)entries[i]);
                }

                unlock_trace(task);
        }
        kfree(entries);

        return err;
}
#endif

#ifdef CONFIG_SCHED_INFO
/*
 * Provides /proc/PID/schedstat
 */
static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
                              struct pid *pid, struct task_struct *task)
{
        if (unlikely(!sched_info_on()))
                seq_puts(m, "0 0 0\n");
        else
                seq_printf(m, "%llu %llu %lu\n",
                   (unsigned long long)task->se.sum_exec_runtime,
                   (unsigned long long)task->sched_info.run_delay,
                   task->sched_info.pcount);

        return 0;
}
#endif

#ifdef CONFIG_LATENCYTOP
static int lstats_show_proc(struct seq_file *m, void *v)
{
        int i;
        struct inode *inode = m->private;
        struct task_struct *task = get_proc_task(inode);

        if (!task)
                return -ESRCH;
        seq_puts(m, "Latency Top version : v0.1\n");
        for (i = 0; i < LT_SAVECOUNT; i++) {
                struct latency_record *lr = &task->latency_record[i];
                if (lr->backtrace[0]) {
                        int q;
                        seq_printf(m, "%i %li %li",
                                   lr->count, lr->time, lr->max);
                        for (q = 0; q < LT_BACKTRACEDEPTH; q++) {
                                unsigned long bt = lr->backtrace[q];

                                if (!bt)
                                        break;
                                seq_printf(m, " %ps", (void *)bt);
                        }
                        seq_putc(m, '\n');
                }

        }
        put_task_struct(task);
        return 0;
}

static int lstats_open(struct inode *inode, struct file *file)
{
        return single_open(file, lstats_show_proc, inode);
}

static ssize_t lstats_write(struct file *file, const char __user *buf,
                            size_t count, loff_t *offs)
{
        struct task_struct *task = get_proc_task(file_inode(file));

        if (!task)
                return -ESRCH;
        clear_tsk_latency_tracing(task);
        put_task_struct(task);

        return count;
}

static const struct file_operations proc_lstats_operations = {
        .open                = lstats_open,
        .read                = seq_read,
        .write                = lstats_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};

#endif

static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns,
                          struct pid *pid, struct task_struct *task)
{
        unsigned long totalpages = totalram_pages() + total_swap_pages;
        unsigned long points = 0;
        long badness;

        badness = oom_badness(task, totalpages);
        /*
         * Special case OOM_SCORE_ADJ_MIN for all others scale the
         * badness value into [0, 2000] range which we have been
         * exporting for a long time so userspace might depend on it.
         */
        if (badness != LONG_MIN)
                points = (1000 + badness * 1000 / (long)totalpages) * 2 / 3;

        seq_printf(m, "%lu\n", points);

        return 0;
}

struct limit_names {
        const char *name;
        const char *unit;
};

static const struct limit_names lnames[RLIM_NLIMITS] = {
        [RLIMIT_CPU] = {"Max cpu time", "seconds"},
        [RLIMIT_FSIZE] = {"Max file size", "bytes"},
        [RLIMIT_DATA] = {"Max data size", "bytes"},
        [RLIMIT_STACK] = {"Max stack size", "bytes"},
        [RLIMIT_CORE] = {"Max core file size", "bytes"},
        [RLIMIT_RSS] = {"Max resident set", "bytes"},
        [RLIMIT_NPROC] = {"Max processes", "processes"},
        [RLIMIT_NOFILE] = {"Max open files", "files"},
        [RLIMIT_MEMLOCK] = {"Max locked memory", "bytes"},
        [RLIMIT_AS] = {"Max address space", "bytes"},
        [RLIMIT_LOCKS] = {"Max file locks", "locks"},
        [RLIMIT_SIGPENDING] = {"Max pending signals", "signals"},
        [RLIMIT_MSGQUEUE] = {"Max msgqueue size", "bytes"},
        [RLIMIT_NICE] = {"Max nice priority", NULL},
        [RLIMIT_RTPRIO] = {"Max realtime priority", NULL},
        [RLIMIT_RTTIME] = {"Max realtime timeout", "us"},
};

/* Display limits for a process */
static int proc_pid_limits(struct seq_file *m, struct pid_namespace *ns,
                           struct pid *pid, struct task_struct *task)
{
        unsigned int i;
        unsigned long flags;

        struct rlimit rlim[RLIM_NLIMITS];

        if (!lock_task_sighand(task, &flags))
                return 0;
        memcpy(rlim, task->signal->rlim, sizeof(struct rlimit) * RLIM_NLIMITS);
        unlock_task_sighand(task, &flags);

        /*
         * print the file header
         */
        seq_puts(m, "Limit                     "
                "Soft Limit           "
                "Hard Limit           "
                "Units     \n");

        for (i = 0; i < RLIM_NLIMITS; i++) {
                if (rlim[i].rlim_cur == RLIM_INFINITY)
                        seq_printf(m, "%-25s %-20s ",
                                   lnames[i].name, "unlimited");
                else
                        seq_printf(m, "%-25s %-20lu ",
                                   lnames[i].name, rlim[i].rlim_cur);

                if (rlim[i].rlim_max == RLIM_INFINITY)
                        seq_printf(m, "%-20s ", "unlimited");
                else
                        seq_printf(m, "%-20lu ", rlim[i].rlim_max);

                if (lnames[i].unit)
                        seq_printf(m, "%-10s\n", lnames[i].unit);
                else
                        seq_putc(m, '\n');
        }

        return 0;
}

#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns,
                            struct pid *pid, struct task_struct *task)
{
        struct syscall_info info;
        u64 *args = &info.data.args[0];
        int res;

        res = lock_trace(task);
        if (res)
                return res;

        if (task_current_syscall(task, &info))
                seq_puts(m, "running\n");
        else if (info.data.nr < 0)
                seq_printf(m, "%d 0x%llx 0x%llx\n",
                           info.data.nr, info.sp, info.data.instruction_pointer);
        else
                seq_printf(m,
                       "%d 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx\n",
                       info.data.nr,
                       args[0], args[1], args[2], args[3], args[4], args[5],
                       info.sp, info.data.instruction_pointer);
        unlock_trace(task);

        return 0;
}
#endif /* CONFIG_HAVE_ARCH_TRACEHOOK */

/************************************************************************/
/*                       Here the fs part begins                        */
/************************************************************************/

/* permission checks */
static bool proc_fd_access_allowed(struct inode *inode)
{
        struct task_struct *task;
        bool allowed = false;
        /* Allow access to a task's file descriptors if it is us or we
         * may use ptrace attach to the process and find out that
         * information.
         */
        task = get_proc_task(inode);
        if (task) {
                allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS);
                put_task_struct(task);
        }
        return allowed;
}

int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                 struct iattr *attr)
{
        int error;
        struct inode *inode = d_inode(dentry);

        if (attr->ia_valid & ATTR_MODE)
                return -EPERM;

        error = setattr_prepare(&nop_mnt_idmap, dentry, attr);
        if (error)
                return error;

        setattr_copy(&nop_mnt_idmap, inode, attr);
        return 0;
}

/*
 * May current process learn task's sched/cmdline info (for hide_pid_min=1)
 * or euid/egid (for hide_pid_min=2)?
 */
static bool has_pid_permissions(struct proc_fs_info *fs_info,
                                 struct task_struct *task,
                                 enum proc_hidepid hide_pid_min)
{
        /*
         * If 'hidpid' mount option is set force a ptrace check,
         * we indicate that we are using a filesystem syscall
         * by passing PTRACE_MODE_READ_FSCREDS
         */
        if (fs_info->hide_pid == HIDEPID_NOT_PTRACEABLE)
                return ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS);

        if (fs_info->hide_pid < hide_pid_min)
                return true;
        if (in_group_p(fs_info->pid_gid))
                return true;
        return ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS);
}


static int proc_pid_permission(struct mnt_idmap *idmap,
                               struct inode *inode, int mask)
{
        struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb);
        struct task_struct *task;
        bool has_perms;

        task = get_proc_task(inode);
        if (!task)
                return -ESRCH;
        has_perms = has_pid_permissions(fs_info, task, HIDEPID_NO_ACCESS);
        put_task_struct(task);

        if (!has_perms) {
                if (fs_info->hide_pid == HIDEPID_INVISIBLE) {
                        /*
                         * Let's make getdents(), stat(), and open()
                         * consistent with each other.  If a process
                         * may not stat() a file, it shouldn't be seen
                         * in procfs at all.
                         */
                        return -ENOENT;
                }

                return -EPERM;
        }
        return generic_permission(&nop_mnt_idmap, inode, mask);
}



static const struct inode_operations proc_def_inode_operations = {
        .setattr        = proc_setattr,
};

static int proc_single_show(struct seq_file *m, void *v)
{
        struct inode *inode = m->private;
        struct pid_namespace *ns = proc_pid_ns(inode->i_sb);
        struct pid *pid = proc_pid(inode);
        struct task_struct *task;
        int ret;

        task = get_pid_task(pid, PIDTYPE_PID);
        if (!task)
                return -ESRCH;

        ret = PROC_I(inode)->op.proc_show(m, ns, pid, task);

        put_task_struct(task);
        return ret;
}

static int proc_single_open(struct inode *inode, struct file *filp)
{
        return single_open(filp, proc_single_show, inode);
}

static const struct file_operations proc_single_file_operations = {
        .open                = proc_single_open,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = single_release,
};


struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode)
{
        struct task_struct *task = get_proc_task(inode);
        struct mm_struct *mm = ERR_PTR(-ESRCH);

        if (task) {
                mm = mm_access(task, mode | PTRACE_MODE_FSCREDS);
                put_task_struct(task);

                if (!IS_ERR_OR_NULL(mm)) {
                        /* ensure this mm_struct can't be freed */
                        mmgrab(mm);
                        /* but do not pin its memory */
                        mmput(mm);
                }
        }

        return mm;
}

static int __mem_open(struct inode *inode, struct file *file, unsigned int mode)
{
        struct mm_struct *mm = proc_mem_open(inode, mode);

        if (IS_ERR(mm))
                return PTR_ERR(mm);

        file->private_data = mm;
        return 0;
}

static int mem_open(struct inode *inode, struct file *file)
{
        if (WARN_ON_ONCE(!(file->f_op->fop_flags & FOP_UNSIGNED_OFFSET)))
                return -EINVAL;
        return __mem_open(inode, file, PTRACE_MODE_ATTACH);
}

static bool proc_mem_foll_force(struct file *file, struct mm_struct *mm)
{
        struct task_struct *task;
        bool ptrace_active = false;

        switch (proc_mem_force_override) {
        case PROC_MEM_FORCE_NEVER:
                return false;
        case PROC_MEM_FORCE_PTRACE:
                task = get_proc_task(file_inode(file));
                if (task) {
                        ptrace_active =        READ_ONCE(task->ptrace) &&
                                        READ_ONCE(task->mm) == mm &&
                                        READ_ONCE(task->parent) == current;
                        put_task_struct(task);
                }
                return ptrace_active;
        default:
                return true;
        }
}

static ssize_t mem_rw(struct file *file, char __user *buf,
                        size_t count, loff_t *ppos, int write)
{
        struct mm_struct *mm = file->private_data;
        unsigned long addr = *ppos;
        ssize_t copied;
        char *page;
        unsigned int flags;

        if (!mm)
                return 0;

        page = (char *)__get_free_page(GFP_KERNEL);
        if (!page)
                return -ENOMEM;

        copied = 0;
        if (!mmget_not_zero(mm))
                goto free;

        flags = write ? FOLL_WRITE : 0;
        if (proc_mem_foll_force(file, mm))
                flags |= FOLL_FORCE;

        while (count > 0) {
                size_t this_len = min_t(size_t, count, PAGE_SIZE);

                if (write && copy_from_user(page, buf, this_len)) {
                        copied = -EFAULT;
                        break;
                }

                this_len = access_remote_vm(mm, addr, page, this_len, flags);
                if (!this_len) {
                        if (!copied)
                                copied = -EIO;
                        break;
                }

                if (!write && copy_to_user(buf, page, this_len)) {
                        copied = -EFAULT;
                        break;
                }

                buf += this_len;
                addr += this_len;
                copied += this_len;
                count -= this_len;
        }
        *ppos = addr;

        mmput(mm);
free:
        free_page((unsigned long) page);
        return copied;
}

static ssize_t mem_read(struct file *file, char __user *buf,
                        size_t count, loff_t *ppos)
{
        return mem_rw(file, buf, count, ppos, 0);
}

static ssize_t mem_write(struct file *file, const char __user *buf,
                         size_t count, loff_t *ppos)
{
        return mem_rw(file, (char __user*)buf, count, ppos, 1);
}

loff_t mem_lseek(struct file *file, loff_t offset, int orig)
{
        switch (orig) {
        case 0:
                file->f_pos = offset;
                break;
        case 1:
                file->f_pos += offset;
                break;
        default:
                return -EINVAL;
        }
        force_successful_syscall_return();
        return file->f_pos;
}

static int mem_release(struct inode *inode, struct file *file)
{
        struct mm_struct *mm = file->private_data;
        if (mm)
                mmdrop(mm);
        return 0;
}

static const struct file_operations proc_mem_operations = {
        .llseek                = mem_lseek,
        .read                = mem_read,
        .write                = mem_write,
        .open                = mem_open,
        .release        = mem_release,
        .fop_flags        = FOP_UNSIGNED_OFFSET,
};

static int environ_open(struct inode *inode, struct file *file)
{
        return __mem_open(inode, file, PTRACE_MODE_READ);
}

static ssize_t environ_read(struct file *file, char __user *buf,
                        size_t count, loff_t *ppos)
{
        char *page;
        unsigned long src = *ppos;
        int ret = 0;
        struct mm_struct *mm = file->private_data;
        unsigned long env_start, env_end;

        /* Ensure the process spawned far enough to have an environment. */
        if (!mm || !mm->env_end)
                return 0;

        page = (char *)__get_free_page(GFP_KERNEL);
        if (!page)
                return -ENOMEM;

        ret = 0;
        if (!mmget_not_zero(mm))
                goto free;

        spin_lock(&mm->arg_lock);
        env_start = mm->env_start;
        env_end = mm->env_end;
        spin_unlock(&mm->arg_lock);

        while (count > 0) {
                size_t this_len, max_len;
                int retval;

                if (src >= (env_end - env_start))
                        break;

                this_len = env_end - (env_start + src);

                max_len = min_t(size_t, PAGE_SIZE, count);
                this_len = min(max_len, this_len);

                retval = access_remote_vm(mm, (env_start + src), page, this_len, FOLL_ANON);

                if (retval <= 0) {
                        ret = retval;
                        break;
                }

                if (copy_to_user(buf, page, retval)) {
                        ret = -EFAULT;
                        break;
                }

                ret += retval;
                src += retval;
                buf += retval;
                count -= retval;
        }
        *ppos = src;
        mmput(mm);

free:
        free_page((unsigned long) page);
        return ret;
}

static const struct file_operations proc_environ_operations = {
        .open                = environ_open,
        .read                = environ_read,
        .llseek                = generic_file_llseek,
        .release        = mem_release,
};

static int auxv_open(struct inode *inode, struct file *file)
{
        return __mem_open(inode, file, PTRACE_MODE_READ_FSCREDS);
}

static ssize_t auxv_read(struct file *file, char __user *buf,
                        size_t count, loff_t *ppos)
{
        struct mm_struct *mm = file->private_data;
        unsigned int nwords = 0;

        if (!mm)
                return 0;
        do {
                nwords += 2;
        } while (mm->saved_auxv[nwords - 2] != 0); /* AT_NULL */
        return simple_read_from_buffer(buf, count, ppos, mm->saved_auxv,
                                       nwords * sizeof(mm->saved_auxv[0]));
}

static const struct file_operations proc_auxv_operations = {
        .open                = auxv_open,
        .read                = auxv_read,
        .llseek                = generic_file_llseek,
        .release        = mem_release,
};

static ssize_t oom_adj_read(struct file *file, char __user *buf, size_t count,
                            loff_t *ppos)
{
        struct task_struct *task = get_proc_task(file_inode(file));
        char buffer[PROC_NUMBUF];
        int oom_adj = OOM_ADJUST_MIN;
        size_t len;

        if (!task)
                return -ESRCH;
        if (task->signal->oom_score_adj == OOM_SCORE_ADJ_MAX)
                oom_adj = OOM_ADJUST_MAX;
        else
                oom_adj = (task->signal->oom_score_adj * -OOM_DISABLE) /
                          OOM_SCORE_ADJ_MAX;
        put_task_struct(task);
        if (oom_adj > OOM_ADJUST_MAX)
                oom_adj = OOM_ADJUST_MAX;
        len = snprintf(buffer, sizeof(buffer), "%d\n", oom_adj);
        return simple_read_from_buffer(buf, count, ppos, buffer, len);
}

static int __set_oom_adj(struct file *file, int oom_adj, bool legacy)
{
        struct mm_struct *mm = NULL;
        struct task_struct *task;
        int err = 0;

        task = get_proc_task(file_inode(file));
        if (!task)
                return -ESRCH;

        mutex_lock(&oom_adj_mutex);
        if (legacy) {
                if (oom_adj < task->signal->oom_score_adj &&
                                !capable(CAP_SYS_RESOURCE)) {
                        err = -EACCES;
                        goto err_unlock;
                }
                /*
                 * /proc/pid/oom_adj is provided for legacy purposes, ask users to use
                 * /proc/pid/oom_score_adj instead.
                 */
                pr_warn_once("%s (%d): /proc/%d/oom_adj is deprecated, please use /proc/%d/oom_score_adj instead.\n",
                          current->comm, task_pid_nr(current), task_pid_nr(task),
                          task_pid_nr(task));
        } else {
                if ((short)oom_adj < task->signal->oom_score_adj_min &&
                                !capable(CAP_SYS_RESOURCE)) {
                        err = -EACCES;
                        goto err_unlock;
                }
        }

        /*
         * Make sure we will check other processes sharing the mm if this is
         * not vfrok which wants its own oom_score_adj.
         * pin the mm so it doesn't go away and get reused after task_unlock
         */
        if (!task->vfork_done) {
                struct task_struct *p = find_lock_task_mm(task);

                if (p) {
                        if (test_bit(MMF_MULTIPROCESS, &p->mm->flags)) {
                                mm = p->mm;
                                mmgrab(mm);
                        }
                        task_unlock(p);
                }
        }

        task->signal->oom_score_adj = oom_adj;
        if (!legacy && has_capability_noaudit(current, CAP_SYS_RESOURCE))
                task->signal->oom_score_adj_min = (short)oom_adj;
        trace_oom_score_adj_update(task);

        if (mm) {
                struct task_struct *p;

                rcu_read_lock();
                for_each_process(p) {
                        if (same_thread_group(task, p))
                                continue;

                        /* do not touch kernel threads or the global init */
                        if (p->flags & PF_KTHREAD || is_global_init(p))
                                continue;

                        task_lock(p);
                        if (!p->vfork_done && process_shares_mm(p, mm)) {
                                p->signal->oom_score_adj = oom_adj;
                                if (!legacy && has_capability_noaudit(current, CAP_SYS_RESOURCE))
                                        p->signal->oom_score_adj_min = (short)oom_adj;
                        }
                        task_unlock(p);
                }
                rcu_read_unlock();
                mmdrop(mm);
        }
err_unlock:
        mutex_unlock(&oom_adj_mutex);
        put_task_struct(task);
        return err;
}

/*
 * /proc/pid/oom_adj exists solely for backwards compatibility with previous
 * kernels.  The effective policy is defined by oom_score_adj, which has a
 * different scale: oom_adj grew exponentially and oom_score_adj grows linearly.
 * Values written to oom_adj are simply mapped linearly to oom_score_adj.
 * Processes that become oom disabled via oom_adj will still be oom disabled
 * with this implementation.
 *
 * oom_adj cannot be removed since existing userspace binaries use it.
 */
static ssize_t oom_adj_write(struct file *file, const char __user *buf,
                             size_t count, loff_t *ppos)
{
        char buffer[PROC_NUMBUF] = {};
        int oom_adj;
        int err;

        if (count > sizeof(buffer) - 1)
                count = sizeof(buffer) - 1;
        if (copy_from_user(buffer, buf, count)) {
                err = -EFAULT;
                goto out;
        }

        err = kstrtoint(strstrip(buffer), 0, &oom_adj);
        if (err)
                goto out;
        if ((oom_adj < OOM_ADJUST_MIN || oom_adj > OOM_ADJUST_MAX) &&
             oom_adj != OOM_DISABLE) {
                err = -EINVAL;
                goto out;
        }

        /*
         * Scale /proc/pid/oom_score_adj appropriately ensuring that a maximum
         * value is always attainable.
         */
        if (oom_adj == OOM_ADJUST_MAX)
                oom_adj = OOM_SCORE_ADJ_MAX;
        else
                oom_adj = (oom_adj * OOM_SCORE_ADJ_MAX) / -OOM_DISABLE;

        err = __set_oom_adj(file, oom_adj, true);
out:
        return err < 0 ? err : count;
}

static const struct file_operations proc_oom_adj_operations = {
        .read                = oom_adj_read,
        .write                = oom_adj_write,
        .llseek                = generic_file_llseek,
};

static ssize_t oom_score_adj_read(struct file *file, char __user *buf,
                                        size_t count, loff_t *ppos)
{
        struct task_struct *task = get_proc_task(file_inode(file));
        char buffer[PROC_NUMBUF];
        short oom_score_adj = OOM_SCORE_ADJ_MIN;
        size_t len;

        if (!task)
                return -ESRCH;
        oom_score_adj = task->signal->oom_score_adj;
        put_task_struct(task);
        len = snprintf(buffer, sizeof(buffer), "%hd\n", oom_score_adj);
        return simple_read_from_buffer(buf, count, ppos, buffer, len);
}

static ssize_t oom_score_adj_write(struct file *file, const char __user *buf,
                                        size_t count, loff_t *ppos)
{
        char buffer[PROC_NUMBUF] = {};
        int oom_score_adj;
        int err;

        if (count > sizeof(buffer) - 1)
                count = sizeof(buffer) - 1;
        if (copy_from_user(buffer, buf, count)) {
                err = -EFAULT;
                goto out;
        }

        err = kstrtoint(strstrip(buffer), 0, &oom_score_adj);
        if (err)
                goto out;
        if (oom_score_adj < OOM_SCORE_ADJ_MIN ||
                        oom_score_adj > OOM_SCORE_ADJ_MAX) {
                err = -EINVAL;
                goto out;
        }

        err = __set_oom_adj(file, oom_score_adj, false);
out:
        return err < 0 ? err : count;
}

static const struct file_operations proc_oom_score_adj_operations = {
        .read                = oom_score_adj_read,
        .write                = oom_score_adj_write,
        .llseek                = default_llseek,
};

#ifdef CONFIG_AUDIT
#define TMPBUFLEN 11
static ssize_t proc_loginuid_read(struct file * file, char __user * buf,
                                  size_t count, loff_t *ppos)
{
        struct inode * inode = file_inode(file);
        struct task_struct *task = get_proc_task(inode);
        ssize_t length;
        char tmpbuf[TMPBUFLEN];

        if (!task)
                return -ESRCH;
        length = scnprintf(tmpbuf, TMPBUFLEN, "%u",
                           from_kuid(file->f_cred->user_ns,
                                     audit_get_loginuid(task)));
        put_task_struct(task);
        return simple_read_from_buffer(buf, count, ppos, tmpbuf, length);
}

static ssize_t proc_loginuid_write(struct file * file, const char __user * buf,
                                   size_t count, loff_t *ppos)
{
        struct inode * inode = file_inode(file);
        uid_t loginuid;
        kuid_t kloginuid;
        int rv;

        /* Don't let kthreads write their own loginuid */
        if (current->flags & PF_KTHREAD)
                return -EPERM;

        rcu_read_lock();
        if (current != pid_task(proc_pid(inode), PIDTYPE_PID)) {
                rcu_read_unlock();
                return -EPERM;
        }
        rcu_read_unlock();

        if (*ppos != 0) {
                /* No partial writes. */
                return -EINVAL;
        }

        rv = kstrtou32_from_user(buf, count, 10, &loginuid);
        if (rv < 0)
                return rv;

        /* is userspace tring to explicitly UNSET the loginuid? */
        if (loginuid == AUDIT_UID_UNSET) {
                kloginuid = INVALID_UID;
        } else {
                kloginuid = make_kuid(file->f_cred->user_ns, loginuid);
                if (!uid_valid(kloginuid))
                        return -EINVAL;
        }

        rv = audit_set_loginuid(kloginuid);
        if (rv < 0)
                return rv;
        return count;
}

static const struct file_operations proc_loginuid_operations = {
        .read                = proc_loginuid_read,
        .write                = proc_loginuid_write,
        .llseek                = generic_file_llseek,
};

static ssize_t proc_sessionid_read(struct file * file, char __user * buf,
                                  size_t count, loff_t *ppos)
{
        struct inode * inode = file_inode(file);
        struct task_struct *task = get_proc_task(inode);
        ssize_t length;
        char tmpbuf[TMPBUFLEN];

        if (!task)
                return -ESRCH;
        length = scnprintf(tmpbuf, TMPBUFLEN, "%u",
                                audit_get_sessionid(task));
        put_task_struct(task);
        return simple_read_from_buffer(buf, count, ppos, tmpbuf, length);
}

static const struct file_operations proc_sessionid_operations = {
        .read                = proc_sessionid_read,
        .llseek                = generic_file_llseek,
};
#endif

#ifdef CONFIG_FAULT_INJECTION
static ssize_t proc_fault_inject_read(struct file * file, char __user * buf,
                                      size_t count, loff_t *ppos)
{
        struct task_struct *task = get_proc_task(file_inode(file));
        char buffer[PROC_NUMBUF];
        size_t len;
        int make_it_fail;

        if (!task)
                return -ESRCH;
        make_it_fail = task->make_it_fail;
        put_task_struct(task);

        len = snprintf(buffer, sizeof(buffer), "%i\n", make_it_fail);

        return simple_read_from_buffer(buf, count, ppos, buffer, len);
}

static ssize_t proc_fault_inject_write(struct file * file,
                        const char __user * buf, size_t count, loff_t *ppos)
{
        struct task_struct *task;
        char buffer[PROC_NUMBUF] = {};
        int make_it_fail;
        int rv;

        if (!capable(CAP_SYS_RESOURCE))
                return -EPERM;

        if (count > sizeof(buffer) - 1)
                count = sizeof(buffer) - 1;
        if (copy_from_user(buffer, buf, count))
                return -EFAULT;
        rv = kstrtoint(strstrip(buffer), 0, &make_it_fail);
        if (rv < 0)
                return rv;
        if (make_it_fail < 0 || make_it_fail > 1)
                return -EINVAL;

        task = get_proc_task(file_inode(file));
        if (!task)
                return -ESRCH;
        task->make_it_fail = make_it_fail;
        put_task_struct(task);

        return count;
}

static const struct file_operations proc_fault_inject_operations = {
        .read                = proc_fault_inject_read,
        .write                = proc_fault_inject_write,
        .llseek                = generic_file_llseek,
};

static ssize_t proc_fail_nth_write(struct file *file, const char __user *buf,
                                   size_t count, loff_t *ppos)
{
        struct task_struct *task;
        int err;
        unsigned int n;

        err = kstrtouint_from_user(buf, count, 0, &n);
        if (err)
                return err;

        task = get_proc_task(file_inode(file));
        if (!task)
                return -ESRCH;
        task->fail_nth = n;
        put_task_struct(task);

        return count;
}

static ssize_t proc_fail_nth_read(struct file *file, char __user *buf,
                                  size_t count, loff_t *ppos)
{
        struct task_struct *task;
        char numbuf[PROC_NUMBUF];
        ssize_t len;

        task = get_proc_task(file_inode(file));
        if (!task)
                return -ESRCH;
        len = snprintf(numbuf, sizeof(numbuf), "%u\n", task->fail_nth);
        put_task_struct(task);
        return simple_read_from_buffer(buf, count, ppos, numbuf, len);
}

static const struct file_operations proc_fail_nth_operations = {
        .read                = proc_fail_nth_read,
        .write                = proc_fail_nth_write,
};
#endif


#ifdef CONFIG_SCHED_DEBUG
/*
 * Print out various scheduling related per-task fields:
 */
static int sched_show(struct seq_file *m, void *v)
{
        struct inode *inode = m->private;
        struct pid_namespace *ns = proc_pid_ns(inode->i_sb);
        struct task_struct *p;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;
        proc_sched_show_task(p, ns, m);

        put_task_struct(p);

        return 0;
}

static ssize_t
sched_write(struct file *file, const char __user *buf,
            size_t count, loff_t *offset)
{
        struct inode *inode = file_inode(file);
        struct task_struct *p;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;
        proc_sched_set_task(p);

        put_task_struct(p);

        return count;
}

static int sched_open(struct inode *inode, struct file *filp)
{
        return single_open(filp, sched_show, inode);
}

static const struct file_operations proc_pid_sched_operations = {
        .open                = sched_open,
        .read                = seq_read,
        .write                = sched_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};

#endif

#ifdef CONFIG_SCHED_AUTOGROUP
/*
 * Print out autogroup related information:
 */
static int sched_autogroup_show(struct seq_file *m, void *v)
{
        struct inode *inode = m->private;
        struct task_struct *p;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;
        proc_sched_autogroup_show_task(p, m);

        put_task_struct(p);

        return 0;
}

static ssize_t
sched_autogroup_write(struct file *file, const char __user *buf,
            size_t count, loff_t *offset)
{
        struct inode *inode = file_inode(file);
        struct task_struct *p;
        char buffer[PROC_NUMBUF] = {};
        int nice;
        int err;

        if (count > sizeof(buffer) - 1)
                count = sizeof(buffer) - 1;
        if (copy_from_user(buffer, buf, count))
                return -EFAULT;

        err = kstrtoint(strstrip(buffer), 0, &nice);
        if (err < 0)
                return err;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;

        err = proc_sched_autogroup_set_nice(p, nice);
        if (err)
                count = err;

        put_task_struct(p);

        return count;
}

static int sched_autogroup_open(struct inode *inode, struct file *filp)
{
        int ret;

        ret = single_open(filp, sched_autogroup_show, NULL);
        if (!ret) {
                struct seq_file *m = filp->private_data;

                m->private = inode;
        }
        return ret;
}

static const struct file_operations proc_pid_sched_autogroup_operations = {
        .open                = sched_autogroup_open,
        .read                = seq_read,
        .write                = sched_autogroup_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};

#endif /* CONFIG_SCHED_AUTOGROUP */

#ifdef CONFIG_TIME_NS
static int timens_offsets_show(struct seq_file *m, void *v)
{
        struct task_struct *p;

        p = get_proc_task(file_inode(m->file));
        if (!p)
                return -ESRCH;
        proc_timens_show_offsets(p, m);

        put_task_struct(p);

        return 0;
}

static ssize_t timens_offsets_write(struct file *file, const char __user *buf,
                                    size_t count, loff_t *ppos)
{
        struct inode *inode = file_inode(file);
        struct proc_timens_offset offsets[2];
        char *kbuf = NULL, *pos, *next_line;
        struct task_struct *p;
        int ret, noffsets;

        /* Only allow < page size writes at the beginning of the file */
        if ((*ppos != 0) || (count >= PAGE_SIZE))
                return -EINVAL;

        /* Slurp in the user data */
        kbuf = memdup_user_nul(buf, count);
        if (IS_ERR(kbuf))
                return PTR_ERR(kbuf);

        /* Parse the user data */
        ret = -EINVAL;
        noffsets = 0;
        for (pos = kbuf; pos; pos = next_line) {
                struct proc_timens_offset *off = &offsets[noffsets];
                char clock[10];
                int err;

                /* Find the end of line and ensure we don't look past it */
                next_line = strchr(pos, '\n');
                if (next_line) {
                        *next_line = '\0';
                        next_line++;
                        if (*next_line == '\0')
                                next_line = NULL;
                }

                err = sscanf(pos, "%9s %lld %lu", clock,
                                &off->val.tv_sec, &off->val.tv_nsec);
                if (err != 3 || off->val.tv_nsec >= NSEC_PER_SEC)
                        goto out;

                clock[sizeof(clock) - 1] = 0;
                if (strcmp(clock, "monotonic") == 0 ||
                    strcmp(clock, __stringify(CLOCK_MONOTONIC)) == 0)
                        off->clockid = CLOCK_MONOTONIC;
                else if (strcmp(clock, "boottime") == 0 ||
                         strcmp(clock, __stringify(CLOCK_BOOTTIME)) == 0)
                        off->clockid = CLOCK_BOOTTIME;
                else
                        goto out;

                noffsets++;
                if (noffsets == ARRAY_SIZE(offsets)) {
                        if (next_line)
                                count = next_line - kbuf;
                        break;
                }
        }

        ret = -ESRCH;
        p = get_proc_task(inode);
        if (!p)
                goto out;
        ret = proc_timens_set_offset(file, p, offsets, noffsets);
        put_task_struct(p);
        if (ret)
                goto out;

        ret = count;
out:
        kfree(kbuf);
        return ret;
}

static int timens_offsets_open(struct inode *inode, struct file *filp)
{
        return single_open(filp, timens_offsets_show, inode);
}

static const struct file_operations proc_timens_offsets_operations = {
        .open                = timens_offsets_open,
        .read                = seq_read,
        .write                = timens_offsets_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};
#endif /* CONFIG_TIME_NS */

static ssize_t comm_write(struct file *file, const char __user *buf,
                                size_t count, loff_t *offset)
{
        struct inode *inode = file_inode(file);
        struct task_struct *p;
        char buffer[TASK_COMM_LEN] = {};
        const size_t maxlen = sizeof(buffer) - 1;

        if (copy_from_user(buffer, buf, count > maxlen ? maxlen : count))
                return -EFAULT;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;

        if (same_thread_group(current, p)) {
                set_task_comm(p, buffer);
                proc_comm_connector(p);
        }
        else
                count = -EINVAL;

        put_task_struct(p);

        return count;
}

static int comm_show(struct seq_file *m, void *v)
{
        struct inode *inode = m->private;
        struct task_struct *p;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;

        proc_task_name(m, p, false);
        seq_putc(m, '\n');

        put_task_struct(p);

        return 0;
}

static int comm_open(struct inode *inode, struct file *filp)
{
        return single_open(filp, comm_show, inode);
}

static const struct file_operations proc_pid_set_comm_operations = {
        .open                = comm_open,
        .read                = seq_read,
        .write                = comm_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};

static int proc_exe_link(struct dentry *dentry, struct path *exe_path)
{
        struct task_struct *task;
        struct file *exe_file;

        task = get_proc_task(d_inode(dentry));
        if (!task)
                return -ENOENT;
        exe_file = get_task_exe_file(task);
        put_task_struct(task);
        if (exe_file) {
                *exe_path = exe_file->f_path;
                path_get(&exe_file->f_path);
                fput(exe_file);
                return 0;
        } else
                return -ENOENT;
}

static const char *proc_pid_get_link(struct dentry *dentry,
                                     struct inode *inode,
                                     struct delayed_call *done)
{
        struct path path;
        int error = -EACCES;

        if (!dentry)
                return ERR_PTR(-ECHILD);

        /* Are we allowed to snoop on the tasks file descriptors? */
        if (!proc_fd_access_allowed(inode))
                goto out;

        error = PROC_I(inode)->op.proc_get_link(dentry, &path);
        if (error)
                goto out;

        error = nd_jump_link(&path);
out:
        return ERR_PTR(error);
}

static int do_proc_readlink(const struct path *path, char __user *buffer, int buflen)
{
        char *tmp = kmalloc(PATH_MAX, GFP_KERNEL);
        char *pathname;
        int len;

        if (!tmp)
                return -ENOMEM;

        pathname = d_path(path, tmp, PATH_MAX);
        len = PTR_ERR(pathname);
        if (IS_ERR(pathname))
                goto out;
        len = tmp + PATH_MAX - 1 - pathname;

        if (len > buflen)
                len = buflen;
        if (copy_to_user(buffer, pathname, len))
                len = -EFAULT;
 out:
        kfree(tmp);
        return len;
}

static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int buflen)
{
        int error = -EACCES;
        struct inode *inode = d_inode(dentry);
        struct path path;

        /* Are we allowed to snoop on the tasks file descriptors? */
        if (!proc_fd_access_allowed(inode))
                goto out;

        error = PROC_I(inode)->op.proc_get_link(dentry, &path);
        if (error)
                goto out;

        error = do_proc_readlink(&path, buffer, buflen);
        path_put(&path);
out:
        return error;
}

const struct inode_operations proc_pid_link_inode_operations = {
        .readlink        = proc_pid_readlink,
        .get_link        = proc_pid_get_link,
        .setattr        = proc_setattr,
};


/* building an inode */

void task_dump_owner(struct task_struct *task, umode_t mode,
                     kuid_t *ruid, kgid_t *rgid)
{
        /* Depending on the state of dumpable compute who should own a
         * proc file for a task.
         */
        const struct cred *cred;
        kuid_t uid;
        kgid_t gid;

        if (unlikely(task->flags & PF_KTHREAD)) {
                *ruid = GLOBAL_ROOT_UID;
                *rgid = GLOBAL_ROOT_GID;
                return;
        }

        /* Default to the tasks effective ownership */
        rcu_read_lock();
        cred = __task_cred(task);
        uid = cred->euid;
        gid = cred->egid;
        rcu_read_unlock();

        /*
         * Before the /proc/pid/status file was created the only way to read
         * the effective uid of a /process was to stat /proc/pid.  Reading
         * /proc/pid/status is slow enough that procps and other packages
         * kept stating /proc/pid.  To keep the rules in /proc simple I have
         * made this apply to all per process world readable and executable
         * directories.
         */
        if (mode != (S_IFDIR|S_IRUGO|S_IXUGO)) {
                struct mm_struct *mm;
                task_lock(task);
                mm = task->mm;
                /* Make non-dumpable tasks owned by some root */
                if (mm) {
                        if (get_dumpable(mm) != SUID_DUMP_USER) {
                                struct user_namespace *user_ns = mm->user_ns;

                                uid = make_kuid(user_ns, 0);
                                if (!uid_valid(uid))
                                        uid = GLOBAL_ROOT_UID;

                                gid = make_kgid(user_ns, 0);
                                if (!gid_valid(gid))
                                        gid = GLOBAL_ROOT_GID;
                        }
                } else {
                        uid = GLOBAL_ROOT_UID;
                        gid = GLOBAL_ROOT_GID;
                }
                task_unlock(task);
        }
        *ruid = uid;
        *rgid = gid;
}

void proc_pid_evict_inode(struct proc_inode *ei)
{
        struct pid *pid = ei->pid;

        if (S_ISDIR(ei->vfs_inode.i_mode)) {
                spin_lock(&pid->lock);
                hlist_del_init_rcu(&ei->sibling_inodes);
                spin_unlock(&pid->lock);
        }
}

struct inode *proc_pid_make_inode(struct super_block *sb,
                                  struct task_struct *task, umode_t mode)
{
        struct inode * inode;
        struct proc_inode *ei;
        struct pid *pid;

        /* We need a new inode */

        inode = new_inode(sb);
        if (!inode)
                goto out;

        /* Common stuff */
        ei = PROC_I(inode);
        inode->i_mode = mode;
        inode->i_ino = get_next_ino();
        simple_inode_init_ts(inode);
        inode->i_op = &proc_def_inode_operations;

        /*
         * grab the reference to task.
         */
        pid = get_task_pid(task, PIDTYPE_PID);
        if (!pid)
                goto out_unlock;

        /* Let the pid remember us for quick removal */
        ei->pid = pid;

        task_dump_owner(task, 0, &inode->i_uid, &inode->i_gid);
        security_task_to_inode(task, inode);

out:
        return inode;

out_unlock:
        iput(inode);
        return NULL;
}

/*
 * Generating an inode and adding it into @pid->inodes, so that task will
 * invalidate inode's dentry before being released.
 *
 * This helper is used for creating dir-type entries under '/proc' and
 * '/proc/<tgid>/task'. Other entries(eg. fd, stat) under '/proc/<tgid>'
 * can be released by invalidating '/proc/<tgid>' dentry.
 * In theory, dentries under '/proc/<tgid>/task' can also be released by
 * invalidating '/proc/<tgid>' dentry, we reserve it to handle single
 * thread exiting situation: Any one of threads should invalidate its
 * '/proc/<tgid>/task/<pid>' dentry before released.
 */
static struct inode *proc_pid_make_base_inode(struct super_block *sb,
                                struct task_struct *task, umode_t mode)
{
        struct inode *inode;
        struct proc_inode *ei;
        struct pid *pid;

        inode = proc_pid_make_inode(sb, task, mode);
        if (!inode)
                return NULL;

        /* Let proc_flush_pid find this directory inode */
        ei = PROC_I(inode);
        pid = ei->pid;
        spin_lock(&pid->lock);
        hlist_add_head_rcu(&ei->sibling_inodes, &pid->inodes);
        spin_unlock(&pid->lock);

        return inode;
}

int pid_getattr(struct mnt_idmap *idmap, const struct path *path,
                struct kstat *stat, u32 request_mask, unsigned int query_flags)
{
        struct inode *inode = d_inode(path->dentry);
        struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb);
        struct task_struct *task;

        generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);

        stat->uid = GLOBAL_ROOT_UID;
        stat->gid = GLOBAL_ROOT_GID;
        rcu_read_lock();
        task = pid_task(proc_pid(inode), PIDTYPE_PID);
        if (task) {
                if (!has_pid_permissions(fs_info, task, HIDEPID_INVISIBLE)) {
                        rcu_read_unlock();
                        /*
                         * This doesn't prevent learning whether PID exists,
                         * it only makes getattr() consistent with readdir().
                         */
                        return -ENOENT;
                }
                task_dump_owner(task, inode->i_mode, &stat->uid, &stat->gid);
        }
        rcu_read_unlock();
        return 0;
}

/* dentry stuff */

/*
 * Set <pid>/... inode ownership (can change due to setuid(), etc.)
 */
void pid_update_inode(struct task_struct *task, struct inode *inode)
{
        task_dump_owner(task, inode->i_mode, &inode->i_uid, &inode->i_gid);

        inode->i_mode &= ~(S_ISUID | S_ISGID);
        security_task_to_inode(task, inode);
}

/*
 * Rewrite the inode's ownerships here because the owning task may have
 * performed a setuid(), etc.
 *
 */
static int pid_revalidate(struct dentry *dentry, unsigned int flags)
{
        struct inode *inode;
        struct task_struct *task;
        int ret = 0;

        rcu_read_lock();
        inode = d_inode_rcu(dentry);
        if (!inode)
                goto out;
        task = pid_task(proc_pid(inode), PIDTYPE_PID);

        if (task) {
                pid_update_inode(task, inode);
                ret = 1;
        }
out:
        rcu_read_unlock();
        return ret;
}

static inline bool proc_inode_is_dead(struct inode *inode)
{
        return !proc_pid(inode)->tasks[PIDTYPE_PID].first;
}

int pid_delete_dentry(const struct dentry *dentry)
{
        /* Is the task we represent dead?
         * If so, then don't put the dentry on the lru list,
         * kill it immediately.
         */
        return proc_inode_is_dead(d_inode(dentry));
}

const struct dentry_operations pid_dentry_operations =
{
        .d_revalidate        = pid_revalidate,
        .d_delete        = pid_delete_dentry,
};

/* Lookups */

/*
 * Fill a directory entry.
 *
 * If possible create the dcache entry and derive our inode number and
 * file type from dcache entry.
 *
 * Since all of the proc inode numbers are dynamically generated, the inode
 * numbers do not exist until the inode is cache.  This means creating
 * the dcache entry in readdir is necessary to keep the inode numbers
 * reported by readdir in sync with the inode numbers reported
 * by stat.
 */
bool proc_fill_cache(struct file *file, struct dir_context *ctx,
        const char *name, unsigned int len,
        instantiate_t instantiate, struct task_struct *task, const void *ptr)
{
        struct dentry *child, *dir = file->f_path.dentry;
        struct qstr qname = QSTR_INIT(name, len);
        struct inode *inode;
        unsigned type = DT_UNKNOWN;
        ino_t ino = 1;

        child = d_hash_and_lookup(dir, &qname);
        if (!child) {
                DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
                child = d_alloc_parallel(dir, &qname, &wq);
                if (IS_ERR(child))
                        goto end_instantiate;
                if (d_in_lookup(child)) {
                        struct dentry *res;
                        res = instantiate(child, task, ptr);
                        d_lookup_done(child);
                        if (unlikely(res)) {
                                dput(child);
                                child = res;
                                if (IS_ERR(child))
                                        goto end_instantiate;
                        }
                }
        }
        inode = d_inode(child);
        ino = inode->i_ino;
        type = inode->i_mode >> 12;
        dput(child);
end_instantiate:
        return dir_emit(ctx, name, len, ino, type);
}

/*
 * dname_to_vma_addr - maps a dentry name into two unsigned longs
 * which represent vma start and end addresses.
 */
static int dname_to_vma_addr(struct dentry *dentry,
                             unsigned long *start, unsigned long *end)
{
        const char *str = dentry->d_name.name;
        unsigned long long sval, eval;
        unsigned int len;

        if (str[0] == '0' && str[1] != '-')
                return -EINVAL;
        len = _parse_integer(str, 16, &sval);
        if (len & KSTRTOX_OVERFLOW)
                return -EINVAL;
        if (sval != (unsigned long)sval)
                return -EINVAL;
        str += len;

        if (*str != '-')
                return -EINVAL;
        str++;

        if (str[0] == '0' && str[1])
                return -EINVAL;
        len = _parse_integer(str, 16, &eval);
        if (len & KSTRTOX_OVERFLOW)
                return -EINVAL;
        if (eval != (unsigned long)eval)
                return -EINVAL;
        str += len;

        if (*str != '\0')
                return -EINVAL;

        *start = sval;
        *end = eval;

        return 0;
}

static int map_files_d_revalidate(struct dentry *dentry, unsigned int flags)
{
        unsigned long vm_start, vm_end;
        bool exact_vma_exists = false;
        struct mm_struct *mm = NULL;
        struct task_struct *task;
        struct inode *inode;
        int status = 0;

        if (flags & LOOKUP_RCU)
                return -ECHILD;

        inode = d_inode(dentry);
        task = get_proc_task(inode);
        if (!task)
                goto out_notask;

        mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
        if (IS_ERR_OR_NULL(mm))
                goto out;

        if (!dname_to_vma_addr(dentry, &vm_start, &vm_end)) {
                status = mmap_read_lock_killable(mm);
                if (!status) {
                        exact_vma_exists = !!find_exact_vma(mm, vm_start,
                                                            vm_end);
                        mmap_read_unlock(mm);
                }
        }

        mmput(mm);

        if (exact_vma_exists) {
                task_dump_owner(task, 0, &inode->i_uid, &inode->i_gid);

                security_task_to_inode(task, inode);
                status = 1;
        }

out:
        put_task_struct(task);

out_notask:
        return status;
}

static const struct dentry_operations tid_map_files_dentry_operations = {
        .d_revalidate        = map_files_d_revalidate,
        .d_delete        = pid_delete_dentry,
};

static int map_files_get_link(struct dentry *dentry, struct path *path)
{
        unsigned long vm_start, vm_end;
        struct vm_area_struct *vma;
        struct task_struct *task;
        struct mm_struct *mm;
        int rc;

        rc = -ENOENT;
        task = get_proc_task(d_inode(dentry));
        if (!task)
                goto out;

        mm = get_task_mm(task);
        put_task_struct(task);
        if (!mm)
                goto out;

        rc = dname_to_vma_addr(dentry, &vm_start, &vm_end);
        if (rc)
                goto out_mmput;

        rc = mmap_read_lock_killable(mm);
        if (rc)
                goto out_mmput;

        rc = -ENOENT;
        vma = find_exact_vma(mm, vm_start, vm_end);
        if (vma && vma->vm_file) {
                *path = *file_user_path(vma->vm_file);
                path_get(path);
                rc = 0;
        }
        mmap_read_unlock(mm);

out_mmput:
        mmput(mm);
out:
        return rc;
}

struct map_files_info {
        unsigned long        start;
        unsigned long        end;
        fmode_t                mode;
};

/*
 * Only allow CAP_SYS_ADMIN and CAP_CHECKPOINT_RESTORE to follow the links, due
 * to concerns about how the symlinks may be used to bypass permissions on
 * ancestor directories in the path to the file in question.
 */
static const char *
proc_map_files_get_link(struct dentry *dentry,
                        struct inode *inode,
                        struct delayed_call *done)
{
        if (!checkpoint_restore_ns_capable(&init_user_ns))
                return ERR_PTR(-EPERM);

        return proc_pid_get_link(dentry, inode, done);
}

/*
 * Identical to proc_pid_link_inode_operations except for get_link()
 */
static const struct inode_operations proc_map_files_link_inode_operations = {
        .readlink        = proc_pid_readlink,
        .get_link        = proc_map_files_get_link,
        .setattr        = proc_setattr,
};

static struct dentry *
proc_map_files_instantiate(struct dentry *dentry,
                           struct task_struct *task, const void *ptr)
{
        fmode_t mode = (fmode_t)(unsigned long)ptr;
        struct proc_inode *ei;
        struct inode *inode;

        inode = proc_pid_make_inode(dentry->d_sb, task, S_IFLNK |
                                    ((mode & FMODE_READ ) ? S_IRUSR : 0) |
                                    ((mode & FMODE_WRITE) ? S_IWUSR : 0));
        if (!inode)
                return ERR_PTR(-ENOENT);

        ei = PROC_I(inode);
        ei->op.proc_get_link = map_files_get_link;

        inode->i_op = &proc_map_files_link_inode_operations;
        inode->i_size = 64;

        return proc_splice_unmountable(inode, dentry,
                                       &tid_map_files_dentry_operations);
}

static struct dentry *proc_map_files_lookup(struct inode *dir,
                struct dentry *dentry, unsigned int flags)
{
        unsigned long vm_start, vm_end;
        struct vm_area_struct *vma;
        struct task_struct *task;
        struct dentry *result;
        struct mm_struct *mm;

        result = ERR_PTR(-ENOENT);
        task = get_proc_task(dir);
        if (!task)
                goto out;

        result = ERR_PTR(-EACCES);
        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
                goto out_put_task;

        result = ERR_PTR(-ENOENT);
        if (dname_to_vma_addr(dentry, &vm_start, &vm_end))
                goto out_put_task;

        mm = get_task_mm(task);
        if (!mm)
                goto out_put_task;

        result = ERR_PTR(-EINTR);
        if (mmap_read_lock_killable(mm))
                goto out_put_mm;

        result = ERR_PTR(-ENOENT);
        vma = find_exact_vma(mm, vm_start, vm_end);
        if (!vma)
                goto out_no_vma;

        if (vma->vm_file)
                result = proc_map_files_instantiate(dentry, task,
                                (void *)(unsigned long)vma->vm_file->f_mode);

out_no_vma:
        mmap_read_unlock(mm);
out_put_mm:
        mmput(mm);
out_put_task:
        put_task_struct(task);
out:
        return result;
}

static const struct inode_operations proc_map_files_inode_operations = {
        .lookup                = proc_map_files_lookup,
        .permission        = proc_fd_permission,
        .setattr        = proc_setattr,
};

static int
proc_map_files_readdir(struct file *file, struct dir_context *ctx)
{
        struct vm_area_struct *vma;
        struct task_struct *task;
        struct mm_struct *mm;
        unsigned long nr_files, pos, i;
        GENRADIX(struct map_files_info) fa;
        struct map_files_info *p;
        int ret;
        struct vma_iterator vmi;

        genradix_init(&fa);

        ret = -ENOENT;
        task = get_proc_task(file_inode(file));
        if (!task)
                goto out;

        ret = -EACCES;
        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS))
                goto out_put_task;

        ret = 0;
        if (!dir_emit_dots(file, ctx))
                goto out_put_task;

        mm = get_task_mm(task);
        if (!mm)
                goto out_put_task;

        ret = mmap_read_lock_killable(mm);
        if (ret) {
                mmput(mm);
                goto out_put_task;
        }

        nr_files = 0;

        /*
         * We need two passes here:
         *
         *  1) Collect vmas of mapped files with mmap_lock taken
         *  2) Release mmap_lock and instantiate entries
         *
         * otherwise we get lockdep complained, since filldir()
         * routine might require mmap_lock taken in might_fault().
         */

        pos = 2;
        vma_iter_init(&vmi, mm, 0);
        for_each_vma(vmi, vma) {
                if (!vma->vm_file)
                        continue;
                if (++pos <= ctx->pos)
                        continue;

                p = genradix_ptr_alloc(&fa, nr_files++, GFP_KERNEL);
                if (!p) {
                        ret = -ENOMEM;
                        mmap_read_unlock(mm);
                        mmput(mm);
                        goto out_put_task;
                }

                p->start = vma->vm_start;
                p->end = vma->vm_end;
                p->mode = vma->vm_file->f_mode;
        }
        mmap_read_unlock(mm);
        mmput(mm);

        for (i = 0; i < nr_files; i++) {
                char buf[4 * sizeof(long) + 2];        /* max: %lx-%lx\0 */
                unsigned int len;

                p = genradix_ptr(&fa, i);
                len = snprintf(buf, sizeof(buf), "%lx-%lx", p->start, p->end);
                if (!proc_fill_cache(file, ctx,
                                      buf, len,
                                      proc_map_files_instantiate,
                                      task,
                                      (void *)(unsigned long)p->mode))
                        break;
                ctx->pos++;
        }

out_put_task:
        put_task_struct(task);
out:
        genradix_free(&fa);
        return ret;
}

static const struct file_operations proc_map_files_operations = {
        .read                = generic_read_dir,
        .iterate_shared        = proc_map_files_readdir,
        .llseek                = generic_file_llseek,
};

#if defined(CONFIG_CHECKPOINT_RESTORE) && defined(CONFIG_POSIX_TIMERS)
struct timers_private {
        struct pid *pid;
        struct task_struct *task;
        struct sighand_struct *sighand;
        struct pid_namespace *ns;
        unsigned long flags;
};

static void *timers_start(struct seq_file *m, loff_t *pos)
{
        struct timers_private *tp = m->private;

        tp->task = get_pid_task(tp->pid, PIDTYPE_PID);
        if (!tp->task)
                return ERR_PTR(-ESRCH);

        tp->sighand = lock_task_sighand(tp->task, &tp->flags);
        if (!tp->sighand)
                return ERR_PTR(-ESRCH);

        return seq_hlist_start(&tp->task->signal->posix_timers, *pos);
}

static void *timers_next(struct seq_file *m, void *v, loff_t *pos)
{
        struct timers_private *tp = m->private;
        return seq_hlist_next(v, &tp->task->signal->posix_timers, pos);
}

static void timers_stop(struct seq_file *m, void *v)
{
        struct timers_private *tp = m->private;

        if (tp->sighand) {
                unlock_task_sighand(tp->task, &tp->flags);
                tp->sighand = NULL;
        }

        if (tp->task) {
                put_task_struct(tp->task);
                tp->task = NULL;
        }
}

static int show_timer(struct seq_file *m, void *v)
{
        struct k_itimer *timer;
        struct timers_private *tp = m->private;
        int notify;
        static const char * const nstr[] = {
                [SIGEV_SIGNAL] = "signal",
                [SIGEV_NONE] = "none",
                [SIGEV_THREAD] = "thread",
        };

        timer = hlist_entry((struct hlist_node *)v, struct k_itimer, list);
        notify = timer->it_sigev_notify;

        seq_printf(m, "ID: %d\n", timer->it_id);
        seq_printf(m, "signal: %d/%px\n",
                   timer->sigq.info.si_signo,
                   timer->sigq.info.si_value.sival_ptr);
        seq_printf(m, "notify: %s/%s.%d\n",
                   nstr[notify & ~SIGEV_THREAD_ID],
                   (notify & SIGEV_THREAD_ID) ? "tid" : "pid",
                   pid_nr_ns(timer->it_pid, tp->ns));
        seq_printf(m, "ClockID: %d\n", timer->it_clock);

        return 0;
}

static const struct seq_operations proc_timers_seq_ops = {
        .start        = timers_start,
        .next        = timers_next,
        .stop        = timers_stop,
        .show        = show_timer,
};

static int proc_timers_open(struct inode *inode, struct file *file)
{
        struct timers_private *tp;

        tp = __seq_open_private(file, &proc_timers_seq_ops,
                        sizeof(struct timers_private));
        if (!tp)
                return -ENOMEM;

        tp->pid = proc_pid(inode);
        tp->ns = proc_pid_ns(inode->i_sb);
        return 0;
}

static const struct file_operations proc_timers_operations = {
        .open                = proc_timers_open,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = seq_release_private,
};
#endif

static ssize_t timerslack_ns_write(struct file *file, const char __user *buf,
                                        size_t count, loff_t *offset)
{
        struct inode *inode = file_inode(file);
        struct task_struct *p;
        u64 slack_ns;
        int err;

        err = kstrtoull_from_user(buf, count, 10, &slack_ns);
        if (err < 0)
                return err;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;

        if (p != current) {
                rcu_read_lock();
                if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
                        rcu_read_unlock();
                        count = -EPERM;
                        goto out;
                }
                rcu_read_unlock();

                err = security_task_setscheduler(p);
                if (err) {
                        count = err;
                        goto out;
                }
        }

        task_lock(p);
        if (rt_or_dl_task_policy(p))
                slack_ns = 0;
        else if (slack_ns == 0)
                slack_ns = p->default_timer_slack_ns;
        p->timer_slack_ns = slack_ns;
        task_unlock(p);

out:
        put_task_struct(p);

        return count;
}

static int timerslack_ns_show(struct seq_file *m, void *v)
{
        struct inode *inode = m->private;
        struct task_struct *p;
        int err = 0;

        p = get_proc_task(inode);
        if (!p)
                return -ESRCH;

        if (p != current) {
                rcu_read_lock();
                if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
                        rcu_read_unlock();
                        err = -EPERM;
                        goto out;
                }
                rcu_read_unlock();

                err = security_task_getscheduler(p);
                if (err)
                        goto out;
        }

        task_lock(p);
        seq_printf(m, "%llu\n", p->timer_slack_ns);
        task_unlock(p);

out:
        put_task_struct(p);

        return err;
}

static int timerslack_ns_open(struct inode *inode, struct file *filp)
{
        return single_open(filp, timerslack_ns_show, inode);
}

static const struct file_operations proc_pid_set_timerslack_ns_operations = {
        .open                = timerslack_ns_open,
        .read                = seq_read,
        .write                = timerslack_ns_write,
        .llseek                = seq_lseek,
        .release        = single_release,
};

static struct dentry *proc_pident_instantiate(struct dentry *dentry,
        struct task_struct *task, const void *ptr)
{
        const struct pid_entry *p = ptr;
        struct inode *inode;
        struct proc_inode *ei;

        inode = proc_pid_make_inode(dentry->d_sb, task, p->mode);
        if (!inode)
                return ERR_PTR(-ENOENT);

        ei = PROC_I(inode);
        if (S_ISDIR(inode->i_mode))
                set_nlink(inode, 2);        /* Use getattr to fix if necessary */
        if (p->iop)
                inode->i_op = p->iop;
        if (p->fop)
                inode->i_fop = p->fop;
        ei->op = p->op;
        pid_update_inode(task, inode);
        d_set_d_op(dentry, &pid_dentry_operations);
        return d_splice_alias(inode, dentry);
}

static struct dentry *proc_pident_lookup(struct inode *dir, 
                                         struct dentry *dentry,
                                         const struct pid_entry *p,
                                         const struct pid_entry *end)
{
        struct task_struct *task = get_proc_task(dir);
        struct dentry *res = ERR_PTR(-ENOENT);

        if (!task)
                goto out_no_task;

        /*
         * Yes, it does not scale. And it should not. Don't add
         * new entries into /proc/<tgid>/ without very good reasons.
         */
        for (; p < end; p++) {
                if (p->len != dentry->d_name.len)
                        continue;
                if (!memcmp(dentry->d_name.name, p->name, p->len)) {
                        res = proc_pident_instantiate(dentry, task, p);
                        break;
                }
        }
        put_task_struct(task);
out_no_task:
        return res;
}

static int proc_pident_readdir(struct file *file, struct dir_context *ctx,
                const struct pid_entry *ents, unsigned int nents)
{
        struct task_struct *task = get_proc_task(file_inode(file));
        const struct pid_entry *p;

        if (!task)
                return -ENOENT;

        if (!dir_emit_dots(file, ctx))
                goto out;

        if (ctx->pos >= nents + 2)
                goto out;

        for (p = ents + (ctx->pos - 2); p < ents + nents; p++) {
                if (!proc_fill_cache(file, ctx, p->name, p->len,
                                proc_pident_instantiate, task, p))
                        break;
                ctx->pos++;
        }
out:
        put_task_struct(task);
        return 0;
}

#ifdef CONFIG_SECURITY
static int proc_pid_attr_open(struct inode *inode, struct file *file)
{
        file->private_data = NULL;
        __mem_open(inode, file, PTRACE_MODE_READ_FSCREDS);
        return 0;
}

static ssize_t proc_pid_attr_read(struct file * file, char __user * buf,
                                  size_t count, loff_t *ppos)
{
        struct inode * inode = file_inode(file);
        char *p = NULL;
        ssize_t length;
        struct task_struct *task = get_proc_task(inode);

        if (!task)
                return -ESRCH;

        length = security_getprocattr(task, PROC_I(inode)->op.lsmid,
                                      file->f_path.dentry->d_name.name,
                                      &p);
        put_task_struct(task);
        if (length > 0)
                length = simple_read_from_buffer(buf, count, ppos, p, length);
        kfree(p);
        return length;
}

static ssize_t proc_pid_attr_write(struct file * file, const char __user * buf,
                                   size_t count, loff_t *ppos)
{
        struct inode * inode = file_inode(file);
        struct task_struct *task;
        void *page;
        int rv;

        /* A task may only write when it was the opener. */
        if (file->private_data != current->mm)
                return -EPERM;

        rcu_read_lock();
        task = pid_task(proc_pid(inode), PIDTYPE_PID);
        if (!task) {
                rcu_read_unlock();
                return -ESRCH;
        }
        /* A task may only write its own attributes. */
        if (current != task) {
                rcu_read_unlock();
                return -EACCES;
        }
        /* Prevent changes to overridden credentials. */
        if (current_cred() != current_real_cred()) {
                rcu_read_unlock();
                return -EBUSY;
        }
        rcu_read_unlock();

        if (count > PAGE_SIZE)
                count = PAGE_SIZE;

        /* No partial writes. */
        if (*ppos != 0)
                return -EINVAL;

        page = memdup_user(buf, count);
        if (IS_ERR(page)) {
                rv = PTR_ERR(page);
                goto out;
        }

        /* Guard against adverse ptrace interaction */
        rv = mutex_lock_interruptible(&current->signal->cred_guard_mutex);
        if (rv < 0)
                goto out_free;

        rv = security_setprocattr(PROC_I(inode)->op.lsmid,
                                  file->f_path.dentry->d_name.name, page,
                                  count);
        mutex_unlock(&current->signal->cred_guard_mutex);
out_free:
        kfree(page);
out:
        return rv;
}

static const struct file_operations proc_pid_attr_operations = {
        .open                = proc_pid_attr_open,
        .read                = proc_pid_attr_read,
        .write                = proc_pid_attr_write,
        .llseek                = generic_file_llseek,
        .release        = mem_release,
};

#define LSM_DIR_OPS(LSM) \
static int proc_##LSM##_attr_dir_iterate(struct file *filp, \
                             struct dir_context *ctx) \
{ \
        return proc_pident_readdir(filp, ctx, \
                                   LSM##_attr_dir_stuff, \
                                   ARRAY_SIZE(LSM##_attr_dir_stuff)); \
} \
\
static const struct file_operations proc_##LSM##_attr_dir_ops = { \
        .read                = generic_read_dir, \
        .iterate_shared        = proc_##LSM##_attr_dir_iterate, \
        .llseek                = default_llseek, \
}; \
\
static struct dentry *proc_##LSM##_attr_dir_lookup(struct inode *dir, \
                                struct dentry *dentry, unsigned int flags) \
{ \
        return proc_pident_lookup(dir, dentry, \
                                  LSM##_attr_dir_stuff, \
                                  LSM##_attr_dir_stuff + ARRAY_SIZE(LSM##_attr_dir_stuff)); \
} \
\
static const struct inode_operations proc_##LSM##_attr_dir_inode_ops = { \
        .lookup                = proc_##LSM##_attr_dir_lookup, \
        .getattr        = pid_getattr, \
        .setattr        = proc_setattr, \
}

#ifdef CONFIG_SECURITY_SMACK
static const struct pid_entry smack_attr_dir_stuff[] = {
        ATTR(LSM_ID_SMACK, "current",        0666),
};
LSM_DIR_OPS(smack);
#endif

#ifdef CONFIG_SECURITY_APPARMOR
static const struct pid_entry apparmor_attr_dir_stuff[] = {
        ATTR(LSM_ID_APPARMOR, "current",        0666),
        ATTR(LSM_ID_APPARMOR, "prev",                0444),
        ATTR(LSM_ID_APPARMOR, "exec",                0666),
};
LSM_DIR_OPS(apparmor);
#endif

static const struct pid_entry attr_dir_stuff[] = {
        ATTR(LSM_ID_UNDEF, "current",        0666),
        ATTR(LSM_ID_UNDEF, "prev",                0444),
        ATTR(LSM_ID_UNDEF, "exec",                0666),
        ATTR(LSM_ID_UNDEF, "fscreate",        0666),
        ATTR(LSM_ID_UNDEF, "keycreate",        0666),
        ATTR(LSM_ID_UNDEF, "sockcreate",        0666),
#ifdef CONFIG_SECURITY_SMACK
        DIR("smack",                        0555,
            proc_smack_attr_dir_inode_ops, proc_smack_attr_dir_ops),
#endif
#ifdef CONFIG_SECURITY_APPARMOR
        DIR("apparmor",                        0555,
            proc_apparmor_attr_dir_inode_ops, proc_apparmor_attr_dir_ops),
#endif
};

static int proc_attr_dir_readdir(struct file *file, struct dir_context *ctx)
{
        return proc_pident_readdir(file, ctx, 
                                   attr_dir_stuff, ARRAY_SIZE(attr_dir_stuff));
}

static const struct file_operations proc_attr_dir_operations = {
        .read                = generic_read_dir,
        .iterate_shared        = proc_attr_dir_readdir,
        .llseek                = generic_file_llseek,
};

static struct dentry *proc_attr_dir_lookup(struct inode *dir,
                                struct dentry *dentry, unsigned int flags)
{
        return proc_pident_lookup(dir, dentry,
                                  attr_dir_stuff,
                                  attr_dir_stuff + ARRAY_SIZE(attr_dir_stuff));
}

static const struct inode_operations proc_attr_dir_inode_operations = {
        .lookup                = proc_attr_dir_lookup,
        .getattr        = pid_getattr,
        .setattr        = proc_setattr,
};

#endif

#ifdef CONFIG_ELF_CORE
static ssize_t proc_coredump_filter_read(struct file *file, char __user *buf,
                                         size_t count, loff_t *ppos)
{
        struct task_struct *task = get_proc_task(file_inode(file));
        struct mm_struct *mm;
        char buffer[PROC_NUMBUF];
        size_t len;
        int ret;

        if (!task)
                return -ESRCH;

        ret = 0;
        mm = get_task_mm(task);
        if (mm) {
                len = snprintf(buffer, sizeof(buffer), "%08lx\n",
                               ((mm->flags & MMF_DUMP_FILTER_MASK) >>
                                MMF_DUMP_FILTER_SHIFT));
                mmput(mm);
                ret = simple_read_from_buffer(buf, count, ppos, buffer, len);
        }

        put_task_struct(task);

        return ret;
}

static ssize_t proc_coredump_filter_write(struct file *file,
                                          const char __user *buf,
                                          size_t count,
                                          loff_t *ppos)
{
        struct task_struct *task;
        struct mm_struct *mm;
        unsigned int val;
        int ret;
        int i;
        unsigned long mask;

        ret = kstrtouint_from_user(buf, count, 0, &val);
        if (ret < 0)
                return ret;

        ret = -ESRCH;
        task = get_proc_task(file_inode(file));
        if (!task)
                goto out_no_task;

        mm = get_task_mm(task);
        if (!mm)
                goto out_no_mm;
        ret = 0;

        for (i = 0, mask = 1; i < MMF_DUMP_FILTER_BITS; i++, mask <<= 1) {
                if (val & mask)
                        set_bit(i + MMF_DUMP_FILTER_SHIFT, &mm->flags);
                else
                        clear_bit(i + MMF_DUMP_FILTER_SHIFT, &mm->flags);
        }

        mmput(mm);
 out_no_mm:
        put_task_struct(task);
 out_no_task:
        if (ret < 0)
                return ret;
        return count;
}

static const struct file_operations proc_coredump_filter_operations = {
        .read                = proc_coredump_filter_read,
        .write                = proc_coredump_filter_write,
        .llseek                = generic_file_llseek,
};
#endif

#ifdef CONFIG_TASK_IO_ACCOUNTING
static int do_io_accounting(struct task_struct *task, struct seq_file *m, int whole)
{
        struct task_io_accounting acct;
        int result;

        result = down_read_killable(&task->signal->exec_update_lock);
        if (result)
                return result;

        if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
                result = -EACCES;
                goto out_unlock;
        }

        if (whole) {
                struct signal_struct *sig = task->signal;
                struct task_struct *t;
                unsigned int seq = 1;
                unsigned long flags;

                rcu_read_lock();
                do {
                        seq++; /* 2 on the 1st/lockless path, otherwise odd */
                        flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);

                        acct = sig->ioac;
                        __for_each_thread(sig, t)
                                task_io_accounting_add(&acct, &t->ioac);

                } while (need_seqretry(&sig->stats_lock, seq));
                done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
                rcu_read_unlock();
        } else {
                acct = task->ioac;
        }

        seq_printf(m,
                   "rchar: %llu\n"
                   "wchar: %llu\n"
                   "syscr: %llu\n"
                   "syscw: %llu\n"
                   "read_bytes: %llu\n"
                   "write_bytes: %llu\n"
                   "cancelled_write_bytes: %llu\n",
                   (unsigned long long)acct.rchar,
                   (unsigned long long)acct.wchar,
                   (unsigned long long)acct.syscr,
                   (unsigned long long)acct.syscw,
                   (unsigned long long)acct.read_bytes,
                   (unsigned long long)acct.write_bytes,
                   (unsigned long long)acct.cancelled_write_bytes);
        result = 0;

out_unlock:
        up_read(&task->signal->exec_update_lock);
        return result;
}

static int proc_tid_io_accounting(struct seq_file *m, struct pid_namespace *ns,
                                  struct pid *pid, struct task_struct *task)
{
        return do_io_accounting(task, m, 0);
}

static int proc_tgid_io_accounting(struct seq_file *m, struct pid_namespace *ns,
                                   struct pid *pid, struct task_struct *task)
{
        return do_io_accounting(task, m, 1);
}
#endif /* CONFIG_TASK_IO_ACCOUNTING */

#ifdef CONFIG_USER_NS
static int proc_id_map_open(struct inode *inode, struct file *file,
        const struct seq_operations *seq_ops)
{
        struct user_namespace *ns = NULL;
        struct task_struct *task;
        struct seq_file *seq;
        int ret = -EINVAL;

        task = get_proc_task(inode);
        if (task) {
                rcu_read_lock();
                ns = get_user_ns(task_cred_xxx(task, user_ns));
                rcu_read_unlock();
                put_task_struct(task);
        }
        if (!ns)
                goto err;

        ret = seq_open(file, seq_ops);
        if (ret)
                goto err_put_ns;

        seq = file->private_data;
        seq->private = ns;

        return 0;
err_put_ns:
        put_user_ns(ns);
err:
        return ret;
}

static int proc_id_map_release(struct inode *inode, struct file *file)
{
        struct seq_file *seq = file->private_data;
        struct user_namespace *ns = seq->private;
        put_user_ns(ns);
        return seq_release(inode, file);
}

static int proc_uid_map_open(struct inode *inode, struct file *file)
{
        return proc_id_map_open(inode, file, &proc_uid_seq_operations);
}

static int proc_gid_map_open(struct inode *inode, struct file *file)
{
        return proc_id_map_open(inode, file, &proc_gid_seq_operations);
}

static int proc_projid_map_open(struct inode *inode, struct file *file)
{
        return proc_id_map_open(inode, file, &proc_projid_seq_operations);
}

static const struct file_operations proc_uid_map_operations = {
        .open                = proc_uid_map_open,
        .write                = proc_uid_map_write,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = proc_id_map_release,
};

static const struct file_operations proc_gid_map_operations = {
        .open                = proc_gid_map_open,
        .write                = proc_gid_map_write,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = proc_id_map_release,
};

static const struct file_operations proc_projid_map_operations = {
        .open                = proc_projid_map_open,
        .write                = proc_projid_map_write,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = proc_id_map_release,
};

static int proc_setgroups_open(struct inode *inode, struct file *file)
{
        struct user_namespace *ns = NULL;
        struct task_struct *task;
        int ret;

        ret = -ESRCH;
        task = get_proc_task(inode);
        if (task) {
                rcu_read_lock();
                ns = get_user_ns(task_cred_xxx(task, user_ns));
                rcu_read_unlock();
                put_task_struct(task);
        }
        if (!ns)
                goto err;

        if (file->f_mode & FMODE_WRITE) {
                ret = -EACCES;
                if (!ns_capable(ns, CAP_SYS_ADMIN))
                        goto err_put_ns;
        }

        ret = single_open(file, &proc_setgroups_show, ns);
        if (ret)
                goto err_put_ns;

        return 0;
err_put_ns:
        put_user_ns(ns);
err:
        return ret;
}

static int proc_setgroups_release(struct inode *inode, struct file *file)
{
        struct seq_file *seq = file->private_data;
        struct user_namespace *ns = seq->private;
        int ret = single_release(inode, file);
        put_user_ns(ns);
        return ret;
}

static const struct file_operations proc_setgroups_operations = {
        .open                = proc_setgroups_open,
        .write                = proc_setgroups_write,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = proc_setgroups_release,
};
#endif /* CONFIG_USER_NS */

static int proc_pid_personality(struct seq_file *m, struct pid_namespace *ns,
                                struct pid *pid, struct task_struct *task)
{
        int err = lock_trace(task);
        if (!err) {
                seq_printf(m, "%08x\n", task->personality);
                unlock_trace(task);
        }
        return err;
}

#ifdef CONFIG_LIVEPATCH
static int proc_pid_patch_state(struct seq_file *m, struct pid_namespace *ns,
                                struct pid *pid, struct task_struct *task)
{
        seq_printf(m, "%d\n", task->patch_state);
        return 0;
}
#endif /* CONFIG_LIVEPATCH */

#ifdef CONFIG_KSM
static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace *ns,
                                struct pid *pid, struct task_struct *task)
{
        struct mm_struct *mm;

        mm = get_task_mm(task);
        if (mm) {
                seq_printf(m, "%lu\n", mm->ksm_merging_pages);
                mmput(mm);
        }

        return 0;
}
static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns,
                                struct pid *pid, struct task_struct *task)
{
        struct mm_struct *mm;

        mm = get_task_mm(task);
        if (mm) {
                seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items);
                seq_printf(m, "ksm_zero_pages %ld\n", mm_ksm_zero_pages(mm));
                seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages);
                seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm));
                mmput(mm);
        }

        return 0;
}
#endif /* CONFIG_KSM */

#ifdef CONFIG_STACKLEAK_METRICS
static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns,
                                struct pid *pid, struct task_struct *task)
{
        unsigned long prev_depth = THREAD_SIZE -
                                (task->prev_lowest_stack & (THREAD_SIZE - 1));
        unsigned long depth = THREAD_SIZE -
                                (task->lowest_stack & (THREAD_SIZE - 1));

        seq_printf(m, "previous stack depth: %lu\nstack depth: %lu\n",
                                                        prev_depth, depth);
        return 0;
}
#endif /* CONFIG_STACKLEAK_METRICS */

/*
 * Thread groups
 */
static const struct file_operations proc_task_operations;
static const struct inode_operations proc_task_inode_operations;

static const struct pid_entry tgid_base_stuff[] = {
        DIR("task",       S_IRUGO|S_IXUGO, proc_task_inode_operations, proc_task_operations),
        DIR("fd",         S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations),
        DIR("map_files",  S_IRUSR|S_IXUSR, proc_map_files_inode_operations, proc_map_files_operations),
        DIR("fdinfo",     S_IRUGO|S_IXUGO, proc_fdinfo_inode_operations, proc_fdinfo_operations),
        DIR("ns",          S_IRUSR|S_IXUGO, proc_ns_dir_inode_operations, proc_ns_dir_operations),
#ifdef CONFIG_NET
        DIR("net",        S_IRUGO|S_IXUGO, proc_net_inode_operations, proc_net_operations),
#endif
        REG("environ",    S_IRUSR, proc_environ_operations),
        REG("auxv",       S_IRUSR, proc_auxv_operations),
        ONE("status",     S_IRUGO, proc_pid_status),
        ONE("personality", S_IRUSR, proc_pid_personality),
        ONE("limits",          S_IRUGO, proc_pid_limits),
#ifdef CONFIG_SCHED_DEBUG
        REG("sched",      S_IRUGO|S_IWUSR, proc_pid_sched_operations),
#endif
#ifdef CONFIG_SCHED_AUTOGROUP
        REG("autogroup",  S_IRUGO|S_IWUSR, proc_pid_sched_autogroup_operations),
#endif
#ifdef CONFIG_TIME_NS
        REG("timens_offsets",  S_IRUGO|S_IWUSR, proc_timens_offsets_operations),
#endif
        REG("comm",      S_IRUGO|S_IWUSR, proc_pid_set_comm_operations),
#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
        ONE("syscall",    S_IRUSR, proc_pid_syscall),
#endif
        REG("cmdline",    S_IRUGO, proc_pid_cmdline_ops),
        ONE("stat",       S_IRUGO, proc_tgid_stat),
        ONE("statm",      S_IRUGO, proc_pid_statm),
        REG("maps",       S_IRUGO, proc_pid_maps_operations),
#ifdef CONFIG_NUMA
        REG("numa_maps",  S_IRUGO, proc_pid_numa_maps_operations),
#endif
        REG("mem",        S_IRUSR|S_IWUSR, proc_mem_operations),
        LNK("cwd",        proc_cwd_link),
        LNK("root",       proc_root_link),
        LNK("exe",        proc_exe_link),
        REG("mounts",     S_IRUGO, proc_mounts_operations),
        REG("mountinfo",  S_IRUGO, proc_mountinfo_operations),
        REG("mountstats", S_IRUSR, proc_mountstats_operations),
#ifdef CONFIG_PROC_PAGE_MONITOR
        REG("clear_refs", S_IWUSR, proc_clear_refs_operations),
        REG("smaps",      S_IRUGO, proc_pid_smaps_operations),
        REG("smaps_rollup", S_IRUGO, proc_pid_smaps_rollup_operations),
        REG("pagemap",    S_IRUSR, proc_pagemap_operations),
#endif
#ifdef CONFIG_SECURITY
        DIR("attr",       S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations),
#endif
#ifdef CONFIG_KALLSYMS
        ONE("wchan",      S_IRUGO, proc_pid_wchan),
#endif
#ifdef CONFIG_STACKTRACE
        ONE("stack",      S_IRUSR, proc_pid_stack),
#endif
#ifdef CONFIG_SCHED_INFO
        ONE("schedstat",  S_IRUGO, proc_pid_schedstat),
#endif
#ifdef CONFIG_LATENCYTOP
        REG("latency",  S_IRUGO, proc_lstats_operations),
#endif
#ifdef CONFIG_PROC_PID_CPUSET
        ONE("cpuset",     S_IRUGO, proc_cpuset_show),
#endif
#ifdef CONFIG_CGROUPS
        ONE("cgroup",  S_IRUGO, proc_cgroup_show),
#endif
#ifdef CONFIG_PROC_CPU_RESCTRL
        ONE("cpu_resctrl_groups", S_IRUGO, proc_resctrl_show),
#endif
        ONE("oom_score",  S_IRUGO, proc_oom_score),
        REG("oom_adj",    S_IRUGO|S_IWUSR, proc_oom_adj_operations),
        REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations),
#ifdef CONFIG_AUDIT
        REG("loginuid",   S_IWUSR|S_IRUGO, proc_loginuid_operations),
        REG("sessionid",  S_IRUGO, proc_sessionid_operations),
#endif
#ifdef CONFIG_FAULT_INJECTION
        REG("make-it-fail", S_IRUGO|S_IWUSR, proc_fault_inject_operations),
        REG("fail-nth", 0644, proc_fail_nth_operations),
#endif
#ifdef CONFIG_ELF_CORE
        REG("coredump_filter", S_IRUGO|S_IWUSR, proc_coredump_filter_operations),
#endif
#ifdef CONFIG_TASK_IO_ACCOUNTING
        ONE("io",        S_IRUSR, proc_tgid_io_accounting),
#endif
#ifdef CONFIG_USER_NS
        REG("uid_map",    S_IRUGO|S_IWUSR, proc_uid_map_operations),
        REG("gid_map",    S_IRUGO|S_IWUSR, proc_gid_map_operations),
        REG("projid_map", S_IRUGO|S_IWUSR, proc_projid_map_operations),
        REG("setgroups",  S_IRUGO|S_IWUSR, proc_setgroups_operations),
#endif
#if defined(CONFIG_CHECKPOINT_RESTORE) && defined(CONFIG_POSIX_TIMERS)
        REG("timers",          S_IRUGO, proc_timers_operations),
#endif
        REG("timerslack_ns", S_IRUGO|S_IWUGO, proc_pid_set_timerslack_ns_operations),
#ifdef CONFIG_LIVEPATCH
        ONE("patch_state",  S_IRUSR, proc_pid_patch_state),
#endif
#ifdef CONFIG_STACKLEAK_METRICS
        ONE("stack_depth", S_IRUGO, proc_stack_depth),
#endif
#ifdef CONFIG_PROC_PID_ARCH_STATUS
        ONE("arch_status", S_IRUGO, proc_pid_arch_status),
#endif
#ifdef CONFIG_SECCOMP_CACHE_DEBUG
        ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache),
#endif
#ifdef CONFIG_KSM
        ONE("ksm_merging_pages",  S_IRUSR, proc_pid_ksm_merging_pages),
        ONE("ksm_stat",  S_IRUSR, proc_pid_ksm_stat),
#endif
};

static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx)
{
        return proc_pident_readdir(file, ctx,
                                   tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff));
}

static const struct file_operations proc_tgid_base_operations = {
        .read                = generic_read_dir,
        .iterate_shared        = proc_tgid_base_readdir,
        .llseek                = generic_file_llseek,
};

struct pid *tgid_pidfd_to_pid(const struct file *file)
{
        if (file->f_op != &proc_tgid_base_operations)
                return ERR_PTR(-EBADF);

        return proc_pid(file_inode(file));
}

static struct dentry *proc_tgid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
        return proc_pident_lookup(dir, dentry,
                                  tgid_base_stuff,
                                  tgid_base_stuff + ARRAY_SIZE(tgid_base_stuff));
}

static const struct inode_operations proc_tgid_base_inode_operations = {
        .lookup                = proc_tgid_base_lookup,
        .getattr        = pid_getattr,
        .setattr        = proc_setattr,
        .permission        = proc_pid_permission,
};

/**
 * proc_flush_pid -  Remove dcache entries for @pid from the /proc dcache.
 * @pid: pid that should be flushed.
 *
 * This function walks a list of inodes (that belong to any proc
 * filesystem) that are attached to the pid and flushes them from
 * the dentry cache.
 *
 * It is safe and reasonable to cache /proc entries for a task until
 * that task exits.  After that they just clog up the dcache with
 * useless entries, possibly causing useful dcache entries to be
 * flushed instead.  This routine is provided to flush those useless
 * dcache entries when a process is reaped.
 *
 * NOTE: This routine is just an optimization so it does not guarantee
 *       that no dcache entries will exist after a process is reaped
 *       it just makes it very unlikely that any will persist.
 */

void proc_flush_pid(struct pid *pid)
{
        proc_invalidate_siblings_dcache(&pid->inodes, &pid->lock);
}

static struct dentry *proc_pid_instantiate(struct dentry * dentry,
                                   struct task_struct *task, const void *ptr)
{
        struct inode *inode;

        inode = proc_pid_make_base_inode(dentry->d_sb, task,
                                         S_IFDIR | S_IRUGO | S_IXUGO);
        if (!inode)
                return ERR_PTR(-ENOENT);

        inode->i_op = &proc_tgid_base_inode_operations;
        inode->i_fop = &proc_tgid_base_operations;
        inode->i_flags|=S_IMMUTABLE;

        set_nlink(inode, nlink_tgid);
        pid_update_inode(task, inode);

        d_set_d_op(dentry, &pid_dentry_operations);
        return d_splice_alias(inode, dentry);
}

struct dentry *proc_pid_lookup(struct dentry *dentry, unsigned int flags)
{
        struct task_struct *task;
        unsigned tgid;
        struct proc_fs_info *fs_info;
        struct pid_namespace *ns;
        struct dentry *result = ERR_PTR(-ENOENT);

        tgid = name_to_int(&dentry->d_name);
        if (tgid == ~0U)
                goto out;

        fs_info = proc_sb_info(dentry->d_sb);
        ns = fs_info->pid_ns;
        rcu_read_lock();
        task = find_task_by_pid_ns(tgid, ns);
        if (task)
                get_task_struct(task);
        rcu_read_unlock();
        if (!task)
                goto out;

        /* Limit procfs to only ptraceable tasks */
        if (fs_info->hide_pid == HIDEPID_NOT_PTRACEABLE) {
                if (!has_pid_permissions(fs_info, task, HIDEPID_NO_ACCESS))
                        goto out_put_task;
        }

        result = proc_pid_instantiate(dentry, task, NULL);
out_put_task:
        put_task_struct(task);
out:
        return result;
}

/*
 * Find the first task with tgid >= tgid
 *
 */
struct tgid_iter {
        unsigned int tgid;
        struct task_struct *task;
};
static struct tgid_iter next_tgid(struct pid_namespace *ns, struct tgid_iter iter)
{
        struct pid *pid;

        if (iter.task)
                put_task_struct(iter.task);
        rcu_read_lock();
retry:
        iter.task = NULL;
        pid = find_ge_pid(iter.tgid, ns);
        if (pid) {
                iter.tgid = pid_nr_ns(pid, ns);
                iter.task = pid_task(pid, PIDTYPE_TGID);
                if (!iter.task) {
                        iter.tgid += 1;
                        goto retry;
                }
                get_task_struct(iter.task);
        }
        rcu_read_unlock();
        return iter;
}

#define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2)

/* for the /proc/ directory itself, after non-process stuff has been done */
int proc_pid_readdir(struct file *file, struct dir_context *ctx)
{
        struct tgid_iter iter;
        struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb);
        struct pid_namespace *ns = proc_pid_ns(file_inode(file)->i_sb);
        loff_t pos = ctx->pos;

        if (pos >= PID_MAX_LIMIT + TGID_OFFSET)
                return 0;

        if (pos == TGID_OFFSET - 2) {
                struct inode *inode = d_inode(fs_info->proc_self);
                if (!dir_emit(ctx, "self", 4, inode->i_ino, DT_LNK))
                        return 0;
                ctx->pos = pos = pos + 1;
        }
        if (pos == TGID_OFFSET - 1) {
                struct inode *inode = d_inode(fs_info->proc_thread_self);
                if (!dir_emit(ctx, "thread-self", 11, inode->i_ino, DT_LNK))
                        return 0;
                ctx->pos = pos = pos + 1;
        }
        iter.tgid = pos - TGID_OFFSET;
        iter.task = NULL;
        for (iter = next_tgid(ns, iter);
             iter.task;
             iter.tgid += 1, iter = next_tgid(ns, iter)) {
                char name[10 + 1];
                unsigned int len;

                cond_resched();
                if (!has_pid_permissions(fs_info, iter.task, HIDEPID_INVISIBLE))
                        continue;

                len = snprintf(name, sizeof(name), "%u", iter.tgid);
                ctx->pos = iter.tgid + TGID_OFFSET;
                if (!proc_fill_cache(file, ctx, name, len,
                                     proc_pid_instantiate, iter.task, NULL)) {
                        put_task_struct(iter.task);
                        return 0;
                }
        }
        ctx->pos = PID_MAX_LIMIT + TGID_OFFSET;
        return 0;
}

/*
 * proc_tid_comm_permission is a special permission function exclusively
 * used for the node /proc/<pid>/task/<tid>/comm.
 * It bypasses generic permission checks in the case where a task of the same
 * task group attempts to access the node.
 * The rationale behind this is that glibc and bionic access this node for
 * cross thread naming (pthread_set/getname_np(!self)). However, if
 * PR_SET_DUMPABLE gets set to 0 this node among others becomes uid=0 gid=0,
 * which locks out the cross thread naming implementation.
 * This function makes sure that the node is always accessible for members of
 * same thread group.
 */
static int proc_tid_comm_permission(struct mnt_idmap *idmap,
                                    struct inode *inode, int mask)
{
        bool is_same_tgroup;
        struct task_struct *task;

        task = get_proc_task(inode);
        if (!task)
                return -ESRCH;
        is_same_tgroup = same_thread_group(current, task);
        put_task_struct(task);

        if (likely(is_same_tgroup && !(mask & MAY_EXEC))) {
                /* This file (/proc/<pid>/task/<tid>/comm) can always be
                 * read or written by the members of the corresponding
                 * thread group.
                 */
                return 0;
        }

        return generic_permission(&nop_mnt_idmap, inode, mask);
}

static const struct inode_operations proc_tid_comm_inode_operations = {
                .setattr        = proc_setattr,
                .permission        = proc_tid_comm_permission,
};

/*
 * Tasks
 */
static const struct pid_entry tid_base_stuff[] = {
        DIR("fd",        S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations),
        DIR("fdinfo",    S_IRUGO|S_IXUGO, proc_fdinfo_inode_operations, proc_fdinfo_operations),
        DIR("ns",         S_IRUSR|S_IXUGO, proc_ns_dir_inode_operations, proc_ns_dir_operations),
#ifdef CONFIG_NET
        DIR("net",        S_IRUGO|S_IXUGO, proc_net_inode_operations, proc_net_operations),
#endif
        REG("environ",   S_IRUSR, proc_environ_operations),
        REG("auxv",      S_IRUSR, proc_auxv_operations),
        ONE("status",    S_IRUGO, proc_pid_status),
        ONE("personality", S_IRUSR, proc_pid_personality),
        ONE("limits",         S_IRUGO, proc_pid_limits),
#ifdef CONFIG_SCHED_DEBUG
        REG("sched",     S_IRUGO|S_IWUSR, proc_pid_sched_operations),
#endif
        NOD("comm",      S_IFREG|S_IRUGO|S_IWUSR,
                         &proc_tid_comm_inode_operations,
                         &proc_pid_set_comm_operations, {}),
#ifdef CONFIG_HAVE_ARCH_TRACEHOOK
        ONE("syscall",   S_IRUSR, proc_pid_syscall),
#endif
        REG("cmdline",   S_IRUGO, proc_pid_cmdline_ops),
        ONE("stat",      S_IRUGO, proc_tid_stat),
        ONE("statm",     S_IRUGO, proc_pid_statm),
        REG("maps",      S_IRUGO, proc_pid_maps_operations),
#ifdef CONFIG_PROC_CHILDREN
        REG("children",  S_IRUGO, proc_tid_children_operations),
#endif
#ifdef CONFIG_NUMA
        REG("numa_maps", S_IRUGO, proc_pid_numa_maps_operations),
#endif
        REG("mem",       S_IRUSR|S_IWUSR, proc_mem_operations),
        LNK("cwd",       proc_cwd_link),
        LNK("root",      proc_root_link),
        LNK("exe",       proc_exe_link),
        REG("mounts",    S_IRUGO, proc_mounts_operations),
        REG("mountinfo",  S_IRUGO, proc_mountinfo_operations),
#ifdef CONFIG_PROC_PAGE_MONITOR
        REG("clear_refs", S_IWUSR, proc_clear_refs_operations),
        REG("smaps",     S_IRUGO, proc_pid_smaps_operations),
        REG("smaps_rollup", S_IRUGO, proc_pid_smaps_rollup_operations),
        REG("pagemap",    S_IRUSR, proc_pagemap_operations),
#endif
#ifdef CONFIG_SECURITY
        DIR("attr",      S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations),
#endif
#ifdef CONFIG_KALLSYMS
        ONE("wchan",     S_IRUGO, proc_pid_wchan),
#endif
#ifdef CONFIG_STACKTRACE
        ONE("stack",      S_IRUSR, proc_pid_stack),
#endif
#ifdef CONFIG_SCHED_INFO
        ONE("schedstat", S_IRUGO, proc_pid_schedstat),
#endif
#ifdef CONFIG_LATENCYTOP
        REG("latency",  S_IRUGO, proc_lstats_operations),
#endif
#ifdef CONFIG_PROC_PID_CPUSET
        ONE("cpuset",    S_IRUGO, proc_cpuset_show),
#endif
#ifdef CONFIG_CGROUPS
        ONE("cgroup",  S_IRUGO, proc_cgroup_show),
#endif
#ifdef CONFIG_PROC_CPU_RESCTRL
        ONE("cpu_resctrl_groups", S_IRUGO, proc_resctrl_show),
#endif
        ONE("oom_score", S_IRUGO, proc_oom_score),
        REG("oom_adj",   S_IRUGO|S_IWUSR, proc_oom_adj_operations),
        REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations),
#ifdef CONFIG_AUDIT
        REG("loginuid",  S_IWUSR|S_IRUGO, proc_loginuid_operations),
        REG("sessionid",  S_IRUGO, proc_sessionid_operations),
#endif
#ifdef CONFIG_FAULT_INJECTION
        REG("make-it-fail", S_IRUGO|S_IWUSR, proc_fault_inject_operations),
        REG("fail-nth", 0644, proc_fail_nth_operations),
#endif
#ifdef CONFIG_TASK_IO_ACCOUNTING
        ONE("io",        S_IRUSR, proc_tid_io_accounting),
#endif
#ifdef CONFIG_USER_NS
        REG("uid_map",    S_IRUGO|S_IWUSR, proc_uid_map_operations),
        REG("gid_map",    S_IRUGO|S_IWUSR, proc_gid_map_operations),
        REG("projid_map", S_IRUGO|S_IWUSR, proc_projid_map_operations),
        REG("setgroups",  S_IRUGO|S_IWUSR, proc_setgroups_operations),
#endif
#ifdef CONFIG_LIVEPATCH
        ONE("patch_state",  S_IRUSR, proc_pid_patch_state),
#endif
#ifdef CONFIG_PROC_PID_ARCH_STATUS
        ONE("arch_status", S_IRUGO, proc_pid_arch_status),
#endif
#ifdef CONFIG_SECCOMP_CACHE_DEBUG
        ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache),
#endif
#ifdef CONFIG_KSM
        ONE("ksm_merging_pages",  S_IRUSR, proc_pid_ksm_merging_pages),
        ONE("ksm_stat",  S_IRUSR, proc_pid_ksm_stat),
#endif
};

static int proc_tid_base_readdir(struct file *file, struct dir_context *ctx)
{
        return proc_pident_readdir(file, ctx,
                                   tid_base_stuff, ARRAY_SIZE(tid_base_stuff));
}

static struct dentry *proc_tid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
        return proc_pident_lookup(dir, dentry,
                                  tid_base_stuff,
                                  tid_base_stuff + ARRAY_SIZE(tid_base_stuff));
}

static const struct file_operations proc_tid_base_operations = {
        .read                = generic_read_dir,
        .iterate_shared        = proc_tid_base_readdir,
        .llseek                = generic_file_llseek,
};

static const struct inode_operations proc_tid_base_inode_operations = {
        .lookup                = proc_tid_base_lookup,
        .getattr        = pid_getattr,
        .setattr        = proc_setattr,
};

static struct dentry *proc_task_instantiate(struct dentry *dentry,
        struct task_struct *task, const void *ptr)
{
        struct inode *inode;
        inode = proc_pid_make_base_inode(dentry->d_sb, task,
                                         S_IFDIR | S_IRUGO | S_IXUGO);
        if (!inode)
                return ERR_PTR(-ENOENT);

        inode->i_op = &proc_tid_base_inode_operations;
        inode->i_fop = &proc_tid_base_operations;
        inode->i_flags |= S_IMMUTABLE;

        set_nlink(inode, nlink_tid);
        pid_update_inode(task, inode);

        d_set_d_op(dentry, &pid_dentry_operations);
        return d_splice_alias(inode, dentry);
}

static struct dentry *proc_task_lookup(struct inode *dir, struct dentry * dentry, unsigned int flags)
{
        struct task_struct *task;
        struct task_struct *leader = get_proc_task(dir);
        unsigned tid;
        struct proc_fs_info *fs_info;
        struct pid_namespace *ns;
        struct dentry *result = ERR_PTR(-ENOENT);

        if (!leader)
                goto out_no_task;

        tid = name_to_int(&dentry->d_name);
        if (tid == ~0U)
                goto out;

        fs_info = proc_sb_info(dentry->d_sb);
        ns = fs_info->pid_ns;
        rcu_read_lock();
        task = find_task_by_pid_ns(tid, ns);
        if (task)
                get_task_struct(task);
        rcu_read_unlock();
        if (!task)
                goto out;
        if (!same_thread_group(leader, task))
                goto out_drop_task;

        result = proc_task_instantiate(dentry, task, NULL);
out_drop_task:
        put_task_struct(task);
out:
        put_task_struct(leader);
out_no_task:
        return result;
}

/*
 * Find the first tid of a thread group to return to user space.
 *
 * Usually this is just the thread group leader, but if the users
 * buffer was too small or there was a seek into the middle of the
 * directory we have more work todo.
 *
 * In the case of a short read we start with find_task_by_pid.
 *
 * In the case of a seek we start with the leader and walk nr
 * threads past it.
 */
static struct task_struct *first_tid(struct pid *pid, int tid, loff_t f_pos,
                                        struct pid_namespace *ns)
{
        struct task_struct *pos, *task;
        unsigned long nr = f_pos;

        if (nr != f_pos)        /* 32bit overflow? */
                return NULL;

        rcu_read_lock();
        task = pid_task(pid, PIDTYPE_PID);
        if (!task)
                goto fail;

        /* Attempt to start with the tid of a thread */
        if (tid && nr) {
                pos = find_task_by_pid_ns(tid, ns);
                if (pos && same_thread_group(pos, task))
                        goto found;
        }

        /* If nr exceeds the number of threads there is nothing todo */
        if (nr >= get_nr_threads(task))
                goto fail;

        /* If we haven't found our starting place yet start
         * with the leader and walk nr threads forward.
         */
        for_each_thread(task, pos) {
                if (!nr--)
                        goto found;
        }
fail:
        pos = NULL;
        goto out;
found:
        get_task_struct(pos);
out:
        rcu_read_unlock();
        return pos;
}

/*
 * Find the next thread in the thread list.
 * Return NULL if there is an error or no next thread.
 *
 * The reference to the input task_struct is released.
 */
static struct task_struct *next_tid(struct task_struct *start)
{
        struct task_struct *pos = NULL;
        rcu_read_lock();
        if (pid_alive(start)) {
                pos = __next_thread(start);
                if (pos)
                        get_task_struct(pos);
        }
        rcu_read_unlock();
        put_task_struct(start);
        return pos;
}

/* for the /proc/TGID/task/ directories */
static int proc_task_readdir(struct file *file, struct dir_context *ctx)
{
        struct inode *inode = file_inode(file);
        struct task_struct *task;
        struct pid_namespace *ns;
        int tid;

        if (proc_inode_is_dead(inode))
                return -ENOENT;

        if (!dir_emit_dots(file, ctx))
                return 0;

        /* We cache the tgid value that the last readdir call couldn't
         * return and lseek resets it to 0.
         */
        ns = proc_pid_ns(inode->i_sb);
        tid = (int)(intptr_t)file->private_data;
        file->private_data = NULL;
        for (task = first_tid(proc_pid(inode), tid, ctx->pos - 2, ns);
             task;
             task = next_tid(task), ctx->pos++) {
                char name[10 + 1];
                unsigned int len;

                tid = task_pid_nr_ns(task, ns);
                if (!tid)
                        continue;        /* The task has just exited. */
                len = snprintf(name, sizeof(name), "%u", tid);
                if (!proc_fill_cache(file, ctx, name, len,
                                proc_task_instantiate, task, NULL)) {
                        /* returning this tgid failed, save it as the first
                         * pid for the next readir call */
                        file->private_data = (void *)(intptr_t)tid;
                        put_task_struct(task);
                        break;
                }
        }

        return 0;
}

static int proc_task_getattr(struct mnt_idmap *idmap,
                             const struct path *path, struct kstat *stat,
                             u32 request_mask, unsigned int query_flags)
{
        struct inode *inode = d_inode(path->dentry);
        struct task_struct *p = get_proc_task(inode);
        generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);

        if (p) {
                stat->nlink += get_nr_threads(p);
                put_task_struct(p);
        }

        return 0;
}

/*
 * proc_task_readdir() set @file->private_data to a positive integer
 * value, so casting that to u64 is safe. generic_llseek_cookie() will
 * set @cookie to 0, so casting to an int is safe. The WARN_ON_ONCE() is
 * here to catch any unexpected change in behavior either in
 * proc_task_readdir() or generic_llseek_cookie().
 */
static loff_t proc_dir_llseek(struct file *file, loff_t offset, int whence)
{
        u64 cookie = (u64)(intptr_t)file->private_data;
        loff_t off;

        off = generic_llseek_cookie(file, offset, whence, &cookie);
        WARN_ON_ONCE(cookie > INT_MAX);
        file->private_data = (void *)(intptr_t)cookie; /* serialized by f_pos_lock */
        return off;
}

static const struct inode_operations proc_task_inode_operations = {
        .lookup                = proc_task_lookup,
        .getattr        = proc_task_getattr,
        .setattr        = proc_setattr,
        .permission        = proc_pid_permission,
};

static const struct file_operations proc_task_operations = {
        .read                = generic_read_dir,
        .iterate_shared        = proc_task_readdir,
        .llseek                = proc_dir_llseek,
};

void __init set_proc_pid_nlink(void)
{
        nlink_tid = pid_entry_nlink(tid_base_stuff, ARRAY_SIZE(tid_base_stuff));
        nlink_tgid = pid_entry_nlink(tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff));
}











































   85 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _TRACE_SYSCALL_H
#define _TRACE_SYSCALL_H

#include <linux/tracepoint.h>
#include <linux/unistd.h>
#include <linux/trace_events.h>
#include <linux/thread_info.h>

#include <asm/ptrace.h>


/*
 * A syscall entry in the ftrace syscalls array.
 *
 * @name: name of the syscall
 * @syscall_nr: number of the syscall
 * @nb_args: number of parameters it takes
 * @types: list of types as strings
 * @args: list of args as strings (args[i] matches types[i])
 * @enter_fields: list of fields for syscall_enter trace event
 * @enter_event: associated syscall_enter trace event
 * @exit_event: associated syscall_exit trace event
 */
struct syscall_metadata {
        const char        *name;
        int                syscall_nr;
        int                nb_args;
        const char        **types;
        const char        **args;
        struct list_head enter_fields;

        struct trace_event_call *enter_event;
        struct trace_event_call *exit_event;
};

#if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_HAVE_SYSCALL_TRACEPOINTS)
static inline void syscall_tracepoint_update(struct task_struct *p)
{
        if (test_syscall_work(SYSCALL_TRACEPOINT))
                set_task_syscall_work(p, SYSCALL_TRACEPOINT);
        else
                clear_task_syscall_work(p, SYSCALL_TRACEPOINT);
}
#else
static inline void syscall_tracepoint_update(struct task_struct *p)
{
}
#endif

#endif /* _TRACE_SYSCALL_H */

































































   16 

   26 


   16 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner, Simon Wunderlich, Antonio Quartulli
 */

#ifndef _NET_BATMAN_ADV_TRANSLATION_TABLE_H_
#define _NET_BATMAN_ADV_TRANSLATION_TABLE_H_

#include "main.h"

#include <linux/kref.h>
#include <linux/netdevice.h>
#include <linux/netlink.h>
#include <linux/skbuff.h>
#include <linux/types.h>

int batadv_tt_init(struct batadv_priv *bat_priv);
bool batadv_tt_local_add(struct net_device *soft_iface, const u8 *addr,
                         unsigned short vid, int ifindex, u32 mark);
u16 batadv_tt_local_remove(struct batadv_priv *bat_priv,
                           const u8 *addr, unsigned short vid,
                           const char *message, bool roaming);
int batadv_tt_local_dump(struct sk_buff *msg, struct netlink_callback *cb);
int batadv_tt_global_dump(struct sk_buff *msg, struct netlink_callback *cb);
void batadv_tt_global_del_orig(struct batadv_priv *bat_priv,
                               struct batadv_orig_node *orig_node,
                               s32 match_vid, const char *message);
struct batadv_tt_global_entry *
batadv_tt_global_hash_find(struct batadv_priv *bat_priv, const u8 *addr,
                           unsigned short vid);
void batadv_tt_global_entry_release(struct kref *ref);
int batadv_tt_global_hash_count(struct batadv_priv *bat_priv,
                                const u8 *addr, unsigned short vid);
struct batadv_orig_node *batadv_transtable_search(struct batadv_priv *bat_priv,
                                                  const u8 *src, const u8 *addr,
                                                  unsigned short vid);
void batadv_tt_free(struct batadv_priv *bat_priv);
bool batadv_is_my_client(struct batadv_priv *bat_priv, const u8 *addr,
                         unsigned short vid);
bool batadv_is_ap_isolated(struct batadv_priv *bat_priv, u8 *src, u8 *dst,
                           unsigned short vid);
void batadv_tt_local_commit_changes(struct batadv_priv *bat_priv);
bool batadv_tt_global_client_is_roaming(struct batadv_priv *bat_priv,
                                        u8 *addr, unsigned short vid);
bool batadv_tt_local_client_is_roaming(struct batadv_priv *bat_priv,
                                       u8 *addr, unsigned short vid);
void batadv_tt_local_resize_to_mtu(struct net_device *soft_iface);
bool batadv_tt_add_temporary_global_entry(struct batadv_priv *bat_priv,
                                          struct batadv_orig_node *orig_node,
                                          const unsigned char *addr,
                                          unsigned short vid);
bool batadv_tt_global_is_isolated(struct batadv_priv *bat_priv,
                                  const u8 *addr, unsigned short vid);

int batadv_tt_cache_init(void);
void batadv_tt_cache_destroy(void);

/**
 * batadv_tt_global_entry_put() - decrement the tt_global_entry refcounter and
 *  possibly release it
 * @tt_global_entry: tt_global_entry to be free'd
 */
static inline void
batadv_tt_global_entry_put(struct batadv_tt_global_entry *tt_global_entry)
{
        if (!tt_global_entry)
                return;

        kref_put(&tt_global_entry->common.refcount,
                 batadv_tt_global_entry_release);
}

#endif /* _NET_BATMAN_ADV_TRANSLATION_TABLE_H_ */
























































































































































































































































































































































































































































































































































































































































































  134 




  134 
  136 
  136 
  124 
  135 
  124 
  136 
  136 

  136 


























   11 























   10 


















   21 
   10 

   11 
























  205 





  205 


  104 





   24 
   24 





    3 

    3 





   21 
















  136 





















   52 


   22 












   23 



   23 

   23 
   23 


   23 
   23 









   34 







   17 





   44 







   23 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
/*
 * Copyright (c) 2015, Mellanox Technologies inc.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "core_priv.h"

#include <linux/in.h>
#include <linux/in6.h>

/* For in6_dev_get/in6_dev_put */
#include <net/addrconf.h>
#include <net/bonding.h>

#include <rdma/ib_cache.h>
#include <rdma/ib_addr.h>

static struct workqueue_struct *gid_cache_wq;

enum gid_op_type {
        GID_DEL = 0,
        GID_ADD
};

struct update_gid_event_work {
        struct work_struct work;
        union ib_gid       gid;
        struct ib_gid_attr gid_attr;
        enum gid_op_type gid_op;
};

#define ROCE_NETDEV_CALLBACK_SZ                3
struct netdev_event_work_cmd {
        roce_netdev_callback        cb;
        roce_netdev_filter        filter;
        struct net_device        *ndev;
        struct net_device        *filter_ndev;
};

struct netdev_event_work {
        struct work_struct                work;
        struct netdev_event_work_cmd        cmds[ROCE_NETDEV_CALLBACK_SZ];
};

static const struct {
        bool (*is_supported)(const struct ib_device *device, u32 port_num);
        enum ib_gid_type gid_type;
} PORT_CAP_TO_GID_TYPE[] = {
        {rdma_protocol_roce_eth_encap, IB_GID_TYPE_ROCE},
        {rdma_protocol_roce_udp_encap, IB_GID_TYPE_ROCE_UDP_ENCAP},
};

#define CAP_TO_GID_TABLE_SIZE        ARRAY_SIZE(PORT_CAP_TO_GID_TYPE)

unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port)
{
        int i;
        unsigned int ret_flags = 0;

        if (!rdma_protocol_roce(ib_dev, port))
                return 1UL << IB_GID_TYPE_IB;

        for (i = 0; i < CAP_TO_GID_TABLE_SIZE; i++)
                if (PORT_CAP_TO_GID_TYPE[i].is_supported(ib_dev, port))
                        ret_flags |= 1UL << PORT_CAP_TO_GID_TYPE[i].gid_type;

        return ret_flags;
}
EXPORT_SYMBOL(roce_gid_type_mask_support);

static void update_gid(enum gid_op_type gid_op, struct ib_device *ib_dev,
                       u32 port, union ib_gid *gid,
                       struct ib_gid_attr *gid_attr)
{
        int i;
        unsigned long gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        for (i = 0; i < IB_GID_TYPE_SIZE; i++) {
                if ((1UL << i) & gid_type_mask) {
                        gid_attr->gid_type = i;
                        switch (gid_op) {
                        case GID_ADD:
                                ib_cache_gid_add(ib_dev, port,
                                                 gid, gid_attr);
                                break;
                        case GID_DEL:
                                ib_cache_gid_del(ib_dev, port,
                                                 gid, gid_attr);
                                break;
                        }
                }
        }
}

enum bonding_slave_state {
        BONDING_SLAVE_STATE_ACTIVE        = 1UL << 0,
        BONDING_SLAVE_STATE_INACTIVE        = 1UL << 1,
        /* No primary slave or the device isn't a slave in bonding */
        BONDING_SLAVE_STATE_NA                = 1UL << 2,
};

static enum bonding_slave_state is_eth_active_slave_of_bonding_rcu(struct net_device *dev,
                                                                   struct net_device *upper)
{
        if (upper && netif_is_bond_master(upper)) {
                struct net_device *pdev =
                        bond_option_active_slave_get_rcu(netdev_priv(upper));

                if (pdev)
                        return dev == pdev ? BONDING_SLAVE_STATE_ACTIVE :
                                BONDING_SLAVE_STATE_INACTIVE;
        }

        return BONDING_SLAVE_STATE_NA;
}

#define REQUIRED_BOND_STATES                (BONDING_SLAVE_STATE_ACTIVE |        \
                                         BONDING_SLAVE_STATE_NA)
static bool
is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *real_dev;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        real_dev = rdma_vlan_dev_real_dev(cookie);
        if (!real_dev)
                real_dev = cookie;

        res = ((rdma_is_upper_dev_rcu(rdma_ndev, cookie) &&
               (is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev) &
                REQUIRED_BOND_STATES)) ||
               real_dev == rdma_ndev);

        rcu_read_unlock();
        return res;
}

static bool
is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u32 port,
                                  struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *master_dev;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        master_dev = netdev_master_upper_dev_get_rcu(rdma_ndev);
        res = is_eth_active_slave_of_bonding_rcu(rdma_ndev, master_dev) ==
                BONDING_SLAVE_STATE_INACTIVE;
        rcu_read_unlock();

        return res;
}

/**
 * is_ndev_for_default_gid_filter - Check if a given netdevice
 * can be considered for default GIDs or not.
 * @ib_dev:                IB device to check
 * @port:                Port to consider for adding default GID
 * @rdma_ndev:                rdma netdevice pointer
 * @cookie:             Netdevice to consider to form a default GID
 *
 * is_ndev_for_default_gid_filter() returns true if a given netdevice can be
 * considered for deriving default RoCE GID, returns false otherwise.
 */
static bool
is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u32 port,
                               struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        bool res;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();

        /*
         * When rdma netdevice is used in bonding, bonding master netdevice
         * should be considered for default GIDs. Therefore, ignore slave rdma
         * netdevices when bonding is considered.
         * Additionally when event(cookie) netdevice is bond master device,
         * make sure that it the upper netdevice of rdma netdevice.
         */
        res = ((cookie_ndev == rdma_ndev && !netif_is_bond_slave(rdma_ndev)) ||
               (netif_is_bond_master(cookie_ndev) &&
                rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev)));

        rcu_read_unlock();
        return res;
}

static bool pass_all_filter(struct ib_device *ib_dev, u32 port,
                            struct net_device *rdma_ndev, void *cookie)
{
        return true;
}

static bool upper_device_filter(struct ib_device *ib_dev, u32 port,
                                struct net_device *rdma_ndev, void *cookie)
{
        bool res;

        if (!rdma_ndev)
                return false;

        if (rdma_ndev == cookie)
                return true;

        rcu_read_lock();
        res = rdma_is_upper_dev_rcu(rdma_ndev, cookie);
        rcu_read_unlock();

        return res;
}

/**
 * is_upper_ndev_bond_master_filter - Check if a given netdevice
 * is bond master device of netdevice of the RDMA device of port.
 * @ib_dev:                IB device to check
 * @port:                Port to consider for adding default GID
 * @rdma_ndev:                Pointer to rdma netdevice
 * @cookie:                Netdevice to consider to form a default GID
 *
 * is_upper_ndev_bond_master_filter() returns true if a cookie_netdev
 * is bond master device and rdma_ndev is its lower netdevice. It might
 * not have been established as slave device yet.
 */
static bool
is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev,
                                 void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        bool match = false;

        if (!rdma_ndev)
                return false;

        rcu_read_lock();
        if (netif_is_bond_master(cookie_ndev) &&
            rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev))
                match = true;
        rcu_read_unlock();
        return match;
}

static void update_gid_ip(enum gid_op_type gid_op,
                          struct ib_device *ib_dev,
                          u32 port, struct net_device *ndev,
                          struct sockaddr *addr)
{
        union ib_gid gid;
        struct ib_gid_attr gid_attr;

        rdma_ip2gid(addr, &gid);
        memset(&gid_attr, 0, sizeof(gid_attr));
        gid_attr.ndev = ndev;

        update_gid(gid_op, ib_dev, port, &gid, &gid_attr);
}

static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,
                                            u32 port,
                                            struct net_device *rdma_ndev,
                                            struct net_device *event_ndev)
{
        struct net_device *real_dev = rdma_vlan_dev_real_dev(event_ndev);
        unsigned long gid_type_mask;

        if (!rdma_ndev)
                return;

        if (!real_dev)
                real_dev = event_ndev;

        rcu_read_lock();

        if (((rdma_ndev != event_ndev &&
              !rdma_is_upper_dev_rcu(rdma_ndev, event_ndev)) ||
             is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev)
                                                 ==
             BONDING_SLAVE_STATE_INACTIVE)) {
                rcu_read_unlock();
                return;
        }

        rcu_read_unlock();

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev,
                                     gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_DELETE);
}

static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,
                                 u32 port, struct net_device *ndev)
{
        const struct in_ifaddr *ifa;
        struct in_device *in_dev;
        struct sin_list {
                struct list_head        list;
                struct sockaddr_in        ip;
        };
        struct sin_list *sin_iter;
        struct sin_list *sin_temp;

        LIST_HEAD(sin_list);
        if (ndev->reg_state >= NETREG_UNREGISTERING)
                return;

        rcu_read_lock();
        in_dev = __in_dev_get_rcu(ndev);
        if (!in_dev) {
                rcu_read_unlock();
                return;
        }

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                struct sin_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);

                if (!entry)
                        continue;

                entry->ip.sin_family = AF_INET;
                entry->ip.sin_addr.s_addr = ifa->ifa_address;
                list_add_tail(&entry->list, &sin_list);
        }

        rcu_read_unlock();

        list_for_each_entry_safe(sin_iter, sin_temp, &sin_list, list) {
                update_gid_ip(GID_ADD, ib_dev, port, ndev,
                              (struct sockaddr *)&sin_iter->ip);
                list_del(&sin_iter->list);
                kfree(sin_iter);
        }
}

static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,
                                 u32 port, struct net_device *ndev)
{
        struct inet6_ifaddr *ifp;
        struct inet6_dev *in6_dev;
        struct sin6_list {
                struct list_head        list;
                struct sockaddr_in6        sin6;
        };
        struct sin6_list *sin6_iter;
        struct sin6_list *sin6_temp;
        struct ib_gid_attr gid_attr = {.ndev = ndev};
        LIST_HEAD(sin6_list);

        if (ndev->reg_state >= NETREG_UNREGISTERING)
                return;

        in6_dev = in6_dev_get(ndev);
        if (!in6_dev)
                return;

        read_lock_bh(&in6_dev->lock);
        list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
                struct sin6_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);

                if (!entry)
                        continue;

                entry->sin6.sin6_family = AF_INET6;
                entry->sin6.sin6_addr = ifp->addr;
                list_add_tail(&entry->list, &sin6_list);
        }
        read_unlock_bh(&in6_dev->lock);

        in6_dev_put(in6_dev);

        list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_list, list) {
                union ib_gid        gid;

                rdma_ip2gid((struct sockaddr *)&sin6_iter->sin6, &gid);
                update_gid(GID_ADD, ib_dev, port, &gid, &gid_attr);
                list_del(&sin6_iter->list);
                kfree(sin6_iter);
        }
}

static void _add_netdev_ips(struct ib_device *ib_dev, u32 port,
                            struct net_device *ndev)
{
        enum_netdev_ipv4_ips(ib_dev, port, ndev);
        if (IS_ENABLED(CONFIG_IPV6))
                enum_netdev_ipv6_ips(ib_dev, port, ndev);
}

static void add_netdev_ips(struct ib_device *ib_dev, u32 port,
                           struct net_device *rdma_ndev, void *cookie)
{
        _add_netdev_ips(ib_dev, port, cookie);
}

static void del_netdev_ips(struct ib_device *ib_dev, u32 port,
                           struct net_device *rdma_ndev, void *cookie)
{
        ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);
}

/**
 * del_default_gids - Delete default GIDs of the event/cookie netdevice
 * @ib_dev:        RDMA device pointer
 * @port:        Port of the RDMA device whose GID table to consider
 * @rdma_ndev:        Unused rdma netdevice
 * @cookie:        Pointer to event netdevice
 *
 * del_default_gids() deletes the default GIDs of the event/cookie netdevice.
 */
static void del_default_gids(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *cookie_ndev = cookie;
        unsigned long gid_type_mask;

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);

        ib_cache_gid_set_default_gid(ib_dev, port, cookie_ndev, gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_DELETE);
}

static void add_default_gids(struct ib_device *ib_dev, u32 port,
                             struct net_device *rdma_ndev, void *cookie)
{
        struct net_device *event_ndev = cookie;
        unsigned long gid_type_mask;

        gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
        ib_cache_gid_set_default_gid(ib_dev, port, event_ndev, gid_type_mask,
                                     IB_CACHE_GID_DEFAULT_MODE_SET);
}

static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,
                                    u32 port,
                                    struct net_device *rdma_ndev,
                                    void *cookie)
{
        struct net *net;
        struct net_device *ndev;

        /* Lock the rtnl to make sure the netdevs does not move under
         * our feet
         */
        rtnl_lock();
        down_read(&net_rwsem);
        for_each_net(net)
                for_each_netdev(net, ndev) {
                        /*
                         * Filter and add default GIDs of the primary netdevice
                         * when not in bonding mode, or add default GIDs
                         * of bond master device, when in bonding mode.
                         */
                        if (is_ndev_for_default_gid_filter(ib_dev, port,
                                                           rdma_ndev, ndev))
                                add_default_gids(ib_dev, port, rdma_ndev, ndev);

                        if (is_eth_port_of_netdev_filter(ib_dev, port,
                                                         rdma_ndev, ndev))
                                _add_netdev_ips(ib_dev, port, ndev);
                }
        up_read(&net_rwsem);
        rtnl_unlock();
}

/**
 * rdma_roce_rescan_device - Rescan all of the network devices in the system
 * and add their gids, as needed, to the relevant RoCE devices.
 *
 * @ib_dev:         the rdma device
 */
void rdma_roce_rescan_device(struct ib_device *ib_dev)
{
        ib_enum_roce_netdev(ib_dev, pass_all_filter, NULL,
                            enum_all_gids_of_dev_cb, NULL);
}
EXPORT_SYMBOL(rdma_roce_rescan_device);

/**
 * rdma_roce_rescan_port - Rescan all of the network devices in the system
 * and add their gids if relevant to the port of the RoCE device.
 *
 * @ib_dev: IB device
 * @port: Port number
 */
void rdma_roce_rescan_port(struct ib_device *ib_dev, u32 port)
{
        struct net_device *ndev = NULL;

        if (rdma_protocol_roce(ib_dev, port)) {
                ndev = ib_device_get_netdev(ib_dev, port);
                if (!ndev)
                        return;
                enum_all_gids_of_dev_cb(ib_dev, port, ndev, ndev);
                dev_put(ndev);
        }
}
EXPORT_SYMBOL(rdma_roce_rescan_port);

static void callback_for_addr_gid_device_scan(struct ib_device *device,
                                              u32 port,
                                              struct net_device *rdma_ndev,
                                              void *cookie)
{
        struct update_gid_event_work *parsed = cookie;

        return update_gid(parsed->gid_op, device,
                          port, &parsed->gid,
                          &parsed->gid_attr);
}

struct upper_list {
        struct list_head list;
        struct net_device *upper;
};

static int netdev_upper_walk(struct net_device *upper,
                             struct netdev_nested_priv *priv)
{
        struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
        struct list_head *upper_list = (struct list_head *)priv->data;

        if (!entry)
                return 0;

        list_add_tail(&entry->list, upper_list);
        dev_hold(upper);
        entry->upper = upper;

        return 0;
}

static void handle_netdev_upper(struct ib_device *ib_dev, u32 port,
                                void *cookie,
                                void (*handle_netdev)(struct ib_device *ib_dev,
                                                      u32 port,
                                                      struct net_device *ndev))
{
        struct net_device *ndev = cookie;
        struct netdev_nested_priv priv;
        struct upper_list *upper_iter;
        struct upper_list *upper_temp;
        LIST_HEAD(upper_list);

        priv.data = &upper_list;
        rcu_read_lock();
        netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &priv);
        rcu_read_unlock();

        handle_netdev(ib_dev, port, ndev);
        list_for_each_entry_safe(upper_iter, upper_temp, &upper_list,
                                 list) {
                handle_netdev(ib_dev, port, upper_iter->upper);
                dev_put(upper_iter->upper);
                list_del(&upper_iter->list);
                kfree(upper_iter);
        }
}

void roce_del_all_netdev_gids(struct ib_device *ib_dev,
                              u32 port, struct net_device *ndev)
{
        ib_cache_gid_del_all_netdev_gids(ib_dev, port, ndev);
}
EXPORT_SYMBOL(roce_del_all_netdev_gids);

static void del_netdev_upper_ips(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev, void *cookie)
{
        handle_netdev_upper(ib_dev, port, cookie, roce_del_all_netdev_gids);
}

static void add_netdev_upper_ips(struct ib_device *ib_dev, u32 port,
                                 struct net_device *rdma_ndev, void *cookie)
{
        handle_netdev_upper(ib_dev, port, cookie, _add_netdev_ips);
}

static void del_netdev_default_ips_join(struct ib_device *ib_dev, u32 port,
                                        struct net_device *rdma_ndev,
                                        void *cookie)
{
        struct net_device *master_ndev;

        rcu_read_lock();
        master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev);
        dev_hold(master_ndev);
        rcu_read_unlock();

        if (master_ndev) {
                bond_delete_netdev_default_gids(ib_dev, port, rdma_ndev,
                                                master_ndev);
                dev_put(master_ndev);
        }
}

/* The following functions operate on all IB devices. netdevice_event and
 * addr_event execute ib_enum_all_roce_netdevs through a work.
 * ib_enum_all_roce_netdevs iterates through all IB devices.
 */

static void netdevice_event_work_handler(struct work_struct *_work)
{
        struct netdev_event_work *work =
                container_of(_work, struct netdev_event_work, work);
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(work->cmds) && work->cmds[i].cb; i++) {
                ib_enum_all_roce_netdevs(work->cmds[i].filter,
                                         work->cmds[i].filter_ndev,
                                         work->cmds[i].cb,
                                         work->cmds[i].ndev);
                dev_put(work->cmds[i].ndev);
                dev_put(work->cmds[i].filter_ndev);
        }

        kfree(work);
}

static int netdevice_queue_work(struct netdev_event_work_cmd *cmds,
                                struct net_device *ndev)
{
        unsigned int i;
        struct netdev_event_work *ndev_work =
                kmalloc(sizeof(*ndev_work), GFP_KERNEL);

        if (!ndev_work)
                return NOTIFY_DONE;

        memcpy(ndev_work->cmds, cmds, sizeof(ndev_work->cmds));
        for (i = 0; i < ARRAY_SIZE(ndev_work->cmds) && ndev_work->cmds[i].cb; i++) {
                if (!ndev_work->cmds[i].ndev)
                        ndev_work->cmds[i].ndev = ndev;
                if (!ndev_work->cmds[i].filter_ndev)
                        ndev_work->cmds[i].filter_ndev = ndev;
                dev_hold(ndev_work->cmds[i].ndev);
                dev_hold(ndev_work->cmds[i].filter_ndev);
        }
        INIT_WORK(&ndev_work->work, netdevice_event_work_handler);

        queue_work(gid_cache_wq, &ndev_work->work);

        return NOTIFY_DONE;
}

static const struct netdev_event_work_cmd add_cmd = {
        .cb        = add_netdev_ips,
        .filter        = is_eth_port_of_netdev_filter
};

static const struct netdev_event_work_cmd add_cmd_upper_ips = {
        .cb        = add_netdev_upper_ips,
        .filter = is_eth_port_of_netdev_filter
};

static void
ndev_event_unlink(struct netdev_notifier_changeupper_info *changeupper_info,
                  struct netdev_event_work_cmd *cmds)
{
        static const struct netdev_event_work_cmd
                        upper_ips_del_cmd = {
                                .cb        = del_netdev_upper_ips,
                                .filter        = upper_device_filter
        };

        cmds[0] = upper_ips_del_cmd;
        cmds[0].ndev = changeupper_info->upper_dev;
        cmds[1] = add_cmd;
}

static const struct netdev_event_work_cmd bonding_default_add_cmd = {
        .cb        = add_default_gids,
        .filter        = is_upper_ndev_bond_master_filter
};

static void
ndev_event_link(struct net_device *event_ndev,
                struct netdev_notifier_changeupper_info *changeupper_info,
                struct netdev_event_work_cmd *cmds)
{
        static const struct netdev_event_work_cmd
                        bonding_default_del_cmd = {
                                .cb        = del_default_gids,
                                .filter        = is_upper_ndev_bond_master_filter
                        };
        /*
         * When a lower netdev is linked to its upper bonding
         * netdev, delete lower slave netdev's default GIDs.
         */
        cmds[0] = bonding_default_del_cmd;
        cmds[0].ndev = event_ndev;
        cmds[0].filter_ndev = changeupper_info->upper_dev;

        /* Now add bonding upper device default GIDs */
        cmds[1] = bonding_default_add_cmd;
        cmds[1].ndev = changeupper_info->upper_dev;
        cmds[1].filter_ndev = changeupper_info->upper_dev;

        /* Now add bonding upper device IP based GIDs */
        cmds[2] = add_cmd_upper_ips;
        cmds[2].ndev = changeupper_info->upper_dev;
        cmds[2].filter_ndev = changeupper_info->upper_dev;
}

static void netdevice_event_changeupper(struct net_device *event_ndev,
                struct netdev_notifier_changeupper_info *changeupper_info,
                struct netdev_event_work_cmd *cmds)
{
        if (changeupper_info->linking)
                ndev_event_link(event_ndev, changeupper_info, cmds);
        else
                ndev_event_unlink(changeupper_info, cmds);
}

static const struct netdev_event_work_cmd add_default_gid_cmd = {
        .cb        = add_default_gids,
        .filter        = is_ndev_for_default_gid_filter,
};

static int netdevice_event(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        static const struct netdev_event_work_cmd del_cmd = {
                .cb = del_netdev_ips, .filter = pass_all_filter};
        static const struct netdev_event_work_cmd
                        bonding_default_del_cmd_join = {
                                .cb        = del_netdev_default_ips_join,
                                .filter        = is_eth_port_inactive_slave_filter
                        };
        static const struct netdev_event_work_cmd
                        netdev_del_cmd = {
                                .cb        = del_netdev_ips,
                                .filter = is_eth_port_of_netdev_filter
                        };
        static const struct netdev_event_work_cmd bonding_event_ips_del_cmd = {
                .cb = del_netdev_upper_ips, .filter = upper_device_filter};
        struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
        struct netdev_event_work_cmd cmds[ROCE_NETDEV_CALLBACK_SZ] = { {NULL} };

        if (ndev->type != ARPHRD_ETHER)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_REGISTER:
        case NETDEV_UP:
                cmds[0] = bonding_default_del_cmd_join;
                cmds[1] = add_default_gid_cmd;
                cmds[2] = add_cmd;
                break;

        case NETDEV_UNREGISTER:
                if (ndev->reg_state < NETREG_UNREGISTERED)
                        cmds[0] = del_cmd;
                else
                        return NOTIFY_DONE;
                break;

        case NETDEV_CHANGEADDR:
                cmds[0] = netdev_del_cmd;
                if (ndev->reg_state == NETREG_REGISTERED) {
                        cmds[1] = add_default_gid_cmd;
                        cmds[2] = add_cmd;
                }
                break;

        case NETDEV_CHANGEUPPER:
                netdevice_event_changeupper(ndev,
                        container_of(ptr, struct netdev_notifier_changeupper_info, info),
                        cmds);
                break;

        case NETDEV_BONDING_FAILOVER:
                cmds[0] = bonding_event_ips_del_cmd;
                /* Add default GIDs of the bond device */
                cmds[1] = bonding_default_add_cmd;
                /* Add IP based GIDs of the bond device */
                cmds[2] = add_cmd_upper_ips;
                break;

        default:
                return NOTIFY_DONE;
        }

        return netdevice_queue_work(cmds, ndev);
}

static void update_gid_event_work_handler(struct work_struct *_work)
{
        struct update_gid_event_work *work =
                container_of(_work, struct update_gid_event_work, work);

        ib_enum_all_roce_netdevs(is_eth_port_of_netdev_filter,
                                 work->gid_attr.ndev,
                                 callback_for_addr_gid_device_scan, work);

        dev_put(work->gid_attr.ndev);
        kfree(work);
}

static int addr_event(struct notifier_block *this, unsigned long event,
                      struct sockaddr *sa, struct net_device *ndev)
{
        struct update_gid_event_work *work;
        enum gid_op_type gid_op;

        if (ndev->type != ARPHRD_ETHER)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_UP:
                gid_op = GID_ADD;
                break;

        case NETDEV_DOWN:
                gid_op = GID_DEL;
                break;

        default:
                return NOTIFY_DONE;
        }

        work = kmalloc(sizeof(*work), GFP_ATOMIC);
        if (!work)
                return NOTIFY_DONE;

        INIT_WORK(&work->work, update_gid_event_work_handler);

        rdma_ip2gid(sa, &work->gid);
        work->gid_op = gid_op;

        memset(&work->gid_attr, 0, sizeof(work->gid_attr));
        dev_hold(ndev);
        work->gid_attr.ndev   = ndev;

        queue_work(gid_cache_wq, &work->work);

        return NOTIFY_DONE;
}

static int inetaddr_event(struct notifier_block *this, unsigned long event,
                          void *ptr)
{
        struct sockaddr_in        in;
        struct net_device        *ndev;
        struct in_ifaddr        *ifa = ptr;

        in.sin_family = AF_INET;
        in.sin_addr.s_addr = ifa->ifa_address;
        ndev = ifa->ifa_dev->dev;

        return addr_event(this, event, (struct sockaddr *)&in, ndev);
}

static int inet6addr_event(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        struct sockaddr_in6        in6;
        struct net_device        *ndev;
        struct inet6_ifaddr        *ifa6 = ptr;

        in6.sin6_family = AF_INET6;
        in6.sin6_addr = ifa6->addr;
        ndev = ifa6->idev->dev;

        return addr_event(this, event, (struct sockaddr *)&in6, ndev);
}

static struct notifier_block nb_netdevice = {
        .notifier_call = netdevice_event
};

static struct notifier_block nb_inetaddr = {
        .notifier_call = inetaddr_event
};

static struct notifier_block nb_inet6addr = {
        .notifier_call = inet6addr_event
};

int __init roce_gid_mgmt_init(void)
{
        gid_cache_wq = alloc_ordered_workqueue("gid-cache-wq", 0);
        if (!gid_cache_wq)
                return -ENOMEM;

        register_inetaddr_notifier(&nb_inetaddr);
        if (IS_ENABLED(CONFIG_IPV6))
                register_inet6addr_notifier(&nb_inet6addr);
        /* We relay on the netdevice notifier to enumerate all
         * existing devices in the system. Register to this notifier
         * last to make sure we will not miss any IP add/del
         * callbacks.
         */
        register_netdevice_notifier(&nb_netdevice);

        return 0;
}

void __exit roce_gid_mgmt_cleanup(void)
{
        if (IS_ENABLED(CONFIG_IPV6))
                unregister_inet6addr_notifier(&nb_inet6addr);
        unregister_inetaddr_notifier(&nb_inetaddr);
        unregister_netdevice_notifier(&nb_netdevice);
        /* Ensure all gid deletion tasks complete before we go down,
         * to avoid any reference to free'd memory. By the time
         * ib-core is removed, all physical devices have been removed,
         * so no issue with remaining hardware contexts.
         */
        destroy_workqueue(gid_cache_wq);
}








































































  123 

  122 



  121 





  168 







  147 











   66 



   66 





   66 











    4 










    4 





    4 

    4 

    4 
    4 


    4 

    4 


    4 






    4 

    1 

    3 

    3 

    3 
    3 





    4 
    4 


    4 






























    8 
    4 



    8 
    8 


    8 
    8 







    8 



    8 
    8 
    7 




    7 















    4 












    4 
    4 

    4 


    4 


    4 



    4 





    4 

    4 


    4 















































































   77 














   72 




   76 








   73 
   73 


   73 






    6 



    6 


    6 


    6 
    6 












    6 



    5 





    5 
    4 


    5 




    5 
    5 








    4 


    6 
    6 


























































































































































   28 



    2 

   26 
   11 
    5 



   12 







    4 



   11 


   11 








   11 




   11 
   11 


   20 

   28 






    7 
    7 





    5 

    7 












    5 







    5 
    4 






















































   20 





   16 
   16 



   16 




    4 
    4 


    4 


    4 


    4 
   20 





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
/*
 * net/tipc/msg.c: TIPC message header routines
 *
 * Copyright (c) 2000-2006, 2014-2015, Ericsson AB
 * Copyright (c) 2005, 2010-2011, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <net/sock.h>
#include "core.h"
#include "msg.h"
#include "addr.h"
#include "name_table.h"
#include "crypto.h"

#define BUF_ALIGN(x) ALIGN(x, 4)
#define MAX_FORWARD_SIZE 1024
#ifdef CONFIG_TIPC_CRYPTO
#define BUF_HEADROOM ALIGN(((LL_MAX_HEADER + 48) + EHDR_MAX_SIZE), 16)
#define BUF_OVERHEAD (BUF_HEADROOM + TIPC_AES_GCM_TAG_SIZE)
#else
#define BUF_HEADROOM (LL_MAX_HEADER + 48)
#define BUF_OVERHEAD BUF_HEADROOM
#endif

const int one_page_mtu = PAGE_SIZE - SKB_DATA_ALIGN(BUF_OVERHEAD) -
                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

/**
 * tipc_buf_acquire - creates a TIPC message buffer
 * @size: message size (including TIPC header)
 * @gfp: memory allocation flags
 *
 * Return: a new buffer with data pointers set to the specified size.
 *
 * NOTE:
 * Headroom is reserved to allow prepending of a data link header.
 * There may also be unrequested tailroom present at the buffer's end.
 */
struct sk_buff *tipc_buf_acquire(u32 size, gfp_t gfp)
{
        struct sk_buff *skb;

        skb = alloc_skb_fclone(BUF_OVERHEAD + size, gfp);
        if (skb) {
                skb_reserve(skb, BUF_HEADROOM);
                skb_put(skb, size);
                skb->next = NULL;
        }
        return skb;
}

void tipc_msg_init(u32 own_node, struct tipc_msg *m, u32 user, u32 type,
                   u32 hsize, u32 dnode)
{
        memset(m, 0, hsize);
        msg_set_version(m);
        msg_set_user(m, user);
        msg_set_hdr_sz(m, hsize);
        msg_set_size(m, hsize);
        msg_set_prevnode(m, own_node);
        msg_set_type(m, type);
        if (hsize > SHORT_H_SIZE) {
                msg_set_orignode(m, own_node);
                msg_set_destnode(m, dnode);
        }
}

struct sk_buff *tipc_msg_create(uint user, uint type,
                                uint hdr_sz, uint data_sz, u32 dnode,
                                u32 onode, u32 dport, u32 oport, int errcode)
{
        struct tipc_msg *msg;
        struct sk_buff *buf;

        buf = tipc_buf_acquire(hdr_sz + data_sz, GFP_ATOMIC);
        if (unlikely(!buf))
                return NULL;

        msg = buf_msg(buf);
        tipc_msg_init(onode, msg, user, type, hdr_sz, dnode);
        msg_set_size(msg, hdr_sz + data_sz);
        msg_set_origport(msg, oport);
        msg_set_destport(msg, dport);
        msg_set_errcode(msg, errcode);
        return buf;
}

/* tipc_buf_append(): Append a buffer to the fragment list of another buffer
 * @*headbuf: in:  NULL for first frag, otherwise value returned from prev call
 *            out: set when successful non-complete reassembly, otherwise NULL
 * @*buf:     in:  the buffer to append. Always defined
 *            out: head buf after successful complete reassembly, otherwise NULL
 * Returns 1 when reassembly complete, otherwise 0
 */
int tipc_buf_append(struct sk_buff **headbuf, struct sk_buff **buf)
{
        struct sk_buff *head = *headbuf;
        struct sk_buff *frag = *buf;
        struct sk_buff *tail = NULL;
        struct tipc_msg *msg;
        u32 fragid;
        int delta;
        bool headstolen;

        if (!frag)
                goto err;

        msg = buf_msg(frag);
        fragid = msg_type(msg);
        frag->next = NULL;
        skb_pull(frag, msg_hdr_sz(msg));

        if (fragid == FIRST_FRAGMENT) {
                if (unlikely(head))
                        goto err;
                if (skb_has_frag_list(frag) && __skb_linearize(frag))
                        goto err;
                *buf = NULL;
                frag = skb_unshare(frag, GFP_ATOMIC);
                if (unlikely(!frag))
                        goto err;
                head = *headbuf = frag;
                TIPC_SKB_CB(head)->tail = NULL;
                return 0;
        }

        if (!head)
                goto err;

        /* Either the input skb ownership is transferred to headskb
         * or the input skb is freed, clear the reference to avoid
         * bad access on error path.
         */
        *buf = NULL;
        if (skb_try_coalesce(head, frag, &headstolen, &delta)) {
                kfree_skb_partial(frag, headstolen);
        } else {
                tail = TIPC_SKB_CB(head)->tail;
                if (!skb_has_frag_list(head))
                        skb_shinfo(head)->frag_list = frag;
                else
                        tail->next = frag;
                head->truesize += frag->truesize;
                head->data_len += frag->len;
                head->len += frag->len;
                TIPC_SKB_CB(head)->tail = frag;
        }

        if (fragid == LAST_FRAGMENT) {
                TIPC_SKB_CB(head)->validated = 0;
                if (unlikely(!tipc_msg_validate(&head)))
                        goto err;
                *buf = head;
                TIPC_SKB_CB(head)->tail = NULL;
                *headbuf = NULL;
                return 1;
        }
        return 0;
err:
        kfree_skb(*buf);
        kfree_skb(*headbuf);
        *buf = *headbuf = NULL;
        return 0;
}

/**
 * tipc_msg_append(): Append data to tail of an existing buffer queue
 * @_hdr: header to be used
 * @m: the data to be appended
 * @mss: max allowable size of buffer
 * @dlen: size of data to be appended
 * @txq: queue to append to
 *
 * Return: the number of 1k blocks appended or errno value
 */
int tipc_msg_append(struct tipc_msg *_hdr, struct msghdr *m, int dlen,
                    int mss, struct sk_buff_head *txq)
{
        struct sk_buff *skb;
        int accounted, total, curr;
        int mlen, cpy, rem = dlen;
        struct tipc_msg *hdr;

        skb = skb_peek_tail(txq);
        accounted = skb ? msg_blocks(buf_msg(skb)) : 0;
        total = accounted;

        do {
                if (!skb || skb->len >= mss) {
                        skb = tipc_buf_acquire(mss, GFP_KERNEL);
                        if (unlikely(!skb))
                                return -ENOMEM;
                        skb_orphan(skb);
                        skb_trim(skb, MIN_H_SIZE);
                        hdr = buf_msg(skb);
                        skb_copy_to_linear_data(skb, _hdr, MIN_H_SIZE);
                        msg_set_hdr_sz(hdr, MIN_H_SIZE);
                        msg_set_size(hdr, MIN_H_SIZE);
                        __skb_queue_tail(txq, skb);
                        total += 1;
                }
                hdr = buf_msg(skb);
                curr = msg_blocks(hdr);
                mlen = msg_size(hdr);
                cpy = min_t(size_t, rem, mss - mlen);
                if (cpy != copy_from_iter(skb->data + mlen, cpy, &m->msg_iter))
                        return -EFAULT;
                msg_set_size(hdr, mlen + cpy);
                skb_put(skb, cpy);
                rem -= cpy;
                total += msg_blocks(hdr) - curr;
        } while (rem > 0);
        return total - accounted;
}

/* tipc_msg_validate - validate basic format of received message
 *
 * This routine ensures a TIPC message has an acceptable header, and at least
 * as much data as the header indicates it should.  The routine also ensures
 * that the entire message header is stored in the main fragment of the message
 * buffer, to simplify future access to message header fields.
 *
 * Note: Having extra info present in the message header or data areas is OK.
 * TIPC will ignore the excess, under the assumption that it is optional info
 * introduced by a later release of the protocol.
 */
bool tipc_msg_validate(struct sk_buff **_skb)
{
        struct sk_buff *skb = *_skb;
        struct tipc_msg *hdr;
        int msz, hsz;

        /* Ensure that flow control ratio condition is satisfied */
        if (unlikely(skb->truesize / buf_roundup_len(skb) >= 4)) {
                skb = skb_copy_expand(skb, BUF_HEADROOM, 0, GFP_ATOMIC);
                if (!skb)
                        return false;
                kfree_skb(*_skb);
                *_skb = skb;
        }

        if (unlikely(TIPC_SKB_CB(skb)->validated))
                return true;

        if (unlikely(!pskb_may_pull(skb, MIN_H_SIZE)))
                return false;

        hsz = msg_hdr_sz(buf_msg(skb));
        if (unlikely(hsz < MIN_H_SIZE) || (hsz > MAX_H_SIZE))
                return false;
        if (unlikely(!pskb_may_pull(skb, hsz)))
                return false;

        hdr = buf_msg(skb);
        if (unlikely(msg_version(hdr) != TIPC_VERSION))
                return false;

        msz = msg_size(hdr);
        if (unlikely(msz < hsz))
                return false;
        if (unlikely((msz - hsz) > TIPC_MAX_USER_MSG_SIZE))
                return false;
        if (unlikely(skb->len < msz))
                return false;

        TIPC_SKB_CB(skb)->validated = 1;
        return true;
}

/**
 * tipc_msg_fragment - build a fragment skb list for TIPC message
 *
 * @skb: TIPC message skb
 * @hdr: internal msg header to be put on the top of the fragments
 * @pktmax: max size of a fragment incl. the header
 * @frags: returned fragment skb list
 *
 * Return: 0 if the fragmentation is successful, otherwise: -EINVAL
 * or -ENOMEM
 */
int tipc_msg_fragment(struct sk_buff *skb, const struct tipc_msg *hdr,
                      int pktmax, struct sk_buff_head *frags)
{
        int pktno, nof_fragms, dsz, dmax, eat;
        struct tipc_msg *_hdr;
        struct sk_buff *_skb;
        u8 *data;

        /* Non-linear buffer? */
        if (skb_linearize(skb))
                return -ENOMEM;

        data = (u8 *)skb->data;
        dsz = msg_size(buf_msg(skb));
        dmax = pktmax - INT_H_SIZE;
        if (dsz <= dmax || !dmax)
                return -EINVAL;

        nof_fragms = dsz / dmax + 1;
        for (pktno = 1; pktno <= nof_fragms; pktno++) {
                if (pktno < nof_fragms)
                        eat = dmax;
                else
                        eat = dsz % dmax;
                /* Allocate a new fragment */
                _skb = tipc_buf_acquire(INT_H_SIZE + eat, GFP_ATOMIC);
                if (!_skb)
                        goto error;
                skb_orphan(_skb);
                __skb_queue_tail(frags, _skb);
                /* Copy header & data to the fragment */
                skb_copy_to_linear_data(_skb, hdr, INT_H_SIZE);
                skb_copy_to_linear_data_offset(_skb, INT_H_SIZE, data, eat);
                data += eat;
                /* Update the fragment's header */
                _hdr = buf_msg(_skb);
                msg_set_fragm_no(_hdr, pktno);
                msg_set_nof_fragms(_hdr, nof_fragms);
                msg_set_size(_hdr, INT_H_SIZE + eat);
        }
        return 0;

error:
        __skb_queue_purge(frags);
        __skb_queue_head_init(frags);
        return -ENOMEM;
}

/**
 * tipc_msg_build - create buffer chain containing specified header and data
 * @mhdr: Message header, to be prepended to data
 * @m: User message
 * @offset: buffer offset for fragmented messages (FIXME)
 * @dsz: Total length of user data
 * @pktmax: Max packet size that can be used
 * @list: Buffer or chain of buffers to be returned to caller
 *
 * Note that the recursive call we are making here is safe, since it can
 * logically go only one further level down.
 *
 * Return: message data size or errno: -ENOMEM, -EFAULT
 */
int tipc_msg_build(struct tipc_msg *mhdr, struct msghdr *m, int offset,
                   int dsz, int pktmax, struct sk_buff_head *list)
{
        int mhsz = msg_hdr_sz(mhdr);
        struct tipc_msg pkthdr;
        int msz = mhsz + dsz;
        int pktrem = pktmax;
        struct sk_buff *skb;
        int drem = dsz;
        int pktno = 1;
        char *pktpos;
        int pktsz;
        int rc;

        msg_set_size(mhdr, msz);

        /* No fragmentation needed? */
        if (likely(msz <= pktmax)) {
                skb = tipc_buf_acquire(msz, GFP_KERNEL);

                /* Fall back to smaller MTU if node local message */
                if (unlikely(!skb)) {
                        if (pktmax != MAX_MSG_SIZE)
                                return -ENOMEM;
                        rc = tipc_msg_build(mhdr, m, offset, dsz,
                                            one_page_mtu, list);
                        if (rc != dsz)
                                return rc;
                        if (tipc_msg_assemble(list))
                                return dsz;
                        return -ENOMEM;
                }
                skb_orphan(skb);
                __skb_queue_tail(list, skb);
                skb_copy_to_linear_data(skb, mhdr, mhsz);
                pktpos = skb->data + mhsz;
                if (copy_from_iter_full(pktpos, dsz, &m->msg_iter))
                        return dsz;
                rc = -EFAULT;
                goto error;
        }

        /* Prepare reusable fragment header */
        tipc_msg_init(msg_prevnode(mhdr), &pkthdr, MSG_FRAGMENTER,
                      FIRST_FRAGMENT, INT_H_SIZE, msg_destnode(mhdr));
        msg_set_size(&pkthdr, pktmax);
        msg_set_fragm_no(&pkthdr, pktno);
        msg_set_importance(&pkthdr, msg_importance(mhdr));

        /* Prepare first fragment */
        skb = tipc_buf_acquire(pktmax, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;
        skb_orphan(skb);
        __skb_queue_tail(list, skb);
        pktpos = skb->data;
        skb_copy_to_linear_data(skb, &pkthdr, INT_H_SIZE);
        pktpos += INT_H_SIZE;
        pktrem -= INT_H_SIZE;
        skb_copy_to_linear_data_offset(skb, INT_H_SIZE, mhdr, mhsz);
        pktpos += mhsz;
        pktrem -= mhsz;

        do {
                if (drem < pktrem)
                        pktrem = drem;

                if (!copy_from_iter_full(pktpos, pktrem, &m->msg_iter)) {
                        rc = -EFAULT;
                        goto error;
                }
                drem -= pktrem;

                if (!drem)
                        break;

                /* Prepare new fragment: */
                if (drem < (pktmax - INT_H_SIZE))
                        pktsz = drem + INT_H_SIZE;
                else
                        pktsz = pktmax;
                skb = tipc_buf_acquire(pktsz, GFP_KERNEL);
                if (!skb) {
                        rc = -ENOMEM;
                        goto error;
                }
                skb_orphan(skb);
                __skb_queue_tail(list, skb);
                msg_set_type(&pkthdr, FRAGMENT);
                msg_set_size(&pkthdr, pktsz);
                msg_set_fragm_no(&pkthdr, ++pktno);
                skb_copy_to_linear_data(skb, &pkthdr, INT_H_SIZE);
                pktpos = skb->data + INT_H_SIZE;
                pktrem = pktsz - INT_H_SIZE;

        } while (1);
        msg_set_type(buf_msg(skb), LAST_FRAGMENT);
        return dsz;
error:
        __skb_queue_purge(list);
        __skb_queue_head_init(list);
        return rc;
}

/**
 * tipc_msg_bundle - Append contents of a buffer to tail of an existing one
 * @bskb: the bundle buffer to append to
 * @msg: message to be appended
 * @max: max allowable size for the bundle buffer
 *
 * Return: "true" if bundling has been performed, otherwise "false"
 */
static bool tipc_msg_bundle(struct sk_buff *bskb, struct tipc_msg *msg,
                            u32 max)
{
        struct tipc_msg *bmsg = buf_msg(bskb);
        u32 msz, bsz, offset, pad;

        msz = msg_size(msg);
        bsz = msg_size(bmsg);
        offset = BUF_ALIGN(bsz);
        pad = offset - bsz;

        if (unlikely(skb_tailroom(bskb) < (pad + msz)))
                return false;
        if (unlikely(max < (offset + msz)))
                return false;

        skb_put(bskb, pad + msz);
        skb_copy_to_linear_data_offset(bskb, offset, msg, msz);
        msg_set_size(bmsg, offset + msz);
        msg_set_msgcnt(bmsg, msg_msgcnt(bmsg) + 1);
        return true;
}

/**
 * tipc_msg_try_bundle - Try to bundle a new message to the last one
 * @tskb: the last/target message to which the new one will be appended
 * @skb: the new message skb pointer
 * @mss: max message size (header inclusive)
 * @dnode: destination node for the message
 * @new_bundle: if this call made a new bundle or not
 *
 * Return: "true" if the new message skb is potential for bundling this time or
 * later, in the case a bundling has been done this time, the skb is consumed
 * (the skb pointer = NULL).
 * Otherwise, "false" if the skb cannot be bundled at all.
 */
bool tipc_msg_try_bundle(struct sk_buff *tskb, struct sk_buff **skb, u32 mss,
                         u32 dnode, bool *new_bundle)
{
        struct tipc_msg *msg, *inner, *outer;
        u32 tsz;

        /* First, check if the new buffer is suitable for bundling */
        msg = buf_msg(*skb);
        if (msg_user(msg) == MSG_FRAGMENTER)
                return false;
        if (msg_user(msg) == TUNNEL_PROTOCOL)
                return false;
        if (msg_user(msg) == BCAST_PROTOCOL)
                return false;
        if (mss <= INT_H_SIZE + msg_size(msg))
                return false;

        /* Ok, but the last/target buffer can be empty? */
        if (unlikely(!tskb))
                return true;

        /* Is it a bundle already? Try to bundle the new message to it */
        if (msg_user(buf_msg(tskb)) == MSG_BUNDLER) {
                *new_bundle = false;
                goto bundle;
        }

        /* Make a new bundle of the two messages if possible */
        tsz = msg_size(buf_msg(tskb));
        if (unlikely(mss < BUF_ALIGN(INT_H_SIZE + tsz) + msg_size(msg)))
                return true;
        if (unlikely(pskb_expand_head(tskb, INT_H_SIZE, mss - tsz - INT_H_SIZE,
                                      GFP_ATOMIC)))
                return true;
        inner = buf_msg(tskb);
        skb_push(tskb, INT_H_SIZE);
        outer = buf_msg(tskb);
        tipc_msg_init(msg_prevnode(inner), outer, MSG_BUNDLER, 0, INT_H_SIZE,
                      dnode);
        msg_set_importance(outer, msg_importance(inner));
        msg_set_size(outer, INT_H_SIZE + tsz);
        msg_set_msgcnt(outer, 1);
        *new_bundle = true;

bundle:
        if (likely(tipc_msg_bundle(tskb, msg, mss))) {
                consume_skb(*skb);
                *skb = NULL;
        }
        return true;
}

/**
 *  tipc_msg_extract(): extract bundled inner packet from buffer
 *  @skb: buffer to be extracted from.
 *  @iskb: extracted inner buffer, to be returned
 *  @pos: position in outer message of msg to be extracted.
 *  Returns position of next msg.
 *  Consumes outer buffer when last packet extracted
 *  Return: true when there is an extracted buffer, otherwise false
 */
bool tipc_msg_extract(struct sk_buff *skb, struct sk_buff **iskb, int *pos)
{
        struct tipc_msg *hdr, *ihdr;
        int imsz;

        *iskb = NULL;
        if (unlikely(skb_linearize(skb)))
                goto none;

        hdr = buf_msg(skb);
        if (unlikely(*pos > (msg_data_sz(hdr) - MIN_H_SIZE)))
                goto none;

        ihdr = (struct tipc_msg *)(msg_data(hdr) + *pos);
        imsz = msg_size(ihdr);

        if ((*pos + imsz) > msg_data_sz(hdr))
                goto none;

        *iskb = tipc_buf_acquire(imsz, GFP_ATOMIC);
        if (!*iskb)
                goto none;

        skb_copy_to_linear_data(*iskb, ihdr, imsz);
        if (unlikely(!tipc_msg_validate(iskb)))
                goto none;

        *pos += BUF_ALIGN(imsz);
        return true;
none:
        kfree_skb(skb);
        kfree_skb(*iskb);
        *iskb = NULL;
        return false;
}

/**
 * tipc_msg_reverse(): swap source and destination addresses and add error code
 * @own_node: originating node id for reversed message
 * @skb:  buffer containing message to be reversed; will be consumed
 * @err:  error code to be set in message, if any
 * Replaces consumed buffer with new one when successful
 * Return: true if success, otherwise false
 */
bool tipc_msg_reverse(u32 own_node,  struct sk_buff **skb, int err)
{
        struct sk_buff *_skb = *skb;
        struct tipc_msg *_hdr, *hdr;
        int hlen, dlen;

        if (skb_linearize(_skb))
                goto exit;
        _hdr = buf_msg(_skb);
        dlen = min_t(uint, msg_data_sz(_hdr), MAX_FORWARD_SIZE);
        hlen = msg_hdr_sz(_hdr);

        if (msg_dest_droppable(_hdr))
                goto exit;
        if (msg_errcode(_hdr))
                goto exit;

        /* Never return SHORT header */
        if (hlen == SHORT_H_SIZE)
                hlen = BASIC_H_SIZE;

        /* Don't return data along with SYN+, - sender has a clone */
        if (msg_is_syn(_hdr) && err == TIPC_ERR_OVERLOAD)
                dlen = 0;

        /* Allocate new buffer to return */
        *skb = tipc_buf_acquire(hlen + dlen, GFP_ATOMIC);
        if (!*skb)
                goto exit;
        memcpy((*skb)->data, _skb->data, msg_hdr_sz(_hdr));
        memcpy((*skb)->data + hlen, msg_data(_hdr), dlen);

        /* Build reverse header in new buffer */
        hdr = buf_msg(*skb);
        msg_set_hdr_sz(hdr, hlen);
        msg_set_errcode(hdr, err);
        msg_set_non_seq(hdr, 0);
        msg_set_origport(hdr, msg_destport(_hdr));
        msg_set_destport(hdr, msg_origport(_hdr));
        msg_set_destnode(hdr, msg_prevnode(_hdr));
        msg_set_prevnode(hdr, own_node);
        msg_set_orignode(hdr, own_node);
        msg_set_size(hdr, hlen + dlen);
        skb_orphan(_skb);
        kfree_skb(_skb);
        return true;
exit:
        kfree_skb(_skb);
        *skb = NULL;
        return false;
}

bool tipc_msg_skb_clone(struct sk_buff_head *msg, struct sk_buff_head *cpy)
{
        struct sk_buff *skb, *_skb;

        skb_queue_walk(msg, skb) {
                _skb = skb_clone(skb, GFP_ATOMIC);
                if (!_skb) {
                        __skb_queue_purge(cpy);
                        pr_err_ratelimited("Failed to clone buffer chain\n");
                        return false;
                }
                __skb_queue_tail(cpy, _skb);
        }
        return true;
}

/**
 * tipc_msg_lookup_dest(): try to find new destination for named message
 * @net: pointer to associated network namespace
 * @skb: the buffer containing the message.
 * @err: error code to be used by caller if lookup fails
 * Does not consume buffer
 * Return: true if a destination is found, false otherwise
 */
bool tipc_msg_lookup_dest(struct net *net, struct sk_buff *skb, int *err)
{
        struct tipc_msg *msg = buf_msg(skb);
        u32 scope = msg_lookup_scope(msg);
        u32 self = tipc_own_addr(net);
        u32 inst = msg_nameinst(msg);
        struct tipc_socket_addr sk;
        struct tipc_uaddr ua;

        if (!msg_isdata(msg))
                return false;
        if (!msg_named(msg))
                return false;
        if (msg_errcode(msg))
                return false;
        *err = TIPC_ERR_NO_NAME;
        if (skb_linearize(skb))
                return false;
        msg = buf_msg(skb);
        if (msg_reroute_cnt(msg))
                return false;
        tipc_uaddr(&ua, TIPC_SERVICE_RANGE, scope,
                   msg_nametype(msg), inst, inst);
        sk.node = tipc_scope2node(net, scope);
        if (!tipc_nametbl_lookup_anycast(net, &ua, &sk))
                return false;
        msg_incr_reroute_cnt(msg);
        if (sk.node != self)
                msg_set_prevnode(msg, self);
        msg_set_destnode(msg, sk.node);
        msg_set_destport(msg, sk.ref);
        *err = TIPC_OK;

        return true;
}

/* tipc_msg_assemble() - assemble chain of fragments into one message
 */
bool tipc_msg_assemble(struct sk_buff_head *list)
{
        struct sk_buff *skb, *tmp = NULL;

        if (skb_queue_len(list) == 1)
                return true;

        while ((skb = __skb_dequeue(list))) {
                skb->next = NULL;
                if (tipc_buf_append(&tmp, &skb)) {
                        __skb_queue_tail(list, skb);
                        return true;
                }
                if (!tmp)
                        break;
        }
        __skb_queue_purge(list);
        __skb_queue_head_init(list);
        pr_warn("Failed do assemble buffer\n");
        return false;
}

/* tipc_msg_reassemble() - clone a buffer chain of fragments and
 *                         reassemble the clones into one message
 */
bool tipc_msg_reassemble(struct sk_buff_head *list, struct sk_buff_head *rcvq)
{
        struct sk_buff *skb, *_skb;
        struct sk_buff *frag = NULL;
        struct sk_buff *head = NULL;
        int hdr_len;

        /* Copy header if single buffer */
        if (skb_queue_len(list) == 1) {
                skb = skb_peek(list);
                hdr_len = skb_headroom(skb) + msg_hdr_sz(buf_msg(skb));
                _skb = __pskb_copy(skb, hdr_len, GFP_ATOMIC);
                if (!_skb)
                        return false;
                __skb_queue_tail(rcvq, _skb);
                return true;
        }

        /* Clone all fragments and reassemble */
        skb_queue_walk(list, skb) {
                frag = skb_clone(skb, GFP_ATOMIC);
                if (!frag)
                        goto error;
                frag->next = NULL;
                if (tipc_buf_append(&head, &frag))
                        break;
                if (!head)
                        goto error;
        }
        __skb_queue_tail(rcvq, frag);
        return true;
error:
        pr_warn("Failed do clone local mcast rcv buffer\n");
        kfree_skb(head);
        return false;
}

bool tipc_msg_pskb_copy(u32 dst, struct sk_buff_head *msg,
                        struct sk_buff_head *cpy)
{
        struct sk_buff *skb, *_skb;

        skb_queue_walk(msg, skb) {
                _skb = pskb_copy(skb, GFP_ATOMIC);
                if (!_skb) {
                        __skb_queue_purge(cpy);
                        return false;
                }
                msg_set_destnode(buf_msg(_skb), dst);
                __skb_queue_tail(cpy, _skb);
        }
        return true;
}

/* tipc_skb_queue_sorted(); sort pkt into list according to sequence number
 * @list: list to be appended to
 * @seqno: sequence number of buffer to add
 * @skb: buffer to add
 */
bool __tipc_skb_queue_sorted(struct sk_buff_head *list, u16 seqno,
                             struct sk_buff *skb)
{
        struct sk_buff *_skb, *tmp;

        if (skb_queue_empty(list) || less(seqno, buf_seqno(skb_peek(list)))) {
                __skb_queue_head(list, skb);
                return true;
        }

        if (more(seqno, buf_seqno(skb_peek_tail(list)))) {
                __skb_queue_tail(list, skb);
                return true;
        }

        skb_queue_walk_safe(list, _skb, tmp) {
                if (more(seqno, buf_seqno(_skb)))
                        continue;
                if (seqno == buf_seqno(_skb))
                        break;
                __skb_queue_before(list, _skb, skb);
                return true;
        }
        kfree_skb(skb);
        return false;
}

void tipc_skb_reject(struct net *net, int err, struct sk_buff *skb,
                     struct sk_buff_head *xmitq)
{
        if (tipc_msg_reverse(tipc_own_addr(net), &skb, err))
                __skb_queue_tail(xmitq, skb);
}

















































  140 




  100 


















































  704 








    5 
























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM filemap

#if !defined(_TRACE_FILEMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_FILEMAP_H

#include <linux/types.h>
#include <linux/tracepoint.h>
#include <linux/mm.h>
#include <linux/memcontrol.h>
#include <linux/device.h>
#include <linux/kdev_t.h>
#include <linux/errseq.h>

DECLARE_EVENT_CLASS(mm_filemap_op_page_cache,

        TP_PROTO(struct folio *folio),

        TP_ARGS(folio),

        TP_STRUCT__entry(
                __field(unsigned long, pfn)
                __field(unsigned long, i_ino)
                __field(unsigned long, index)
                __field(dev_t, s_dev)
                __field(unsigned char, order)
        ),

        TP_fast_assign(
                __entry->pfn = folio_pfn(folio);
                __entry->i_ino = folio->mapping->host->i_ino;
                __entry->index = folio->index;
                if (folio->mapping->host->i_sb)
                        __entry->s_dev = folio->mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = folio->mapping->host->i_rdev;
                __entry->order = folio_order(folio);
        ),

        TP_printk("dev %d:%d ino %lx pfn=0x%lx ofs=%lu order=%u",
                MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
                __entry->i_ino,
                __entry->pfn,
                __entry->index << PAGE_SHIFT,
                __entry->order)
);

DEFINE_EVENT(mm_filemap_op_page_cache, mm_filemap_delete_from_page_cache,
        TP_PROTO(struct folio *folio),
        TP_ARGS(folio)
        );

DEFINE_EVENT(mm_filemap_op_page_cache, mm_filemap_add_to_page_cache,
        TP_PROTO(struct folio *folio),
        TP_ARGS(folio)
        );

DECLARE_EVENT_CLASS(mm_filemap_op_page_cache_range,

        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),

        TP_ARGS(mapping, index, last_index),

        TP_STRUCT__entry(
                __field(unsigned long, i_ino)
                __field(dev_t, s_dev)
                __field(unsigned long, index)
                __field(unsigned long, last_index)
        ),

        TP_fast_assign(
                __entry->i_ino = mapping->host->i_ino;
                if (mapping->host->i_sb)
                        __entry->s_dev =
                                mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = mapping->host->i_rdev;
                __entry->index = index;
                __entry->last_index = last_index;
        ),

        TP_printk(
                "dev=%d:%d ino=%lx ofs=%lld-%lld",
                MAJOR(__entry->s_dev),
                MINOR(__entry->s_dev), __entry->i_ino,
                ((loff_t)__entry->index) << PAGE_SHIFT,
                ((((loff_t)__entry->last_index + 1) << PAGE_SHIFT) - 1)
        )
);

DEFINE_EVENT(mm_filemap_op_page_cache_range, mm_filemap_get_pages,
        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),
        TP_ARGS(mapping, index, last_index)
);

DEFINE_EVENT(mm_filemap_op_page_cache_range, mm_filemap_map_pages,
        TP_PROTO(
                struct address_space *mapping,
                pgoff_t index,
                pgoff_t last_index
        ),
        TP_ARGS(mapping, index, last_index)
);

TRACE_EVENT(mm_filemap_fault,
        TP_PROTO(struct address_space *mapping, pgoff_t index),

        TP_ARGS(mapping, index),

        TP_STRUCT__entry(
                __field(unsigned long, i_ino)
                __field(dev_t, s_dev)
                __field(unsigned long, index)
        ),

        TP_fast_assign(
                __entry->i_ino = mapping->host->i_ino;
                if (mapping->host->i_sb)
                        __entry->s_dev =
                                mapping->host->i_sb->s_dev;
                else
                        __entry->s_dev = mapping->host->i_rdev;
                __entry->index = index;
        ),

        TP_printk(
                "dev=%d:%d ino=%lx ofs=%lld",
                MAJOR(__entry->s_dev),
                MINOR(__entry->s_dev), __entry->i_ino,
                ((loff_t)__entry->index) << PAGE_SHIFT
        )
);

TRACE_EVENT(filemap_set_wb_err,
                TP_PROTO(struct address_space *mapping, errseq_t eseq),

                TP_ARGS(mapping, eseq),

                TP_STRUCT__entry(
                        __field(unsigned long, i_ino)
                        __field(dev_t, s_dev)
                        __field(errseq_t, errseq)
                ),

                TP_fast_assign(
                        __entry->i_ino = mapping->host->i_ino;
                        __entry->errseq = eseq;
                        if (mapping->host->i_sb)
                                __entry->s_dev = mapping->host->i_sb->s_dev;
                        else
                                __entry->s_dev = mapping->host->i_rdev;
                ),

                TP_printk("dev=%d:%d ino=0x%lx errseq=0x%x",
                        MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
                        __entry->i_ino, __entry->errseq)
);

TRACE_EVENT(file_check_and_advance_wb_err,
                TP_PROTO(struct file *file, errseq_t old),

                TP_ARGS(file, old),

                TP_STRUCT__entry(
                        __field(struct file *, file)
                        __field(unsigned long, i_ino)
                        __field(dev_t, s_dev)
                        __field(errseq_t, old)
                        __field(errseq_t, new)
                ),

                TP_fast_assign(
                        __entry->file = file;
                        __entry->i_ino = file->f_mapping->host->i_ino;
                        if (file->f_mapping->host->i_sb)
                                __entry->s_dev =
                                        file->f_mapping->host->i_sb->s_dev;
                        else
                                __entry->s_dev =
                                        file->f_mapping->host->i_rdev;
                        __entry->old = old;
                        __entry->new = file->f_wb_err;
                ),

                TP_printk("file=%p dev=%d:%d ino=0x%lx old=0x%x new=0x%x",
                        __entry->file, MAJOR(__entry->s_dev),
                        MINOR(__entry->s_dev), __entry->i_ino, __entry->old,
                        __entry->new)
);
#endif /* _TRACE_FILEMAP_H */

/* This part must be outside protection */
#include <trace/define_trace.h>
































































   17 









































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Trace point definitions for the RDMA Connect Manager.
 *
 * Author: Chuck Lever <chuck.lever@oracle.com>
 *
 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
 */

#undef TRACE_SYSTEM
#define TRACE_SYSTEM rdma_cma

#if !defined(_TRACE_RDMA_CMA_H) || defined(TRACE_HEADER_MULTI_READ)

#define _TRACE_RDMA_CMA_H

#include <linux/tracepoint.h>
#include <trace/misc/rdma.h>


DECLARE_EVENT_CLASS(cma_fsm_class,
        TP_PROTO(
                const struct rdma_id_private *id_priv
        ),

        TP_ARGS(id_priv),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, tos)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->tos = id_priv->tos;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos
        )
);

#define DEFINE_CMA_FSM_EVENT(name)                                                \
                DEFINE_EVENT(cma_fsm_class, cm_##name,                                \
                                TP_PROTO(                                        \
                                        const struct rdma_id_private *id_priv        \
                                ),                                                \
                                TP_ARGS(id_priv))

DEFINE_CMA_FSM_EVENT(send_rtu);
DEFINE_CMA_FSM_EVENT(send_rej);
DEFINE_CMA_FSM_EVENT(send_mra);
DEFINE_CMA_FSM_EVENT(send_sidr_req);
DEFINE_CMA_FSM_EVENT(send_sidr_rep);
DEFINE_CMA_FSM_EVENT(disconnect);
DEFINE_CMA_FSM_EVENT(sent_drep);
DEFINE_CMA_FSM_EVENT(sent_dreq);
DEFINE_CMA_FSM_EVENT(id_destroy);

TRACE_EVENT(cm_id_attach,
        TP_PROTO(
                const struct rdma_id_private *id_priv,
                const struct ib_device *device
        ),

        TP_ARGS(id_priv, device),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
                __string(devname, device->name)
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
                __assign_str(devname);
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc device=%s",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr,
                __get_str(devname)
        )
);

DECLARE_EVENT_CLASS(cma_qp_class,
        TP_PROTO(
                const struct rdma_id_private *id_priv
        ),

        TP_ARGS(id_priv),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, tos)
                __field(u32, qp_num)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->tos = id_priv->tos;
                __entry->qp_num = id_priv->qp_num;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u qp_num=%u",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos,
                __entry->qp_num
        )
);

#define DEFINE_CMA_QP_EVENT(name)                                                \
                DEFINE_EVENT(cma_qp_class, cm_##name,                                \
                                TP_PROTO(                                        \
                                        const struct rdma_id_private *id_priv        \
                                ),                                                \
                                TP_ARGS(id_priv))

DEFINE_CMA_QP_EVENT(send_req);
DEFINE_CMA_QP_EVENT(send_rep);
DEFINE_CMA_QP_EVENT(qp_destroy);

/*
 * enum ib_wp_type, from include/rdma/ib_verbs.h
 */
#define IB_QP_TYPE_LIST                                \
        ib_qp_type(SMI)                                \
        ib_qp_type(GSI)                                \
        ib_qp_type(RC)                                \
        ib_qp_type(UC)                                \
        ib_qp_type(UD)                                \
        ib_qp_type(RAW_IPV6)                        \
        ib_qp_type(RAW_ETHERTYPE)                \
        ib_qp_type(RAW_PACKET)                        \
        ib_qp_type(XRC_INI)                        \
        ib_qp_type_end(XRC_TGT)

#undef ib_qp_type
#undef ib_qp_type_end

#define ib_qp_type(x)                TRACE_DEFINE_ENUM(IB_QPT_##x);
#define ib_qp_type_end(x)        TRACE_DEFINE_ENUM(IB_QPT_##x);

IB_QP_TYPE_LIST

#undef ib_qp_type
#undef ib_qp_type_end

#define ib_qp_type(x)                { IB_QPT_##x, #x },
#define ib_qp_type_end(x)        { IB_QPT_##x, #x }

#define rdma_show_qp_type(x) \
                __print_symbolic(x, IB_QP_TYPE_LIST)


TRACE_EVENT(cm_qp_create,
        TP_PROTO(
                const struct rdma_id_private *id_priv,
                const struct ib_pd *pd,
                const struct ib_qp_init_attr *qp_init_attr,
                int rc
        ),

        TP_ARGS(id_priv, pd, qp_init_attr, rc),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, pd_id)
                __field(u32, tos)
                __field(u32, qp_num)
                __field(u32, send_wr)
                __field(u32, recv_wr)
                __field(int, rc)
                __field(unsigned long, qp_type)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->pd_id = pd->res.id;
                __entry->tos = id_priv->tos;
                __entry->send_wr = qp_init_attr->cap.max_send_wr;
                __entry->recv_wr = qp_init_attr->cap.max_recv_wr;
                __entry->rc = rc;
                if (!rc) {
                        __entry->qp_num = id_priv->qp_num;
                        __entry->qp_type = id_priv->id.qp_type;
                } else {
                        __entry->qp_num = 0;
                        __entry->qp_type = 0;
                }
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u pd.id=%u qp_type=%s"
                " send_wr=%u recv_wr=%u qp_num=%u rc=%d",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr,
                __entry->tos, __entry->pd_id,
                rdma_show_qp_type(__entry->qp_type), __entry->send_wr,
                __entry->recv_wr, __entry->qp_num, __entry->rc
        )
);

TRACE_EVENT(cm_req_handler,
        TP_PROTO(
                const struct rdma_id_private *id_priv,
                int event
        ),

        TP_ARGS(id_priv, event),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, tos)
                __field(unsigned long, event)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->tos = id_priv->tos;
                __entry->event = event;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu)",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos,
                rdma_show_ib_cm_event(__entry->event), __entry->event
        )
);

TRACE_EVENT(cm_event_handler,
        TP_PROTO(
                const struct rdma_id_private *id_priv,
                const struct rdma_cm_event *event
        ),

        TP_ARGS(id_priv, event),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, tos)
                __field(unsigned long, event)
                __field(int, status)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->tos = id_priv->tos;
                __entry->event = event->event;
                __entry->status = event->status;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s (%lu/%d)",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos,
                rdma_show_cm_event(__entry->event), __entry->event,
                __entry->status
        )
);

TRACE_EVENT(cm_event_done,
        TP_PROTO(
                const struct rdma_id_private *id_priv,
                const struct rdma_cm_event *event,
                int result
        ),

        TP_ARGS(id_priv, event, result),

        TP_STRUCT__entry(
                __field(u32, cm_id)
                __field(u32, tos)
                __field(unsigned long, event)
                __field(int, result)
                __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
                __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
        ),

        TP_fast_assign(
                __entry->cm_id = id_priv->res.id;
                __entry->tos = id_priv->tos;
                __entry->event = event->event;
                __entry->result = result;
                memcpy(__entry->srcaddr, &id_priv->id.route.addr.src_addr,
                       sizeof(struct sockaddr_in6));
                memcpy(__entry->dstaddr, &id_priv->id.route.addr.dst_addr,
                       sizeof(struct sockaddr_in6));
        ),

        TP_printk("cm.id=%u src=%pISpc dst=%pISpc tos=%u %s consumer returns %d",
                __entry->cm_id, __entry->srcaddr, __entry->dstaddr, __entry->tos,
                rdma_show_cm_event(__entry->event), __entry->result
        )
);

DECLARE_EVENT_CLASS(cma_client_class,
        TP_PROTO(
                const struct ib_device *device
        ),

        TP_ARGS(device),

        TP_STRUCT__entry(
                __string(name, device->name)
        ),

        TP_fast_assign(
                __assign_str(name);
        ),

        TP_printk("device name=%s",
                __get_str(name)
        )
);

#define DEFINE_CMA_CLIENT_EVENT(name)                                                \
                DEFINE_EVENT(cma_client_class, cm_##name,                        \
                                TP_PROTO(                                        \
                                        const struct ib_device *device                \
                                ),                                                \
                                TP_ARGS(device))

DEFINE_CMA_CLIENT_EVENT(add_one);
DEFINE_CMA_CLIENT_EVENT(remove_one);

#endif /* _TRACE_RDMA_CMA_H */

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE cma_trace

#include <trace/define_trace.h>
































    2 



    2 



    2 

































































   70 







































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common code for control of lockd and nfsv4 grace periods.
 *
 * Transplanted from lockd code
 */

#include <linux/module.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <linux/fs.h>
#include <linux/filelock.h>

static unsigned int grace_net_id;
static DEFINE_SPINLOCK(grace_lock);

/**
 * locks_start_grace
 * @net: net namespace that this lock manager belongs to
 * @lm: who this grace period is for
 *
 * A grace period is a period during which locks should not be given
 * out.  Currently grace periods are only enforced by the two lock
 * managers (lockd and nfsd), using the locks_in_grace() function to
 * check when they are in a grace period.
 *
 * This function is called to start a grace period.
 */
void
locks_start_grace(struct net *net, struct lock_manager *lm)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        spin_lock(&grace_lock);
        if (list_empty(&lm->list))
                list_add(&lm->list, grace_list);
        else
                WARN(1, "double list_add attempt detected in net %x %s\n",
                     net->ns.inum, (net == &init_net) ? "(init_net)" : "");
        spin_unlock(&grace_lock);
}
EXPORT_SYMBOL_GPL(locks_start_grace);

/**
 * locks_end_grace
 * @lm: who this grace period is for
 *
 * Call this function to state that the given lock manager is ready to
 * resume regular locking.  The grace period will not end until all lock
 * managers that called locks_start_grace() also call locks_end_grace().
 * Note that callers count on it being safe to call this more than once,
 * and the second call should be a no-op.
 */
void
locks_end_grace(struct lock_manager *lm)
{
        spin_lock(&grace_lock);
        list_del_init(&lm->list);
        spin_unlock(&grace_lock);
}
EXPORT_SYMBOL_GPL(locks_end_grace);

static bool
__state_in_grace(struct net *net, bool open)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);
        struct lock_manager *lm;

        if (!open)
                return !list_empty(grace_list);

        spin_lock(&grace_lock);
        list_for_each_entry(lm, grace_list, list) {
                if (lm->block_opens) {
                        spin_unlock(&grace_lock);
                        return true;
                }
        }
        spin_unlock(&grace_lock);
        return false;
}

/**
 * locks_in_grace
 * @net: network namespace
 *
 * Lock managers call this function to determine when it is OK for them
 * to answer ordinary lock requests, and when they should accept only
 * lock reclaims.
 */
bool locks_in_grace(struct net *net)
{
        return __state_in_grace(net, false);
}
EXPORT_SYMBOL_GPL(locks_in_grace);

bool opens_in_grace(struct net *net)
{
        return __state_in_grace(net, true);
}
EXPORT_SYMBOL_GPL(opens_in_grace);

static int __net_init
grace_init_net(struct net *net)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        INIT_LIST_HEAD(grace_list);
        return 0;
}

static void __net_exit
grace_exit_net(struct net *net)
{
        struct list_head *grace_list = net_generic(net, grace_net_id);

        WARN_ONCE(!list_empty(grace_list),
                  "net %x %s: grace_list is not empty\n",
                  net->ns.inum, __func__);
}

static struct pernet_operations grace_net_ops = {
        .init = grace_init_net,
        .exit = grace_exit_net,
        .id   = &grace_net_id,
        .size = sizeof(struct list_head),
};

static int __init
init_grace(void)
{
        return register_pernet_subsys(&grace_net_ops);
}

static void __exit
exit_grace(void)
{
        unregister_pernet_subsys(&grace_net_ops);
}

MODULE_AUTHOR("Jeff Layton <jlayton@primarydata.com>");
MODULE_DESCRIPTION("NFS client and server infrastructure");
MODULE_LICENSE("GPL");
module_init(init_grace)
module_exit(exit_grace)



















































































































































































































    5 


























































































































































































    1 



    1 




































































































































































    7 



    7 


    7 
































































































   10 


   10 
   10 

    9 
   10 







   10 





    7 










    7 








    7 


    7 

    7 





















    3 









    3 

    3 


    3 

































    3 
    3 

    3 
    3 




    3 

    3 













    3 


    1 






    3 






    2 



    1 


    1 






    2 










   10 
    9 


    1 




    9 








   10 











    7 







    7 








   13 














   11 


   10 


    2 
    2 



    1 









    8 
    6 






    7 
    1 

    2 



    6 


    1 

    1 





    5 







    5 

    5 
















    3 
    1 




    5 
    5 

   13 

































    5 








    1 



    4 

    4 


































    5 






    5 



    4 


    3 










    3 





    5 





    5 





    4 
    2 



    2 

    1 

    3 





    5 




   12 












    4 





    5 


    5 


    2 





    1 





















   12 






   10 





    9 




    9 

    9 




    7 

    9 


   10 



































    4 


    3 



    3 









    4 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015, Sony Mobile Communications Inc.
 * Copyright (c) 2013, The Linux Foundation. All rights reserved.
 */
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/qrtr.h>
#include <linux/termios.h>        /* For TIOCINQ/OUTQ */
#include <linux/spinlock.h>
#include <linux/wait.h>

#include <net/sock.h>

#include "qrtr.h"

#define QRTR_PROTO_VER_1 1
#define QRTR_PROTO_VER_2 3

/* auto-bind range */
#define QRTR_MIN_EPH_SOCKET 0x4000
#define QRTR_MAX_EPH_SOCKET 0x7fff
#define QRTR_EPH_PORT_RANGE \
                XA_LIMIT(QRTR_MIN_EPH_SOCKET, QRTR_MAX_EPH_SOCKET)

#define QRTR_PORT_CTRL_LEGACY 0xffff

/**
 * struct qrtr_hdr_v1 - (I|R)PCrouter packet header version 1
 * @version: protocol version
 * @type: packet type; one of QRTR_TYPE_*
 * @src_node_id: source node
 * @src_port_id: source port
 * @confirm_rx: boolean; whether a resume-tx packet should be send in reply
 * @size: length of packet, excluding this header
 * @dst_node_id: destination node
 * @dst_port_id: destination port
 */
struct qrtr_hdr_v1 {
        __le32 version;
        __le32 type;
        __le32 src_node_id;
        __le32 src_port_id;
        __le32 confirm_rx;
        __le32 size;
        __le32 dst_node_id;
        __le32 dst_port_id;
} __packed;

/**
 * struct qrtr_hdr_v2 - (I|R)PCrouter packet header later versions
 * @version: protocol version
 * @type: packet type; one of QRTR_TYPE_*
 * @flags: bitmask of QRTR_FLAGS_*
 * @optlen: length of optional header data
 * @size: length of packet, excluding this header and optlen
 * @src_node_id: source node
 * @src_port_id: source port
 * @dst_node_id: destination node
 * @dst_port_id: destination port
 */
struct qrtr_hdr_v2 {
        u8 version;
        u8 type;
        u8 flags;
        u8 optlen;
        __le32 size;
        __le16 src_node_id;
        __le16 src_port_id;
        __le16 dst_node_id;
        __le16 dst_port_id;
};

#define QRTR_FLAGS_CONFIRM_RX        BIT(0)

struct qrtr_cb {
        u32 src_node;
        u32 src_port;
        u32 dst_node;
        u32 dst_port;

        u8 type;
        u8 confirm_rx;
};

#define QRTR_HDR_MAX_SIZE max_t(size_t, sizeof(struct qrtr_hdr_v1), \
                                        sizeof(struct qrtr_hdr_v2))

struct qrtr_sock {
        /* WARNING: sk must be the first member */
        struct sock sk;
        struct sockaddr_qrtr us;
        struct sockaddr_qrtr peer;
};

static inline struct qrtr_sock *qrtr_sk(struct sock *sk)
{
        BUILD_BUG_ON(offsetof(struct qrtr_sock, sk) != 0);
        return container_of(sk, struct qrtr_sock, sk);
}

static unsigned int qrtr_local_nid = 1;

/* for node ids */
static RADIX_TREE(qrtr_nodes, GFP_ATOMIC);
static DEFINE_SPINLOCK(qrtr_nodes_lock);
/* broadcast list */
static LIST_HEAD(qrtr_all_nodes);
/* lock for qrtr_all_nodes and node reference */
static DEFINE_MUTEX(qrtr_node_lock);

/* local port allocation management */
static DEFINE_XARRAY_ALLOC(qrtr_ports);

/**
 * struct qrtr_node - endpoint node
 * @ep_lock: lock for endpoint management and callbacks
 * @ep: endpoint
 * @ref: reference count for node
 * @nid: node id
 * @qrtr_tx_flow: tree of qrtr_tx_flow, keyed by node << 32 | port
 * @qrtr_tx_lock: lock for qrtr_tx_flow inserts
 * @rx_queue: receive queue
 * @item: list item for broadcast list
 */
struct qrtr_node {
        struct mutex ep_lock;
        struct qrtr_endpoint *ep;
        struct kref ref;
        unsigned int nid;

        struct radix_tree_root qrtr_tx_flow;
        struct mutex qrtr_tx_lock; /* for qrtr_tx_flow */

        struct sk_buff_head rx_queue;
        struct list_head item;
};

/**
 * struct qrtr_tx_flow - tx flow control
 * @resume_tx: waiters for a resume tx from the remote
 * @pending: number of waiting senders
 * @tx_failed: indicates that a message with confirm_rx flag was lost
 */
struct qrtr_tx_flow {
        struct wait_queue_head resume_tx;
        int pending;
        int tx_failed;
};

#define QRTR_TX_FLOW_HIGH        10
#define QRTR_TX_FLOW_LOW        5

static int qrtr_local_enqueue(struct qrtr_node *node, struct sk_buff *skb,
                              int type, struct sockaddr_qrtr *from,
                              struct sockaddr_qrtr *to);
static int qrtr_bcast_enqueue(struct qrtr_node *node, struct sk_buff *skb,
                              int type, struct sockaddr_qrtr *from,
                              struct sockaddr_qrtr *to);
static struct qrtr_sock *qrtr_port_lookup(int port);
static void qrtr_port_put(struct qrtr_sock *ipc);

/* Release node resources and free the node.
 *
 * Do not call directly, use qrtr_node_release.  To be used with
 * kref_put_mutex.  As such, the node mutex is expected to be locked on call.
 */
static void __qrtr_node_release(struct kref *kref)
{
        struct qrtr_node *node = container_of(kref, struct qrtr_node, ref);
        struct radix_tree_iter iter;
        struct qrtr_tx_flow *flow;
        unsigned long flags;
        void __rcu **slot;

        spin_lock_irqsave(&qrtr_nodes_lock, flags);
        /* If the node is a bridge for other nodes, there are possibly
         * multiple entries pointing to our released node, delete them all.
         */
        radix_tree_for_each_slot(slot, &qrtr_nodes, &iter, 0) {
                if (*slot == node)
                        radix_tree_iter_delete(&qrtr_nodes, &iter, slot);
        }
        spin_unlock_irqrestore(&qrtr_nodes_lock, flags);

        list_del(&node->item);
        mutex_unlock(&qrtr_node_lock);

        skb_queue_purge(&node->rx_queue);

        /* Free tx flow counters */
        radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) {
                flow = *slot;
                radix_tree_iter_delete(&node->qrtr_tx_flow, &iter, slot);
                kfree(flow);
        }
        kfree(node);
}

/* Increment reference to node. */
static struct qrtr_node *qrtr_node_acquire(struct qrtr_node *node)
{
        if (node)
                kref_get(&node->ref);
        return node;
}

/* Decrement reference to node and release as necessary. */
static void qrtr_node_release(struct qrtr_node *node)
{
        if (!node)
                return;
        kref_put_mutex(&node->ref, __qrtr_node_release, &qrtr_node_lock);
}

/**
 * qrtr_tx_resume() - reset flow control counter
 * @node:        qrtr_node that the QRTR_TYPE_RESUME_TX packet arrived on
 * @skb:        resume_tx packet
 */
static void qrtr_tx_resume(struct qrtr_node *node, struct sk_buff *skb)
{
        struct qrtr_ctrl_pkt *pkt = (struct qrtr_ctrl_pkt *)skb->data;
        u64 remote_node = le32_to_cpu(pkt->client.node);
        u32 remote_port = le32_to_cpu(pkt->client.port);
        struct qrtr_tx_flow *flow;
        unsigned long key;

        key = remote_node << 32 | remote_port;

        rcu_read_lock();
        flow = radix_tree_lookup(&node->qrtr_tx_flow, key);
        rcu_read_unlock();
        if (flow) {
                spin_lock(&flow->resume_tx.lock);
                flow->pending = 0;
                spin_unlock(&flow->resume_tx.lock);
                wake_up_interruptible_all(&flow->resume_tx);
        }

        consume_skb(skb);
}

/**
 * qrtr_tx_wait() - flow control for outgoing packets
 * @node:        qrtr_node that the packet is to be send to
 * @dest_node:        node id of the destination
 * @dest_port:        port number of the destination
 * @type:        type of message
 *
 * The flow control scheme is based around the low and high "watermarks". When
 * the low watermark is passed the confirm_rx flag is set on the outgoing
 * message, which will trigger the remote to send a control message of the type
 * QRTR_TYPE_RESUME_TX to reset the counter. If the high watermark is hit
 * further transmision should be paused.
 *
 * Return: 1 if confirm_rx should be set, 0 otherwise or errno failure
 */
static int qrtr_tx_wait(struct qrtr_node *node, int dest_node, int dest_port,
                        int type)
{
        unsigned long key = (u64)dest_node << 32 | dest_port;
        struct qrtr_tx_flow *flow;
        int confirm_rx = 0;
        int ret;

        /* Never set confirm_rx on non-data packets */
        if (type != QRTR_TYPE_DATA)
                return 0;

        mutex_lock(&node->qrtr_tx_lock);
        flow = radix_tree_lookup(&node->qrtr_tx_flow, key);
        if (!flow) {
                flow = kzalloc(sizeof(*flow), GFP_KERNEL);
                if (flow) {
                        init_waitqueue_head(&flow->resume_tx);
                        if (radix_tree_insert(&node->qrtr_tx_flow, key, flow)) {
                                kfree(flow);
                                flow = NULL;
                        }
                }
        }
        mutex_unlock(&node->qrtr_tx_lock);

        /* Set confirm_rx if we where unable to find and allocate a flow */
        if (!flow)
                return 1;

        spin_lock_irq(&flow->resume_tx.lock);
        ret = wait_event_interruptible_locked_irq(flow->resume_tx,
                                                  flow->pending < QRTR_TX_FLOW_HIGH ||
                                                  flow->tx_failed ||
                                                  !node->ep);
        if (ret < 0) {
                confirm_rx = ret;
        } else if (!node->ep) {
                confirm_rx = -EPIPE;
        } else if (flow->tx_failed) {
                flow->tx_failed = 0;
                confirm_rx = 1;
        } else {
                flow->pending++;
                confirm_rx = flow->pending == QRTR_TX_FLOW_LOW;
        }
        spin_unlock_irq(&flow->resume_tx.lock);

        return confirm_rx;
}

/**
 * qrtr_tx_flow_failed() - flag that tx of confirm_rx flagged messages failed
 * @node:        qrtr_node that the packet is to be send to
 * @dest_node:        node id of the destination
 * @dest_port:        port number of the destination
 *
 * Signal that the transmission of a message with confirm_rx flag failed. The
 * flow's "pending" counter will keep incrementing towards QRTR_TX_FLOW_HIGH,
 * at which point transmission would stall forever waiting for the resume TX
 * message associated with the dropped confirm_rx message.
 * Work around this by marking the flow as having a failed transmission and
 * cause the next transmission attempt to be sent with the confirm_rx.
 */
static void qrtr_tx_flow_failed(struct qrtr_node *node, int dest_node,
                                int dest_port)
{
        unsigned long key = (u64)dest_node << 32 | dest_port;
        struct qrtr_tx_flow *flow;

        rcu_read_lock();
        flow = radix_tree_lookup(&node->qrtr_tx_flow, key);
        rcu_read_unlock();
        if (flow) {
                spin_lock_irq(&flow->resume_tx.lock);
                flow->tx_failed = 1;
                spin_unlock_irq(&flow->resume_tx.lock);
        }
}

/* Pass an outgoing packet socket buffer to the endpoint driver. */
static int qrtr_node_enqueue(struct qrtr_node *node, struct sk_buff *skb,
                             int type, struct sockaddr_qrtr *from,
                             struct sockaddr_qrtr *to)
{
        struct qrtr_hdr_v1 *hdr;
        size_t len = skb->len;
        int rc, confirm_rx;

        confirm_rx = qrtr_tx_wait(node, to->sq_node, to->sq_port, type);
        if (confirm_rx < 0) {
                kfree_skb(skb);
                return confirm_rx;
        }

        hdr = skb_push(skb, sizeof(*hdr));
        hdr->version = cpu_to_le32(QRTR_PROTO_VER_1);
        hdr->type = cpu_to_le32(type);
        hdr->src_node_id = cpu_to_le32(from->sq_node);
        hdr->src_port_id = cpu_to_le32(from->sq_port);
        if (to->sq_port == QRTR_PORT_CTRL) {
                hdr->dst_node_id = cpu_to_le32(node->nid);
                hdr->dst_port_id = cpu_to_le32(QRTR_PORT_CTRL);
        } else {
                hdr->dst_node_id = cpu_to_le32(to->sq_node);
                hdr->dst_port_id = cpu_to_le32(to->sq_port);
        }

        hdr->size = cpu_to_le32(len);
        hdr->confirm_rx = !!confirm_rx;

        rc = skb_put_padto(skb, ALIGN(len, 4) + sizeof(*hdr));

        if (!rc) {
                mutex_lock(&node->ep_lock);
                rc = -ENODEV;
                if (node->ep)
                        rc = node->ep->xmit(node->ep, skb);
                else
                        kfree_skb(skb);
                mutex_unlock(&node->ep_lock);
        }
        /* Need to ensure that a subsequent message carries the otherwise lost
         * confirm_rx flag if we dropped this one */
        if (rc && confirm_rx)
                qrtr_tx_flow_failed(node, to->sq_node, to->sq_port);

        return rc;
}

/* Lookup node by id.
 *
 * callers must release with qrtr_node_release()
 */
static struct qrtr_node *qrtr_node_lookup(unsigned int nid)
{
        struct qrtr_node *node;
        unsigned long flags;

        mutex_lock(&qrtr_node_lock);
        spin_lock_irqsave(&qrtr_nodes_lock, flags);
        node = radix_tree_lookup(&qrtr_nodes, nid);
        node = qrtr_node_acquire(node);
        spin_unlock_irqrestore(&qrtr_nodes_lock, flags);
        mutex_unlock(&qrtr_node_lock);

        return node;
}

/* Assign node id to node.
 *
 * This is mostly useful for automatic node id assignment, based on
 * the source id in the incoming packet.
 */
static void qrtr_node_assign(struct qrtr_node *node, unsigned int nid)
{
        unsigned long flags;

        if (nid == QRTR_EP_NID_AUTO)
                return;

        spin_lock_irqsave(&qrtr_nodes_lock, flags);
        radix_tree_insert(&qrtr_nodes, nid, node);
        if (node->nid == QRTR_EP_NID_AUTO)
                node->nid = nid;
        spin_unlock_irqrestore(&qrtr_nodes_lock, flags);
}

/**
 * qrtr_endpoint_post() - post incoming data
 * @ep: endpoint handle
 * @data: data pointer
 * @len: size of data in bytes
 *
 * Return: 0 on success; negative error code on failure
 */
int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len)
{
        struct qrtr_node *node = ep->node;
        const struct qrtr_hdr_v1 *v1;
        const struct qrtr_hdr_v2 *v2;
        struct qrtr_sock *ipc;
        struct sk_buff *skb;
        struct qrtr_cb *cb;
        size_t size;
        unsigned int ver;
        size_t hdrlen;

        if (len == 0 || len & 3)
                return -EINVAL;

        skb = __netdev_alloc_skb(NULL, len, GFP_ATOMIC | __GFP_NOWARN);
        if (!skb)
                return -ENOMEM;

        cb = (struct qrtr_cb *)skb->cb;

        /* Version field in v1 is little endian, so this works for both cases */
        ver = *(u8*)data;

        switch (ver) {
        case QRTR_PROTO_VER_1:
                if (len < sizeof(*v1))
                        goto err;
                v1 = data;
                hdrlen = sizeof(*v1);

                cb->type = le32_to_cpu(v1->type);
                cb->src_node = le32_to_cpu(v1->src_node_id);
                cb->src_port = le32_to_cpu(v1->src_port_id);
                cb->confirm_rx = !!v1->confirm_rx;
                cb->dst_node = le32_to_cpu(v1->dst_node_id);
                cb->dst_port = le32_to_cpu(v1->dst_port_id);

                size = le32_to_cpu(v1->size);
                break;
        case QRTR_PROTO_VER_2:
                if (len < sizeof(*v2))
                        goto err;
                v2 = data;
                hdrlen = sizeof(*v2) + v2->optlen;

                cb->type = v2->type;
                cb->confirm_rx = !!(v2->flags & QRTR_FLAGS_CONFIRM_RX);
                cb->src_node = le16_to_cpu(v2->src_node_id);
                cb->src_port = le16_to_cpu(v2->src_port_id);
                cb->dst_node = le16_to_cpu(v2->dst_node_id);
                cb->dst_port = le16_to_cpu(v2->dst_port_id);

                if (cb->src_port == (u16)QRTR_PORT_CTRL)
                        cb->src_port = QRTR_PORT_CTRL;
                if (cb->dst_port == (u16)QRTR_PORT_CTRL)
                        cb->dst_port = QRTR_PORT_CTRL;

                size = le32_to_cpu(v2->size);
                break;
        default:
                pr_err("qrtr: Invalid version %d\n", ver);
                goto err;
        }

        if (cb->dst_port == QRTR_PORT_CTRL_LEGACY)
                cb->dst_port = QRTR_PORT_CTRL;

        if (!size || len != ALIGN(size, 4) + hdrlen)
                goto err;

        if ((cb->type == QRTR_TYPE_NEW_SERVER ||
             cb->type == QRTR_TYPE_RESUME_TX) &&
            size < sizeof(struct qrtr_ctrl_pkt))
                goto err;

        if (cb->dst_port != QRTR_PORT_CTRL && cb->type != QRTR_TYPE_DATA &&
            cb->type != QRTR_TYPE_RESUME_TX)
                goto err;

        skb_put_data(skb, data + hdrlen, size);

        qrtr_node_assign(node, cb->src_node);

        if (cb->type == QRTR_TYPE_NEW_SERVER) {
                /* Remote node endpoint can bridge other distant nodes */
                const struct qrtr_ctrl_pkt *pkt;

                pkt = data + hdrlen;
                qrtr_node_assign(node, le32_to_cpu(pkt->server.node));
        }

        if (cb->type == QRTR_TYPE_RESUME_TX) {
                qrtr_tx_resume(node, skb);
        } else {
                ipc = qrtr_port_lookup(cb->dst_port);
                if (!ipc)
                        goto err;

                if (sock_queue_rcv_skb(&ipc->sk, skb)) {
                        qrtr_port_put(ipc);
                        goto err;
                }

                qrtr_port_put(ipc);
        }

        return 0;

err:
        kfree_skb(skb);
        return -EINVAL;

}
EXPORT_SYMBOL_GPL(qrtr_endpoint_post);

/**
 * qrtr_alloc_ctrl_packet() - allocate control packet skb
 * @pkt: reference to qrtr_ctrl_pkt pointer
 * @flags: the type of memory to allocate
 *
 * Returns newly allocated sk_buff, or NULL on failure
 *
 * This function allocates a sk_buff large enough to carry a qrtr_ctrl_pkt and
 * on success returns a reference to the control packet in @pkt.
 */
static struct sk_buff *qrtr_alloc_ctrl_packet(struct qrtr_ctrl_pkt **pkt,
                                              gfp_t flags)
{
        const int pkt_len = sizeof(struct qrtr_ctrl_pkt);
        struct sk_buff *skb;

        skb = alloc_skb(QRTR_HDR_MAX_SIZE + pkt_len, flags);
        if (!skb)
                return NULL;

        skb_reserve(skb, QRTR_HDR_MAX_SIZE);
        *pkt = skb_put_zero(skb, pkt_len);

        return skb;
}

/**
 * qrtr_endpoint_register() - register a new endpoint
 * @ep: endpoint to register
 * @nid: desired node id; may be QRTR_EP_NID_AUTO for auto-assignment
 * Return: 0 on success; negative error code on failure
 *
 * The specified endpoint must have the xmit function pointer set on call.
 */
int qrtr_endpoint_register(struct qrtr_endpoint *ep, unsigned int nid)
{
        struct qrtr_node *node;

        if (!ep || !ep->xmit)
                return -EINVAL;

        node = kzalloc(sizeof(*node), GFP_KERNEL);
        if (!node)
                return -ENOMEM;

        kref_init(&node->ref);
        mutex_init(&node->ep_lock);
        skb_queue_head_init(&node->rx_queue);
        node->nid = QRTR_EP_NID_AUTO;
        node->ep = ep;

        INIT_RADIX_TREE(&node->qrtr_tx_flow, GFP_KERNEL);
        mutex_init(&node->qrtr_tx_lock);

        qrtr_node_assign(node, nid);

        mutex_lock(&qrtr_node_lock);
        list_add(&node->item, &qrtr_all_nodes);
        mutex_unlock(&qrtr_node_lock);
        ep->node = node;

        return 0;
}
EXPORT_SYMBOL_GPL(qrtr_endpoint_register);

/**
 * qrtr_endpoint_unregister - unregister endpoint
 * @ep: endpoint to unregister
 */
void qrtr_endpoint_unregister(struct qrtr_endpoint *ep)
{
        struct qrtr_node *node = ep->node;
        struct sockaddr_qrtr src = {AF_QIPCRTR, node->nid, QRTR_PORT_CTRL};
        struct sockaddr_qrtr dst = {AF_QIPCRTR, qrtr_local_nid, QRTR_PORT_CTRL};
        struct radix_tree_iter iter;
        struct qrtr_ctrl_pkt *pkt;
        struct qrtr_tx_flow *flow;
        struct sk_buff *skb;
        unsigned long flags;
        void __rcu **slot;

        mutex_lock(&node->ep_lock);
        node->ep = NULL;
        mutex_unlock(&node->ep_lock);

        /* Notify the local controller about the event */
        spin_lock_irqsave(&qrtr_nodes_lock, flags);
        radix_tree_for_each_slot(slot, &qrtr_nodes, &iter, 0) {
                if (*slot != node)
                        continue;
                src.sq_node = iter.index;
                skb = qrtr_alloc_ctrl_packet(&pkt, GFP_ATOMIC);
                if (skb) {
                        pkt->cmd = cpu_to_le32(QRTR_TYPE_BYE);
                        qrtr_local_enqueue(NULL, skb, QRTR_TYPE_BYE, &src, &dst);
                }
        }
        spin_unlock_irqrestore(&qrtr_nodes_lock, flags);

        /* Wake up any transmitters waiting for resume-tx from the node */
        mutex_lock(&node->qrtr_tx_lock);
        radix_tree_for_each_slot(slot, &node->qrtr_tx_flow, &iter, 0) {
                flow = *slot;
                wake_up_interruptible_all(&flow->resume_tx);
        }
        mutex_unlock(&node->qrtr_tx_lock);

        qrtr_node_release(node);
        ep->node = NULL;
}
EXPORT_SYMBOL_GPL(qrtr_endpoint_unregister);

/* Lookup socket by port.
 *
 * Callers must release with qrtr_port_put()
 */
static struct qrtr_sock *qrtr_port_lookup(int port)
{
        struct qrtr_sock *ipc;

        if (port == QRTR_PORT_CTRL)
                port = 0;

        rcu_read_lock();
        ipc = xa_load(&qrtr_ports, port);
        if (ipc)
                sock_hold(&ipc->sk);
        rcu_read_unlock();

        return ipc;
}

/* Release acquired socket. */
static void qrtr_port_put(struct qrtr_sock *ipc)
{
        sock_put(&ipc->sk);
}

/* Remove port assignment. */
static void qrtr_port_remove(struct qrtr_sock *ipc)
{
        struct qrtr_ctrl_pkt *pkt;
        struct sk_buff *skb;
        int port = ipc->us.sq_port;
        struct sockaddr_qrtr to;

        to.sq_family = AF_QIPCRTR;
        to.sq_node = QRTR_NODE_BCAST;
        to.sq_port = QRTR_PORT_CTRL;

        skb = qrtr_alloc_ctrl_packet(&pkt, GFP_KERNEL);
        if (skb) {
                pkt->cmd = cpu_to_le32(QRTR_TYPE_DEL_CLIENT);
                pkt->client.node = cpu_to_le32(ipc->us.sq_node);
                pkt->client.port = cpu_to_le32(ipc->us.sq_port);

                skb_set_owner_w(skb, &ipc->sk);
                qrtr_bcast_enqueue(NULL, skb, QRTR_TYPE_DEL_CLIENT, &ipc->us,
                                   &to);
        }

        if (port == QRTR_PORT_CTRL)
                port = 0;

        __sock_put(&ipc->sk);

        xa_erase(&qrtr_ports, port);

        /* Ensure that if qrtr_port_lookup() did enter the RCU read section we
         * wait for it to up increment the refcount */
        synchronize_rcu();
}

/* Assign port number to socket.
 *
 * Specify port in the integer pointed to by port, and it will be adjusted
 * on return as necesssary.
 *
 * Port may be:
 *   0: Assign ephemeral port in [QRTR_MIN_EPH_SOCKET, QRTR_MAX_EPH_SOCKET]
 *   <QRTR_MIN_EPH_SOCKET: Specified; requires CAP_NET_ADMIN
 *   >QRTR_MIN_EPH_SOCKET: Specified; available to all
 */
static int qrtr_port_assign(struct qrtr_sock *ipc, int *port)
{
        int rc;

        if (!*port) {
                rc = xa_alloc(&qrtr_ports, port, ipc, QRTR_EPH_PORT_RANGE,
                                GFP_KERNEL);
        } else if (*port < QRTR_MIN_EPH_SOCKET && !capable(CAP_NET_ADMIN)) {
                rc = -EACCES;
        } else if (*port == QRTR_PORT_CTRL) {
                rc = xa_insert(&qrtr_ports, 0, ipc, GFP_KERNEL);
        } else {
                rc = xa_insert(&qrtr_ports, *port, ipc, GFP_KERNEL);
        }

        if (rc == -EBUSY)
                return -EADDRINUSE;
        else if (rc < 0)
                return rc;

        sock_hold(&ipc->sk);

        return 0;
}

/* Reset all non-control ports */
static void qrtr_reset_ports(void)
{
        struct qrtr_sock *ipc;
        unsigned long index;

        rcu_read_lock();
        xa_for_each_start(&qrtr_ports, index, ipc, 1) {
                sock_hold(&ipc->sk);
                ipc->sk.sk_err = ENETRESET;
                sk_error_report(&ipc->sk);
                sock_put(&ipc->sk);
        }
        rcu_read_unlock();
}

/* Bind socket to address.
 *
 * Socket should be locked upon call.
 */
static int __qrtr_bind(struct socket *sock,
                       const struct sockaddr_qrtr *addr, int zapped)
{
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sock *sk = sock->sk;
        int port;
        int rc;

        /* rebinding ok */
        if (!zapped && addr->sq_port == ipc->us.sq_port)
                return 0;

        port = addr->sq_port;
        rc = qrtr_port_assign(ipc, &port);
        if (rc)
                return rc;

        /* unbind previous, if any */
        if (!zapped)
                qrtr_port_remove(ipc);
        ipc->us.sq_port = port;

        sock_reset_flag(sk, SOCK_ZAPPED);

        /* Notify all open ports about the new controller */
        if (port == QRTR_PORT_CTRL)
                qrtr_reset_ports();

        return 0;
}

/* Auto bind to an ephemeral port. */
static int qrtr_autobind(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct sockaddr_qrtr addr;

        if (!sock_flag(sk, SOCK_ZAPPED))
                return 0;

        addr.sq_family = AF_QIPCRTR;
        addr.sq_node = qrtr_local_nid;
        addr.sq_port = 0;

        return __qrtr_bind(sock, &addr, 1);
}

/* Bind socket to specified sockaddr. */
static int qrtr_bind(struct socket *sock, struct sockaddr *saddr, int len)
{
        DECLARE_SOCKADDR(struct sockaddr_qrtr *, addr, saddr);
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sock *sk = sock->sk;
        int rc;

        if (len < sizeof(*addr) || addr->sq_family != AF_QIPCRTR)
                return -EINVAL;

        if (addr->sq_node != ipc->us.sq_node)
                return -EINVAL;

        lock_sock(sk);
        rc = __qrtr_bind(sock, addr, sock_flag(sk, SOCK_ZAPPED));
        release_sock(sk);

        return rc;
}

/* Queue packet to local peer socket. */
static int qrtr_local_enqueue(struct qrtr_node *node, struct sk_buff *skb,
                              int type, struct sockaddr_qrtr *from,
                              struct sockaddr_qrtr *to)
{
        struct qrtr_sock *ipc;
        struct qrtr_cb *cb;

        ipc = qrtr_port_lookup(to->sq_port);
        if (!ipc || &ipc->sk == skb->sk) { /* do not send to self */
                if (ipc)
                        qrtr_port_put(ipc);
                kfree_skb(skb);
                return -ENODEV;
        }

        cb = (struct qrtr_cb *)skb->cb;
        cb->src_node = from->sq_node;
        cb->src_port = from->sq_port;

        if (sock_queue_rcv_skb(&ipc->sk, skb)) {
                qrtr_port_put(ipc);
                kfree_skb(skb);
                return -ENOSPC;
        }

        qrtr_port_put(ipc);

        return 0;
}

/* Queue packet for broadcast. */
static int qrtr_bcast_enqueue(struct qrtr_node *node, struct sk_buff *skb,
                              int type, struct sockaddr_qrtr *from,
                              struct sockaddr_qrtr *to)
{
        struct sk_buff *skbn;

        mutex_lock(&qrtr_node_lock);
        list_for_each_entry(node, &qrtr_all_nodes, item) {
                skbn = pskb_copy(skb, GFP_KERNEL);
                if (!skbn)
                        break;
                skb_set_owner_w(skbn, skb->sk);
                qrtr_node_enqueue(node, skbn, type, from, to);
        }
        mutex_unlock(&qrtr_node_lock);

        qrtr_local_enqueue(NULL, skb, type, from, to);

        return 0;
}

static int qrtr_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
        DECLARE_SOCKADDR(struct sockaddr_qrtr *, addr, msg->msg_name);
        int (*enqueue_fn)(struct qrtr_node *, struct sk_buff *, int,
                          struct sockaddr_qrtr *, struct sockaddr_qrtr *);
        __le32 qrtr_type = cpu_to_le32(QRTR_TYPE_DATA);
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sock *sk = sock->sk;
        struct qrtr_node *node;
        struct sk_buff *skb;
        size_t plen;
        u32 type;
        int rc;

        if (msg->msg_flags & ~(MSG_DONTWAIT))
                return -EINVAL;

        if (len > 65535)
                return -EMSGSIZE;

        lock_sock(sk);

        if (addr) {
                if (msg->msg_namelen < sizeof(*addr)) {
                        release_sock(sk);
                        return -EINVAL;
                }

                if (addr->sq_family != AF_QIPCRTR) {
                        release_sock(sk);
                        return -EINVAL;
                }

                rc = qrtr_autobind(sock);
                if (rc) {
                        release_sock(sk);
                        return rc;
                }
        } else if (sk->sk_state == TCP_ESTABLISHED) {
                addr = &ipc->peer;
        } else {
                release_sock(sk);
                return -ENOTCONN;
        }

        node = NULL;
        if (addr->sq_node == QRTR_NODE_BCAST) {
                if (addr->sq_port != QRTR_PORT_CTRL &&
                    qrtr_local_nid != QRTR_NODE_BCAST) {
                        release_sock(sk);
                        return -ENOTCONN;
                }
                enqueue_fn = qrtr_bcast_enqueue;
        } else if (addr->sq_node == ipc->us.sq_node) {
                enqueue_fn = qrtr_local_enqueue;
        } else {
                node = qrtr_node_lookup(addr->sq_node);
                if (!node) {
                        release_sock(sk);
                        return -ECONNRESET;
                }
                enqueue_fn = qrtr_node_enqueue;
        }

        plen = (len + 3) & ~3;
        skb = sock_alloc_send_skb(sk, plen + QRTR_HDR_MAX_SIZE,
                                  msg->msg_flags & MSG_DONTWAIT, &rc);
        if (!skb) {
                rc = -ENOMEM;
                goto out_node;
        }

        skb_reserve(skb, QRTR_HDR_MAX_SIZE);

        rc = memcpy_from_msg(skb_put(skb, len), msg, len);
        if (rc) {
                kfree_skb(skb);
                goto out_node;
        }

        if (ipc->us.sq_port == QRTR_PORT_CTRL) {
                if (len < 4) {
                        rc = -EINVAL;
                        kfree_skb(skb);
                        goto out_node;
                }

                /* control messages already require the type as 'command' */
                skb_copy_bits(skb, 0, &qrtr_type, 4);
        }

        type = le32_to_cpu(qrtr_type);
        rc = enqueue_fn(node, skb, type, &ipc->us, addr);
        if (rc >= 0)
                rc = len;

out_node:
        qrtr_node_release(node);
        release_sock(sk);

        return rc;
}

static int qrtr_send_resume_tx(struct qrtr_cb *cb)
{
        struct sockaddr_qrtr remote = { AF_QIPCRTR, cb->src_node, cb->src_port };
        struct sockaddr_qrtr local = { AF_QIPCRTR, cb->dst_node, cb->dst_port };
        struct qrtr_ctrl_pkt *pkt;
        struct qrtr_node *node;
        struct sk_buff *skb;
        int ret;

        node = qrtr_node_lookup(remote.sq_node);
        if (!node)
                return -EINVAL;

        skb = qrtr_alloc_ctrl_packet(&pkt, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        pkt->cmd = cpu_to_le32(QRTR_TYPE_RESUME_TX);
        pkt->client.node = cpu_to_le32(cb->dst_node);
        pkt->client.port = cpu_to_le32(cb->dst_port);

        ret = qrtr_node_enqueue(node, skb, QRTR_TYPE_RESUME_TX, &local, &remote);

        qrtr_node_release(node);

        return ret;
}

static int qrtr_recvmsg(struct socket *sock, struct msghdr *msg,
                        size_t size, int flags)
{
        DECLARE_SOCKADDR(struct sockaddr_qrtr *, addr, msg->msg_name);
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        struct qrtr_cb *cb;
        int copied, rc;

        lock_sock(sk);

        if (sock_flag(sk, SOCK_ZAPPED)) {
                release_sock(sk);
                return -EADDRNOTAVAIL;
        }

        skb = skb_recv_datagram(sk, flags, &rc);
        if (!skb) {
                release_sock(sk);
                return rc;
        }
        cb = (struct qrtr_cb *)skb->cb;

        copied = skb->len;
        if (copied > size) {
                copied = size;
                msg->msg_flags |= MSG_TRUNC;
        }

        rc = skb_copy_datagram_msg(skb, 0, msg, copied);
        if (rc < 0)
                goto out;
        rc = copied;

        if (addr) {
                /* There is an anonymous 2-byte hole after sq_family,
                 * make sure to clear it.
                 */
                memset(addr, 0, sizeof(*addr));

                addr->sq_family = AF_QIPCRTR;
                addr->sq_node = cb->src_node;
                addr->sq_port = cb->src_port;
                msg->msg_namelen = sizeof(*addr);
        }

out:
        if (cb->confirm_rx)
                qrtr_send_resume_tx(cb);

        skb_free_datagram(sk, skb);
        release_sock(sk);

        return rc;
}

static int qrtr_connect(struct socket *sock, struct sockaddr *saddr,
                        int len, int flags)
{
        DECLARE_SOCKADDR(struct sockaddr_qrtr *, addr, saddr);
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sock *sk = sock->sk;
        int rc;

        if (len < sizeof(*addr) || addr->sq_family != AF_QIPCRTR)
                return -EINVAL;

        lock_sock(sk);

        sk->sk_state = TCP_CLOSE;
        sock->state = SS_UNCONNECTED;

        rc = qrtr_autobind(sock);
        if (rc) {
                release_sock(sk);
                return rc;
        }

        ipc->peer = *addr;
        sock->state = SS_CONNECTED;
        sk->sk_state = TCP_ESTABLISHED;

        release_sock(sk);

        return 0;
}

static int qrtr_getname(struct socket *sock, struct sockaddr *saddr,
                        int peer)
{
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sockaddr_qrtr qaddr;
        struct sock *sk = sock->sk;

        lock_sock(sk);
        if (peer) {
                if (sk->sk_state != TCP_ESTABLISHED) {
                        release_sock(sk);
                        return -ENOTCONN;
                }

                qaddr = ipc->peer;
        } else {
                qaddr = ipc->us;
        }
        release_sock(sk);

        qaddr.sq_family = AF_QIPCRTR;

        memcpy(saddr, &qaddr, sizeof(qaddr));

        return sizeof(qaddr);
}

static int qrtr_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
{
        void __user *argp = (void __user *)arg;
        struct qrtr_sock *ipc = qrtr_sk(sock->sk);
        struct sock *sk = sock->sk;
        struct sockaddr_qrtr *sq;
        struct sk_buff *skb;
        struct ifreq ifr;
        long len = 0;
        int rc = 0;

        lock_sock(sk);

        switch (cmd) {
        case TIOCOUTQ:
                len = sk->sk_sndbuf - sk_wmem_alloc_get(sk);
                if (len < 0)
                        len = 0;
                rc = put_user(len, (int __user *)argp);
                break;
        case TIOCINQ:
                skb = skb_peek(&sk->sk_receive_queue);
                if (skb)
                        len = skb->len;
                rc = put_user(len, (int __user *)argp);
                break;
        case SIOCGIFADDR:
                if (get_user_ifreq(&ifr, NULL, argp)) {
                        rc = -EFAULT;
                        break;
                }

                sq = (struct sockaddr_qrtr *)&ifr.ifr_addr;
                *sq = ipc->us;
                if (put_user_ifreq(&ifr, argp)) {
                        rc = -EFAULT;
                        break;
                }
                break;
        case SIOCADDRT:
        case SIOCDELRT:
        case SIOCSIFADDR:
        case SIOCGIFDSTADDR:
        case SIOCSIFDSTADDR:
        case SIOCGIFBRDADDR:
        case SIOCSIFBRDADDR:
        case SIOCGIFNETMASK:
        case SIOCSIFNETMASK:
                rc = -EINVAL;
                break;
        default:
                rc = -ENOIOCTLCMD;
                break;
        }

        release_sock(sk);

        return rc;
}

static int qrtr_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct qrtr_sock *ipc;

        if (!sk)
                return 0;

        lock_sock(sk);

        ipc = qrtr_sk(sk);
        sk->sk_shutdown = SHUTDOWN_MASK;
        if (!sock_flag(sk, SOCK_DEAD))
                sk->sk_state_change(sk);

        sock_set_flag(sk, SOCK_DEAD);
        sock_orphan(sk);
        sock->sk = NULL;

        if (!sock_flag(sk, SOCK_ZAPPED))
                qrtr_port_remove(ipc);

        skb_queue_purge(&sk->sk_receive_queue);

        release_sock(sk);
        sock_put(sk);

        return 0;
}

static const struct proto_ops qrtr_proto_ops = {
        .owner                = THIS_MODULE,
        .family                = AF_QIPCRTR,
        .bind                = qrtr_bind,
        .connect        = qrtr_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = sock_no_accept,
        .listen                = sock_no_listen,
        .sendmsg        = qrtr_sendmsg,
        .recvmsg        = qrtr_recvmsg,
        .getname        = qrtr_getname,
        .ioctl                = qrtr_ioctl,
        .gettstamp        = sock_gettstamp,
        .poll                = datagram_poll,
        .shutdown        = sock_no_shutdown,
        .release        = qrtr_release,
        .mmap                = sock_no_mmap,
};

static struct proto qrtr_proto = {
        .name                = "QIPCRTR",
        .owner                = THIS_MODULE,
        .obj_size        = sizeof(struct qrtr_sock),
};

static int qrtr_create(struct net *net, struct socket *sock,
                       int protocol, int kern)
{
        struct qrtr_sock *ipc;
        struct sock *sk;

        if (sock->type != SOCK_DGRAM)
                return -EPROTOTYPE;

        sk = sk_alloc(net, AF_QIPCRTR, GFP_KERNEL, &qrtr_proto, kern);
        if (!sk)
                return -ENOMEM;

        sock_set_flag(sk, SOCK_ZAPPED);

        sock_init_data(sock, sk);
        sock->ops = &qrtr_proto_ops;

        ipc = qrtr_sk(sk);
        ipc->us.sq_family = AF_QIPCRTR;
        ipc->us.sq_node = qrtr_local_nid;
        ipc->us.sq_port = 0;

        return 0;
}

static const struct net_proto_family qrtr_family = {
        .owner        = THIS_MODULE,
        .family        = AF_QIPCRTR,
        .create        = qrtr_create,
};

static int __init qrtr_proto_init(void)
{
        int rc;

        rc = proto_register(&qrtr_proto, 1);
        if (rc)
                return rc;

        rc = sock_register(&qrtr_family);
        if (rc)
                goto err_proto;

        rc = qrtr_ns_init();
        if (rc)
                goto err_sock;

        return 0;

err_sock:
        sock_unregister(qrtr_family.family);
err_proto:
        proto_unregister(&qrtr_proto);
        return rc;
}
postcore_initcall(qrtr_proto_init);

static void __exit qrtr_proto_fini(void)
{
        qrtr_ns_remove();
        sock_unregister(qrtr_family.family);
        proto_unregister(&qrtr_proto);
}
module_exit(qrtr_proto_fini);

MODULE_DESCRIPTION("Qualcomm IPC-router driver");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_NETPROTO(PF_QIPCRTR);


































































































































































































































  200 





























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_address.h>
#include <linux/of_iommu.h>
#include <linux/of_reserved_mem.h>
#include <linux/dma-direct.h> /* for bus_dma_region */
#include <linux/dma-map-ops.h>
#include <linux/init.h>
#include <linux/mod_devicetable.h>
#include <linux/slab.h>
#include <linux/platform_device.h>

#include <asm/errno.h>
#include "of_private.h"

/**
 * of_match_device - Tell if a struct device matches an of_device_id list
 * @matches: array of of device match structures to search in
 * @dev: the of device structure to match against
 *
 * Used by a driver to check whether an platform_device present in the
 * system is in its list of supported devices.
 */
const struct of_device_id *of_match_device(const struct of_device_id *matches,
                                           const struct device *dev)
{
        if (!matches || !dev->of_node || dev->of_node_reused)
                return NULL;
        return of_match_node(matches, dev->of_node);
}
EXPORT_SYMBOL(of_match_device);

static void
of_dma_set_restricted_buffer(struct device *dev, struct device_node *np)
{
        struct device_node *node, *of_node = dev->of_node;
        int count, i;

        if (!IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL))
                return;

        count = of_property_count_elems_of_size(of_node, "memory-region",
                                                sizeof(u32));
        /*
         * If dev->of_node doesn't exist or doesn't contain memory-region, try
         * the OF node having DMA configuration.
         */
        if (count <= 0) {
                of_node = np;
                count = of_property_count_elems_of_size(
                        of_node, "memory-region", sizeof(u32));
        }

        for (i = 0; i < count; i++) {
                node = of_parse_phandle(of_node, "memory-region", i);
                /*
                 * There might be multiple memory regions, but only one
                 * restricted-dma-pool region is allowed.
                 */
                if (of_device_is_compatible(node, "restricted-dma-pool") &&
                    of_device_is_available(node)) {
                        of_node_put(node);
                        break;
                }
                of_node_put(node);
        }

        /*
         * Attempt to initialize a restricted-dma-pool region if one was found.
         * Note that count can hold a negative error code.
         */
        if (i < count && of_reserved_mem_device_init_by_idx(dev, of_node, i))
                dev_warn(dev, "failed to initialise \"restricted-dma-pool\" memory node\n");
}

/**
 * of_dma_configure_id - Setup DMA configuration
 * @dev:        Device to apply DMA configuration
 * @np:                Pointer to OF node having DMA configuration
 * @force_dma:  Whether device is to be set up by of_dma_configure() even if
 *                DMA capability is not explicitly described by firmware.
 * @id:                Optional const pointer value input id
 *
 * Try to get devices's DMA configuration from DT and update it
 * accordingly.
 *
 * If platform code needs to use its own special DMA configuration, it
 * can use a platform bus notifier and handle BUS_NOTIFY_ADD_DEVICE events
 * to fix up DMA configuration.
 */
int of_dma_configure_id(struct device *dev, struct device_node *np,
                        bool force_dma, const u32 *id)
{
        const struct bus_dma_region *map = NULL;
        struct device_node *bus_np;
        u64 mask, end = 0;
        bool coherent, set_map = false;
        int ret;

        if (np == dev->of_node)
                bus_np = __of_get_dma_parent(np);
        else
                bus_np = of_node_get(np);

        ret = of_dma_get_range(bus_np, &map);
        of_node_put(bus_np);
        if (ret < 0) {
                /*
                 * For legacy reasons, we have to assume some devices need
                 * DMA configuration regardless of whether "dma-ranges" is
                 * correctly specified or not.
                 */
                if (!force_dma)
                        return ret == -ENODEV ? 0 : ret;
        } else {
                /* Determine the overall bounds of all DMA regions */
                end = dma_range_map_max(map);
                set_map = true;
        }

        /*
         * If @dev is expected to be DMA-capable then the bus code that created
         * it should have initialised its dma_mask pointer by this point. For
         * now, we'll continue the legacy behaviour of coercing it to the
         * coherent mask if not, but we'll no longer do so quietly.
         */
        if (!dev->dma_mask) {
                dev_warn(dev, "DMA mask not set\n");
                dev->dma_mask = &dev->coherent_dma_mask;
        }

        if (!end && dev->coherent_dma_mask)
                end = dev->coherent_dma_mask;
        else if (!end)
                end = (1ULL << 32) - 1;

        /*
         * Limit coherent and dma mask based on size and default mask
         * set by the driver.
         */
        mask = DMA_BIT_MASK(ilog2(end) + 1);
        dev->coherent_dma_mask &= mask;
        *dev->dma_mask &= mask;
        /* ...but only set bus limit and range map if we found valid dma-ranges earlier */
        if (set_map) {
                dev->bus_dma_limit = end;
                dev->dma_range_map = map;
        }

        coherent = of_dma_is_coherent(np);
        dev_dbg(dev, "device is%sdma coherent\n",
                coherent ? " " : " not ");

        ret = of_iommu_configure(dev, np, id);
        if (ret == -EPROBE_DEFER) {
                /* Don't touch range map if it wasn't set from a valid dma-ranges */
                if (set_map)
                        dev->dma_range_map = NULL;
                kfree(map);
                return -EPROBE_DEFER;
        }
        /* Take all other IOMMU errors to mean we'll just carry on without it */
        dev_dbg(dev, "device is%sbehind an iommu\n",
                !ret ? " " : " not ");

        arch_setup_dma_ops(dev, coherent);

        if (ret)
                of_dma_set_restricted_buffer(dev, np);

        return 0;
}
EXPORT_SYMBOL_GPL(of_dma_configure_id);

const void *of_device_get_match_data(const struct device *dev)
{
        const struct of_device_id *match;

        match = of_match_device(dev->driver->of_match_table, dev);
        if (!match)
                return NULL;

        return match->data;
}
EXPORT_SYMBOL(of_device_get_match_data);

/**
 * of_device_modalias - Fill buffer with newline terminated modalias string
 * @dev:        Calling device
 * @str:        Modalias string
 * @len:        Size of @str
 */
ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len)
{
        ssize_t sl;

        if (!dev || !dev->of_node || dev->of_node_reused)
                return -ENODEV;

        sl = of_modalias(dev->of_node, str, len - 2);
        if (sl < 0)
                return sl;
        if (sl > len - 2)
                return -ENOMEM;

        str[sl++] = '\n';
        str[sl] = 0;
        return sl;
}
EXPORT_SYMBOL_GPL(of_device_modalias);

/**
 * of_device_uevent - Display OF related uevent information
 * @dev:        Device to display the uevent information for
 * @env:        Kernel object's userspace event reference to fill up
 */
void of_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
        const char *compat, *type;
        struct alias_prop *app;
        struct property *p;
        int seen = 0;

        if ((!dev) || (!dev->of_node))
                return;

        add_uevent_var(env, "OF_NAME=%pOFn", dev->of_node);
        add_uevent_var(env, "OF_FULLNAME=%pOF", dev->of_node);
        type = of_node_get_device_type(dev->of_node);
        if (type)
                add_uevent_var(env, "OF_TYPE=%s", type);

        /* Since the compatible field can contain pretty much anything
         * it's not really legal to split it out with commas. We split it
         * up using a number of environment variables instead. */
        of_property_for_each_string(dev->of_node, "compatible", p, compat) {
                add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);
                seen++;
        }
        add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);

        seen = 0;
        mutex_lock(&of_mutex);
        list_for_each_entry(app, &aliases_lookup, link) {
                if (dev->of_node == app->np) {
                        add_uevent_var(env, "OF_ALIAS_%d=%s", seen,
                                       app->alias);
                        seen++;
                }
        }
        mutex_unlock(&of_mutex);
}
EXPORT_SYMBOL_GPL(of_device_uevent);

int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *env)
{
        int sl;

        if ((!dev) || (!dev->of_node) || dev->of_node_reused)
                return -ENODEV;

        /* Devicetree modalias is tricky, we add it in 2 steps */
        if (add_uevent_var(env, "MODALIAS="))
                return -ENOMEM;

        sl = of_modalias(dev->of_node, &env->buf[env->buflen-1],
                         sizeof(env->buf) - env->buflen);
        if (sl < 0)
                return sl;
        if (sl >= (sizeof(env->buf) - env->buflen))
                return -ENOMEM;
        env->buflen += sl;

        return 0;
}
EXPORT_SYMBOL_GPL(of_device_uevent_modalias);

/**
 * of_device_make_bus_id - Use the device node data to assign a unique name
 * @dev: pointer to device structure that is linked to a device tree node
 *
 * This routine will first try using the translated bus address to
 * derive a unique name. If it cannot, then it will prepend names from
 * parent nodes until a unique name can be derived.
 */
void of_device_make_bus_id(struct device *dev)
{
        struct device_node *node = dev->of_node;
        const __be32 *reg;
        u64 addr;
        u32 mask;

        /* Construct the name, using parent nodes if necessary to ensure uniqueness */
        while (node->parent) {
                /*
                 * If the address can be translated, then that is as much
                 * uniqueness as we need. Make it the first component and return
                 */
                reg = of_get_property(node, "reg", NULL);
                if (reg && (addr = of_translate_address(node, reg)) != OF_BAD_ADDR) {
                        if (!of_property_read_u32(node, "mask", &mask))
                                dev_set_name(dev, dev_name(dev) ? "%llx.%x.%pOFn:%s" : "%llx.%x.%pOFn",
                                             addr, ffs(mask) - 1, node, dev_name(dev));

                        else
                                dev_set_name(dev, dev_name(dev) ? "%llx.%pOFn:%s" : "%llx.%pOFn",
                                             addr, node, dev_name(dev));
                        return;
                }

                /* format arguments only used if dev_name() resolves to NULL */
                dev_set_name(dev, dev_name(dev) ? "%s:%s" : "%s",
                             kbasename(node->full_name), dev_name(dev));
                node = node->parent;
        }
}
EXPORT_SYMBOL_GPL(of_device_make_bus_id);































  167 







  169 



  169 
  114 
  169 



  167 


    1 





































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
// SPDX-License-Identifier: GPL-2.0-only
/*
 * fs/kernfs/symlink.c - kernfs symlink implementation
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
 */

#include <linux/fs.h>
#include <linux/gfp.h>
#include <linux/namei.h>

#include "kernfs-internal.h"

/**
 * kernfs_create_link - create a symlink
 * @parent: directory to create the symlink in
 * @name: name of the symlink
 * @target: target node for the symlink to point to
 *
 * Return: the created node on success, ERR_PTR() value on error.
 * Ownership of the link matches ownership of the target.
 */
struct kernfs_node *kernfs_create_link(struct kernfs_node *parent,
                                       const char *name,
                                       struct kernfs_node *target)
{
        struct kernfs_node *kn;
        int error;
        kuid_t uid = GLOBAL_ROOT_UID;
        kgid_t gid = GLOBAL_ROOT_GID;

        if (target->iattr) {
                uid = target->iattr->ia_uid;
                gid = target->iattr->ia_gid;
        }

        kn = kernfs_new_node(parent, name, S_IFLNK|0777, uid, gid, KERNFS_LINK);
        if (!kn)
                return ERR_PTR(-ENOMEM);

        if (kernfs_ns_enabled(parent))
                kn->ns = target->ns;
        kn->symlink.target_kn = target;
        kernfs_get(target);        /* ref owned by symlink */

        error = kernfs_add_one(kn);
        if (!error)
                return kn;

        kernfs_put(kn);
        return ERR_PTR(error);
}

static int kernfs_get_target_path(struct kernfs_node *parent,
                                  struct kernfs_node *target, char *path)
{
        struct kernfs_node *base, *kn;
        char *s = path;
        int len = 0;

        /* go up to the root, stop at the base */
        base = parent;
        while (base->parent) {
                kn = target->parent;
                while (kn->parent && base != kn)
                        kn = kn->parent;

                if (base == kn)
                        break;

                if ((s - path) + 3 >= PATH_MAX)
                        return -ENAMETOOLONG;

                strcpy(s, "../");
                s += 3;
                base = base->parent;
        }

        /* determine end of target string for reverse fillup */
        kn = target;
        while (kn->parent && kn != base) {
                len += strlen(kn->name) + 1;
                kn = kn->parent;
        }

        /* check limits */
        if (len < 2)
                return -EINVAL;
        len--;
        if ((s - path) + len >= PATH_MAX)
                return -ENAMETOOLONG;

        /* reverse fillup of target string from target to base */
        kn = target;
        while (kn->parent && kn != base) {
                int slen = strlen(kn->name);

                len -= slen;
                memcpy(s + len, kn->name, slen);
                if (len)
                        s[--len] = '/';

                kn = kn->parent;
        }

        return 0;
}

static int kernfs_getlink(struct inode *inode, char *path)
{
        struct kernfs_node *kn = inode->i_private;
        struct kernfs_node *parent = kn->parent;
        struct kernfs_node *target = kn->symlink.target_kn;
        struct kernfs_root *root = kernfs_root(parent);
        int error;

        down_read(&root->kernfs_rwsem);
        error = kernfs_get_target_path(parent, target, path);
        up_read(&root->kernfs_rwsem);

        return error;
}

static const char *kernfs_iop_get_link(struct dentry *dentry,
                                       struct inode *inode,
                                       struct delayed_call *done)
{
        char *body;
        int error;

        if (!dentry)
                return ERR_PTR(-ECHILD);
        body = kzalloc(PAGE_SIZE, GFP_KERNEL);
        if (!body)
                return ERR_PTR(-ENOMEM);
        error = kernfs_getlink(inode, body);
        if (unlikely(error < 0)) {
                kfree(body);
                return ERR_PTR(error);
        }
        set_delayed_call(done, kfree_link, body);
        return body;
}

const struct inode_operations kernfs_symlink_iops = {
        .listxattr        = kernfs_iop_listxattr,
        .get_link        = kernfs_iop_get_link,
        .setattr        = kernfs_iop_setattr,
        .getattr        = kernfs_iop_getattr,
        .permission        = kernfs_iop_permission,
};




























































































































































































































































































































































































































































































































    1 





















































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Asynchronous Cryptographic Hash operations.
 *
 * This is the implementation of the ahash (asynchronous hash) API.  It differs
 * from shash (synchronous hash) in that ahash supports asynchronous operations,
 * and it hashes data from scatterlists instead of virtually addressed buffers.
 *
 * The ahash API provides access to both ahash and shash algorithms.  The shash
 * API only provides access to shash algorithms.
 *
 * Copyright (c) 2008 Loc Ho <lho@amcc.com>
 */

#include <crypto/scatterwalk.h>
#include <linux/cryptouser.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <net/netlink.h>

#include "hash.h"

#define CRYPTO_ALG_TYPE_AHASH_MASK        0x0000000e

/*
 * For an ahash tfm that is using an shash algorithm (instead of an ahash
 * algorithm), this returns the underlying shash tfm.
 */
static inline struct crypto_shash *ahash_to_shash(struct crypto_ahash *tfm)
{
        return *(struct crypto_shash **)crypto_ahash_ctx(tfm);
}

static inline struct shash_desc *prepare_shash_desc(struct ahash_request *req,
                                                    struct crypto_ahash *tfm)
{
        struct shash_desc *desc = ahash_request_ctx(req);

        desc->tfm = ahash_to_shash(tfm);
        return desc;
}

int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc)
{
        struct crypto_hash_walk walk;
        int nbytes;

        for (nbytes = crypto_hash_walk_first(req, &walk); nbytes > 0;
             nbytes = crypto_hash_walk_done(&walk, nbytes))
                nbytes = crypto_shash_update(desc, walk.data, nbytes);

        return nbytes;
}
EXPORT_SYMBOL_GPL(shash_ahash_update);

int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc)
{
        struct crypto_hash_walk walk;
        int nbytes;

        nbytes = crypto_hash_walk_first(req, &walk);
        if (!nbytes)
                return crypto_shash_final(desc, req->result);

        do {
                nbytes = crypto_hash_walk_last(&walk) ?
                         crypto_shash_finup(desc, walk.data, nbytes,
                                            req->result) :
                         crypto_shash_update(desc, walk.data, nbytes);
                nbytes = crypto_hash_walk_done(&walk, nbytes);
        } while (nbytes > 0);

        return nbytes;
}
EXPORT_SYMBOL_GPL(shash_ahash_finup);

int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc)
{
        unsigned int nbytes = req->nbytes;
        struct scatterlist *sg;
        unsigned int offset;
        int err;

        if (nbytes &&
            (sg = req->src, offset = sg->offset,
             nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) {
                void *data;

                data = kmap_local_page(sg_page(sg));
                err = crypto_shash_digest(desc, data + offset, nbytes,
                                          req->result);
                kunmap_local(data);
        } else
                err = crypto_shash_init(desc) ?:
                      shash_ahash_finup(req, desc);

        return err;
}
EXPORT_SYMBOL_GPL(shash_ahash_digest);

static void crypto_exit_ahash_using_shash(struct crypto_tfm *tfm)
{
        struct crypto_shash **ctx = crypto_tfm_ctx(tfm);

        crypto_free_shash(*ctx);
}

static int crypto_init_ahash_using_shash(struct crypto_tfm *tfm)
{
        struct crypto_alg *calg = tfm->__crt_alg;
        struct crypto_ahash *crt = __crypto_ahash_cast(tfm);
        struct crypto_shash **ctx = crypto_tfm_ctx(tfm);
        struct crypto_shash *shash;

        if (!crypto_mod_get(calg))
                return -EAGAIN;

        shash = crypto_create_tfm(calg, &crypto_shash_type);
        if (IS_ERR(shash)) {
                crypto_mod_put(calg);
                return PTR_ERR(shash);
        }

        crt->using_shash = true;
        *ctx = shash;
        tfm->exit = crypto_exit_ahash_using_shash;

        crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) &
                                    CRYPTO_TFM_NEED_KEY);
        crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash);

        return 0;
}

static int hash_walk_next(struct crypto_hash_walk *walk)
{
        unsigned int offset = walk->offset;
        unsigned int nbytes = min(walk->entrylen,
                                  ((unsigned int)(PAGE_SIZE)) - offset);

        walk->data = kmap_local_page(walk->pg);
        walk->data += offset;
        walk->entrylen -= nbytes;
        return nbytes;
}

static int hash_walk_new_entry(struct crypto_hash_walk *walk)
{
        struct scatterlist *sg;

        sg = walk->sg;
        walk->offset = sg->offset;
        walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT);
        walk->offset = offset_in_page(walk->offset);
        walk->entrylen = sg->length;

        if (walk->entrylen > walk->total)
                walk->entrylen = walk->total;
        walk->total -= walk->entrylen;

        return hash_walk_next(walk);
}

int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
{
        walk->data -= walk->offset;

        kunmap_local(walk->data);
        crypto_yield(walk->flags);

        if (err)
                return err;

        if (walk->entrylen) {
                walk->offset = 0;
                walk->pg++;
                return hash_walk_next(walk);
        }

        if (!walk->total)
                return 0;

        walk->sg = sg_next(walk->sg);

        return hash_walk_new_entry(walk);
}
EXPORT_SYMBOL_GPL(crypto_hash_walk_done);

int crypto_hash_walk_first(struct ahash_request *req,
                           struct crypto_hash_walk *walk)
{
        walk->total = req->nbytes;

        if (!walk->total) {
                walk->entrylen = 0;
                return 0;
        }

        walk->sg = req->src;
        walk->flags = req->base.flags;

        return hash_walk_new_entry(walk);
}
EXPORT_SYMBOL_GPL(crypto_hash_walk_first);

static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key,
                          unsigned int keylen)
{
        return -ENOSYS;
}

static void ahash_set_needkey(struct crypto_ahash *tfm, struct ahash_alg *alg)
{
        if (alg->setkey != ahash_nosetkey &&
            !(alg->halg.base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
                crypto_ahash_set_flags(tfm, CRYPTO_TFM_NEED_KEY);
}

int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
                        unsigned int keylen)
{
        if (likely(tfm->using_shash)) {
                struct crypto_shash *shash = ahash_to_shash(tfm);
                int err;

                err = crypto_shash_setkey(shash, key, keylen);
                if (unlikely(err)) {
                        crypto_ahash_set_flags(tfm,
                                               crypto_shash_get_flags(shash) &
                                               CRYPTO_TFM_NEED_KEY);
                        return err;
                }
        } else {
                struct ahash_alg *alg = crypto_ahash_alg(tfm);
                int err;

                err = alg->setkey(tfm, key, keylen);
                if (unlikely(err)) {
                        ahash_set_needkey(tfm, alg);
                        return err;
                }
        }
        crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_ahash_setkey);

int crypto_ahash_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_init(prepare_shash_desc(req, tfm));
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;
        return crypto_ahash_alg(tfm)->init(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_init);

static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt,
                          bool has_state)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        unsigned int ds = crypto_ahash_digestsize(tfm);
        struct ahash_request *subreq;
        unsigned int subreq_size;
        unsigned int reqsize;
        u8 *result;
        gfp_t gfp;
        u32 flags;

        subreq_size = sizeof(*subreq);
        reqsize = crypto_ahash_reqsize(tfm);
        reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
        subreq_size += reqsize;
        subreq_size += ds;

        flags = ahash_request_flags(req);
        gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?  GFP_KERNEL : GFP_ATOMIC;
        subreq = kmalloc(subreq_size, gfp);
        if (!subreq)
                return -ENOMEM;

        ahash_request_set_tfm(subreq, tfm);
        ahash_request_set_callback(subreq, flags, cplt, req);

        result = (u8 *)(subreq + 1) + reqsize;

        ahash_request_set_crypt(subreq, req->src, result, req->nbytes);

        if (has_state) {
                void *state;

                state = kmalloc(crypto_ahash_statesize(tfm), gfp);
                if (!state) {
                        kfree(subreq);
                        return -ENOMEM;
                }

                crypto_ahash_export(req, state);
                crypto_ahash_import(subreq, state);
                kfree_sensitive(state);
        }

        req->priv = subreq;

        return 0;
}

static void ahash_restore_req(struct ahash_request *req, int err)
{
        struct ahash_request *subreq = req->priv;

        if (!err)
                memcpy(req->result, subreq->result,
                       crypto_ahash_digestsize(crypto_ahash_reqtfm(req)));

        req->priv = NULL;

        kfree_sensitive(subreq);
}

int crypto_ahash_update(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_update(req, ahash_request_ctx(req));

        return crypto_ahash_alg(tfm)->update(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_update);

int crypto_ahash_final(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_final(ahash_request_ctx(req), req->result);

        return crypto_ahash_alg(tfm)->final(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_final);

int crypto_ahash_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_finup(req, ahash_request_ctx(req));

        return crypto_ahash_alg(tfm)->finup(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_finup);

int crypto_ahash_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return shash_ahash_digest(req, prepare_shash_desc(req, tfm));

        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;

        return crypto_ahash_alg(tfm)->digest(req);
}
EXPORT_SYMBOL_GPL(crypto_ahash_digest);

static void ahash_def_finup_done2(void *data, int err)
{
        struct ahash_request *areq = data;

        if (err == -EINPROGRESS)
                return;

        ahash_restore_req(areq, err);

        ahash_request_complete(areq, err);
}

static int ahash_def_finup_finish1(struct ahash_request *req, int err)
{
        struct ahash_request *subreq = req->priv;

        if (err)
                goto out;

        subreq->base.complete = ahash_def_finup_done2;

        err = crypto_ahash_alg(crypto_ahash_reqtfm(req))->final(subreq);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

out:
        ahash_restore_req(req, err);
        return err;
}

static void ahash_def_finup_done1(void *data, int err)
{
        struct ahash_request *areq = data;
        struct ahash_request *subreq;

        if (err == -EINPROGRESS)
                goto out;

        subreq = areq->priv;
        subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG;

        err = ahash_def_finup_finish1(areq, err);
        if (err == -EINPROGRESS || err == -EBUSY)
                return;

out:
        ahash_request_complete(areq, err);
}

static int ahash_def_finup(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        int err;

        err = ahash_save_req(req, ahash_def_finup_done1, true);
        if (err)
                return err;

        err = crypto_ahash_alg(tfm)->update(req->priv);
        if (err == -EINPROGRESS || err == -EBUSY)
                return err;

        return ahash_def_finup_finish1(req, err);
}

int crypto_ahash_export(struct ahash_request *req, void *out)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_export(ahash_request_ctx(req), out);
        return crypto_ahash_alg(tfm)->export(req, out);
}
EXPORT_SYMBOL_GPL(crypto_ahash_export);

int crypto_ahash_import(struct ahash_request *req, const void *in)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);

        if (likely(tfm->using_shash))
                return crypto_shash_import(prepare_shash_desc(req, tfm), in);
        if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
                return -ENOKEY;
        return crypto_ahash_alg(tfm)->import(req, in);
}
EXPORT_SYMBOL_GPL(crypto_ahash_import);

static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        alg->exit_tfm(hash);
}

static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
{
        struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
        struct ahash_alg *alg = crypto_ahash_alg(hash);

        crypto_ahash_set_statesize(hash, alg->halg.statesize);

        if (tfm->__crt_alg->cra_type == &crypto_shash_type)
                return crypto_init_ahash_using_shash(tfm);

        ahash_set_needkey(hash, alg);

        if (alg->exit_tfm)
                tfm->exit = crypto_ahash_exit_tfm;

        return alg->init_tfm ? alg->init_tfm(hash) : 0;
}

static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
{
        if (alg->cra_type == &crypto_shash_type)
                return sizeof(struct crypto_shash *);

        return crypto_alg_extsize(alg);
}

static void crypto_ahash_free_instance(struct crypto_instance *inst)
{
        struct ahash_instance *ahash = ahash_instance(inst);

        ahash->free(ahash);
}

static int __maybe_unused crypto_ahash_report(
        struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_hash rhash;

        memset(&rhash, 0, sizeof(rhash));

        strscpy(rhash.type, "ahash", sizeof(rhash.type));

        rhash.blocksize = alg->cra_blocksize;
        rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;

        return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash);
}

static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
        __maybe_unused;
static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
{
        seq_printf(m, "type         : ahash\n");
        seq_printf(m, "async        : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ?
                                             "yes" : "no");
        seq_printf(m, "blocksize    : %u\n", alg->cra_blocksize);
        seq_printf(m, "digestsize   : %u\n",
                   __crypto_hash_alg_common(alg)->digestsize);
}

static const struct crypto_type crypto_ahash_type = {
        .extsize = crypto_ahash_extsize,
        .init_tfm = crypto_ahash_init_tfm,
        .free = crypto_ahash_free_instance,
#ifdef CONFIG_PROC_FS
        .show = crypto_ahash_show,
#endif
#if IS_ENABLED(CONFIG_CRYPTO_USER)
        .report = crypto_ahash_report,
#endif
        .maskclear = ~CRYPTO_ALG_TYPE_MASK,
        .maskset = CRYPTO_ALG_TYPE_AHASH_MASK,
        .type = CRYPTO_ALG_TYPE_AHASH,
        .tfmsize = offsetof(struct crypto_ahash, base),
};

int crypto_grab_ahash(struct crypto_ahash_spawn *spawn,
                      struct crypto_instance *inst,
                      const char *name, u32 type, u32 mask)
{
        spawn->base.frontend = &crypto_ahash_type;
        return crypto_grab_spawn(&spawn->base, inst, name, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_grab_ahash);

struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type,
                                        u32 mask)
{
        return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_alloc_ahash);

int crypto_has_ahash(const char *alg_name, u32 type, u32 mask)
{
        return crypto_type_has_alg(alg_name, &crypto_ahash_type, type, mask);
}
EXPORT_SYMBOL_GPL(crypto_has_ahash);

static bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg)
{
        struct crypto_alg *alg = &halg->base;

        if (alg->cra_type == &crypto_shash_type)
                return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg));

        return __crypto_ahash_alg(alg)->setkey != ahash_nosetkey;
}

struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *hash)
{
        struct hash_alg_common *halg = crypto_hash_alg_common(hash);
        struct crypto_tfm *tfm = crypto_ahash_tfm(hash);
        struct crypto_ahash *nhash;
        struct ahash_alg *alg;
        int err;

        if (!crypto_hash_alg_has_setkey(halg)) {
                tfm = crypto_tfm_get(tfm);
                if (IS_ERR(tfm))
                        return ERR_CAST(tfm);

                return hash;
        }

        nhash = crypto_clone_tfm(&crypto_ahash_type, tfm);

        if (IS_ERR(nhash))
                return nhash;

        nhash->reqsize = hash->reqsize;
        nhash->statesize = hash->statesize;

        if (likely(hash->using_shash)) {
                struct crypto_shash **nctx = crypto_ahash_ctx(nhash);
                struct crypto_shash *shash;

                shash = crypto_clone_shash(ahash_to_shash(hash));
                if (IS_ERR(shash)) {
                        err = PTR_ERR(shash);
                        goto out_free_nhash;
                }
                nhash->using_shash = true;
                *nctx = shash;
                return nhash;
        }

        err = -ENOSYS;
        alg = crypto_ahash_alg(hash);
        if (!alg->clone_tfm)
                goto out_free_nhash;

        err = alg->clone_tfm(nhash, hash);
        if (err)
                goto out_free_nhash;

        return nhash;

out_free_nhash:
        crypto_free_ahash(nhash);
        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(crypto_clone_ahash);

static int ahash_prepare_alg(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;
        int err;

        if (alg->halg.statesize == 0)
                return -EINVAL;

        err = hash_prepare_alg(&alg->halg);
        if (err)
                return err;

        base->cra_type = &crypto_ahash_type;
        base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;

        if (!alg->finup)
                alg->finup = ahash_def_finup;
        if (!alg->setkey)
                alg->setkey = ahash_nosetkey;

        return 0;
}

int crypto_register_ahash(struct ahash_alg *alg)
{
        struct crypto_alg *base = &alg->halg.base;
        int err;

        err = ahash_prepare_alg(alg);
        if (err)
                return err;

        return crypto_register_alg(base);
}
EXPORT_SYMBOL_GPL(crypto_register_ahash);

void crypto_unregister_ahash(struct ahash_alg *alg)
{
        crypto_unregister_alg(&alg->halg.base);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahash);

int crypto_register_ahashes(struct ahash_alg *algs, int count)
{
        int i, ret;

        for (i = 0; i < count; i++) {
                ret = crypto_register_ahash(&algs[i]);
                if (ret)
                        goto err;
        }

        return 0;

err:
        for (--i; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);

        return ret;
}
EXPORT_SYMBOL_GPL(crypto_register_ahashes);

void crypto_unregister_ahashes(struct ahash_alg *algs, int count)
{
        int i;

        for (i = count - 1; i >= 0; --i)
                crypto_unregister_ahash(&algs[i]);
}
EXPORT_SYMBOL_GPL(crypto_unregister_ahashes);

int ahash_register_instance(struct crypto_template *tmpl,
                            struct ahash_instance *inst)
{
        int err;

        if (WARN_ON(!inst->free))
                return -EINVAL;

        err = ahash_prepare_alg(&inst->alg);
        if (err)
                return err;

        return crypto_register_instance(tmpl, ahash_crypto_instance(inst));
}
EXPORT_SYMBOL_GPL(ahash_register_instance);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Asynchronous cryptographic hash type");














































































































    2 



    2 
    2 

    2 

























    2 















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
// SPDX-License-Identifier: GPL-2.0
/*
 *  Block device concurrent positioning ranges.
 *
 *  Copyright (C) 2021 Western Digital Corporation or its Affiliates.
 */
#include <linux/kernel.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/init.h>

#include "blk.h"

static ssize_t
blk_ia_range_sector_show(struct blk_independent_access_range *iar,
                         char *buf)
{
        return sprintf(buf, "%llu\n", iar->sector);
}

static ssize_t
blk_ia_range_nr_sectors_show(struct blk_independent_access_range *iar,
                             char *buf)
{
        return sprintf(buf, "%llu\n", iar->nr_sectors);
}

struct blk_ia_range_sysfs_entry {
        struct attribute attr;
        ssize_t (*show)(struct blk_independent_access_range *iar, char *buf);
};

static struct blk_ia_range_sysfs_entry blk_ia_range_sector_entry = {
        .attr = { .name = "sector", .mode = 0444 },
        .show = blk_ia_range_sector_show,
};

static struct blk_ia_range_sysfs_entry blk_ia_range_nr_sectors_entry = {
        .attr = { .name = "nr_sectors", .mode = 0444 },
        .show = blk_ia_range_nr_sectors_show,
};

static struct attribute *blk_ia_range_attrs[] = {
        &blk_ia_range_sector_entry.attr,
        &blk_ia_range_nr_sectors_entry.attr,
        NULL,
};
ATTRIBUTE_GROUPS(blk_ia_range);

static ssize_t blk_ia_range_sysfs_show(struct kobject *kobj,
                                      struct attribute *attr, char *buf)
{
        struct blk_ia_range_sysfs_entry *entry =
                container_of(attr, struct blk_ia_range_sysfs_entry, attr);
        struct blk_independent_access_range *iar =
                container_of(kobj, struct blk_independent_access_range, kobj);

        return entry->show(iar, buf);
}

static const struct sysfs_ops blk_ia_range_sysfs_ops = {
        .show        = blk_ia_range_sysfs_show,
};

/*
 * Independent access range entries are not freed individually, but alltogether
 * with struct blk_independent_access_ranges and its array of ranges. Since
 * kobject_add() takes a reference on the parent kobject contained in
 * struct blk_independent_access_ranges, the array of independent access range
 * entries cannot be freed until kobject_del() is called for all entries.
 * So we do not need to do anything here, but still need this no-op release
 * operation to avoid complaints from the kobject code.
 */
static void blk_ia_range_sysfs_nop_release(struct kobject *kobj)
{
}

static const struct kobj_type blk_ia_range_ktype = {
        .sysfs_ops        = &blk_ia_range_sysfs_ops,
        .default_groups        = blk_ia_range_groups,
        .release        = blk_ia_range_sysfs_nop_release,
};

/*
 * This will be executed only after all independent access range entries are
 * removed with kobject_del(), at which point, it is safe to free everything,
 * including the array of ranges.
 */
static void blk_ia_ranges_sysfs_release(struct kobject *kobj)
{
        struct blk_independent_access_ranges *iars =
                container_of(kobj, struct blk_independent_access_ranges, kobj);

        kfree(iars);
}

static const struct kobj_type blk_ia_ranges_ktype = {
        .release        = blk_ia_ranges_sysfs_release,
};

/**
 * disk_register_independent_access_ranges - register with sysfs a set of
 *                independent access ranges
 * @disk:        Target disk
 *
 * Register with sysfs a set of independent access ranges for @disk.
 */
int disk_register_independent_access_ranges(struct gendisk *disk)
{
        struct blk_independent_access_ranges *iars = disk->ia_ranges;
        struct request_queue *q = disk->queue;
        int i, ret;

        lockdep_assert_held(&q->sysfs_dir_lock);
        lockdep_assert_held(&q->sysfs_lock);

        if (!iars)
                return 0;

        /*
         * At this point, iars is the new set of sector access ranges that needs
         * to be registered with sysfs.
         */
        WARN_ON(iars->sysfs_registered);
        ret = kobject_init_and_add(&iars->kobj, &blk_ia_ranges_ktype,
                                   &disk->queue_kobj, "%s",
                                   "independent_access_ranges");
        if (ret) {
                disk->ia_ranges = NULL;
                kobject_put(&iars->kobj);
                return ret;
        }

        for (i = 0; i < iars->nr_ia_ranges; i++) {
                ret = kobject_init_and_add(&iars->ia_range[i].kobj,
                                           &blk_ia_range_ktype, &iars->kobj,
                                           "%d", i);
                if (ret) {
                        while (--i >= 0)
                                kobject_del(&iars->ia_range[i].kobj);
                        kobject_del(&iars->kobj);
                        kobject_put(&iars->kobj);
                        return ret;
                }
        }

        iars->sysfs_registered = true;

        return 0;
}

void disk_unregister_independent_access_ranges(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;
        struct blk_independent_access_ranges *iars = disk->ia_ranges;
        int i;

        lockdep_assert_held(&q->sysfs_dir_lock);
        lockdep_assert_held(&q->sysfs_lock);

        if (!iars)
                return;

        if (iars->sysfs_registered) {
                for (i = 0; i < iars->nr_ia_ranges; i++)
                        kobject_del(&iars->ia_range[i].kobj);
                kobject_del(&iars->kobj);
                kobject_put(&iars->kobj);
        } else {
                kfree(iars);
        }

        disk->ia_ranges = NULL;
}

static struct blk_independent_access_range *
disk_find_ia_range(struct blk_independent_access_ranges *iars,
                  sector_t sector)
{
        struct blk_independent_access_range *iar;
        int i;

        for (i = 0; i < iars->nr_ia_ranges; i++) {
                iar = &iars->ia_range[i];
                if (sector >= iar->sector &&
                    sector < iar->sector + iar->nr_sectors)
                        return iar;
        }

        return NULL;
}

static bool disk_check_ia_ranges(struct gendisk *disk,
                                struct blk_independent_access_ranges *iars)
{
        struct blk_independent_access_range *iar, *tmp;
        sector_t capacity = get_capacity(disk);
        sector_t sector = 0;
        int i;

        if (WARN_ON_ONCE(!iars->nr_ia_ranges))
                return false;

        /*
         * While sorting the ranges in increasing LBA order, check that the
         * ranges do not overlap, that there are no sector holes and that all
         * sectors belong to one range.
         */
        for (i = 0; i < iars->nr_ia_ranges; i++) {
                tmp = disk_find_ia_range(iars, sector);
                if (!tmp || tmp->sector != sector) {
                        pr_warn("Invalid non-contiguous independent access ranges\n");
                        return false;
                }

                iar = &iars->ia_range[i];
                if (tmp != iar) {
                        swap(iar->sector, tmp->sector);
                        swap(iar->nr_sectors, tmp->nr_sectors);
                }

                sector += iar->nr_sectors;
        }

        if (sector != capacity) {
                pr_warn("Independent access ranges do not match disk capacity\n");
                return false;
        }

        return true;
}

static bool disk_ia_ranges_changed(struct gendisk *disk,
                                   struct blk_independent_access_ranges *new)
{
        struct blk_independent_access_ranges *old = disk->ia_ranges;
        int i;

        if (!old)
                return true;

        if (old->nr_ia_ranges != new->nr_ia_ranges)
                return true;

        for (i = 0; i < old->nr_ia_ranges; i++) {
                if (new->ia_range[i].sector != old->ia_range[i].sector ||
                    new->ia_range[i].nr_sectors != old->ia_range[i].nr_sectors)
                        return true;
        }

        return false;
}

/**
 * disk_alloc_independent_access_ranges - Allocate an independent access ranges
 *                                        data structure
 * @disk:                target disk
 * @nr_ia_ranges:        Number of independent access ranges
 *
 * Allocate a struct blk_independent_access_ranges structure with @nr_ia_ranges
 * access range descriptors.
 */
struct blk_independent_access_ranges *
disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges)
{
        struct blk_independent_access_ranges *iars;

        iars = kzalloc_node(struct_size(iars, ia_range, nr_ia_ranges),
                            GFP_KERNEL, disk->queue->node);
        if (iars)
                iars->nr_ia_ranges = nr_ia_ranges;
        return iars;
}
EXPORT_SYMBOL_GPL(disk_alloc_independent_access_ranges);

/**
 * disk_set_independent_access_ranges - Set a disk independent access ranges
 * @disk:        target disk
 * @iars:        independent access ranges structure
 *
 * Set the independent access ranges information of the request queue
 * of @disk to @iars. If @iars is NULL and the independent access ranges
 * structure already set is cleared. If there are no differences between
 * @iars and the independent access ranges structure already set, @iars
 * is freed.
 */
void disk_set_independent_access_ranges(struct gendisk *disk,
                                struct blk_independent_access_ranges *iars)
{
        struct request_queue *q = disk->queue;

        mutex_lock(&q->sysfs_dir_lock);
        mutex_lock(&q->sysfs_lock);
        if (iars && !disk_check_ia_ranges(disk, iars)) {
                kfree(iars);
                iars = NULL;
        }
        if (iars && !disk_ia_ranges_changed(disk, iars)) {
                kfree(iars);
                goto unlock;
        }

        /*
         * This may be called for a registered queue. E.g. during a device
         * revalidation. If that is the case, we need to unregister the old
         * set of independent access ranges and register the new set. If the
         * queue is not registered, registration of the device request queue
         * will register the independent access ranges.
         */
        disk_unregister_independent_access_ranges(disk);
        disk->ia_ranges = iars;
        if (blk_queue_registered(q))
                disk_register_independent_access_ranges(disk);
unlock:
        mutex_unlock(&q->sysfs_lock);
        mutex_unlock(&q->sysfs_dir_lock);
}
EXPORT_SYMBOL_GPL(disk_set_independent_access_ranges);





















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
/* SPDX-License-Identifier: GPL-2.0-only */
/* L2TP internal definitions.
 *
 * Copyright (c) 2008,2009 Katalix Systems Ltd
 */
#include <linux/refcount.h>

#ifndef _L2TP_CORE_H_
#define _L2TP_CORE_H_

#include <net/dst.h>
#include <net/sock.h>

#ifdef CONFIG_XFRM
#include <net/xfrm.h>
#endif

/* Random numbers used for internal consistency checks of tunnel and session structures */
#define L2TP_SESSION_MAGIC        0x0C04EB7D

struct sk_buff;

struct l2tp_stats {
        atomic_long_t                tx_packets;
        atomic_long_t                tx_bytes;
        atomic_long_t                tx_errors;
        atomic_long_t                rx_packets;
        atomic_long_t                rx_bytes;
        atomic_long_t                rx_seq_discards;
        atomic_long_t                rx_oos_packets;
        atomic_long_t                rx_errors;
        atomic_long_t                rx_cookie_discards;
        atomic_long_t                rx_invalid;
};

struct l2tp_tunnel;

/* L2TP session configuration */
struct l2tp_session_cfg {
        enum l2tp_pwtype        pw_type;
        unsigned int                recv_seq:1;        /* expect receive packets with sequence numbers? */
        unsigned int                send_seq:1;        /* send packets with sequence numbers? */
        unsigned int                lns_mode:1;        /* behave as LNS?
                                                 * LAC enables sequence numbers under LNS control.
                                                 */
        u16                        l2specific_type; /* Layer 2 specific type */
        u8                        cookie[8];        /* optional cookie */
        int                        cookie_len;        /* 0, 4 or 8 bytes */
        u8                        peer_cookie[8];        /* peer's cookie */
        int                        peer_cookie_len; /* 0, 4 or 8 bytes */
        int                        reorder_timeout; /* configured reorder timeout (in jiffies) */
        char                        *ifname;
};

struct l2tp_session_coll_list {
        spinlock_t lock;        /* for access to list */
        struct list_head list;
        refcount_t ref_count;
};

/* Represents a session (pseudowire) instance.
 * Tracks runtime state including cookies, dataplane packet sequencing, and IO statistics.
 * Is linked into a per-tunnel session list and a per-net ("global") IDR tree.
 */
#define L2TP_SESSION_NAME_MAX 32
struct l2tp_session {
        int                        magic;                /* should be L2TP_SESSION_MAGIC */
        long                        dead;
        struct rcu_head                rcu;

        struct l2tp_tunnel        *tunnel;        /* back pointer to tunnel context */
        u32                        session_id;
        u32                        peer_session_id;
        u8                        cookie[8];
        int                        cookie_len;
        u8                        peer_cookie[8];
        int                        peer_cookie_len;
        u16                        l2specific_type;
        u16                        hdr_len;
        u32                        nr;                /* session NR state (receive) */
        u32                        ns;                /* session NR state (send) */
        struct sk_buff_head        reorder_q;        /* receive reorder queue */
        u32                        nr_max;                /* max NR. Depends on tunnel */
        u32                        nr_window_size;        /* NR window size */
        u32                        nr_oos;                /* NR of last OOS packet */
        int                        nr_oos_count;        /* for OOS recovery */
        int                        nr_oos_count_max;
        struct list_head        list;                /* per-tunnel list node */
        refcount_t                ref_count;
        struct hlist_node        hlist;                /* per-net session hlist */
        unsigned long                hlist_key;        /* key for session hlist */
        struct l2tp_session_coll_list *coll_list; /* session collision list */
        struct list_head        clist;                /* for coll_list */

        char                        name[L2TP_SESSION_NAME_MAX]; /* for logging */
        char                        ifname[IFNAMSIZ];
        unsigned int                recv_seq:1;        /* expect receive packets with sequence numbers? */
        unsigned int                send_seq:1;        /* send packets with sequence numbers? */
        unsigned int                lns_mode:1;        /* behave as LNS?
                                                 * LAC enables sequence numbers under LNS control.
                                                 */
        int                        reorder_timeout; /* configured reorder timeout (in jiffies) */
        int                        reorder_skip;        /* set if skip to next nr */
        enum l2tp_pwtype        pwtype;
        struct l2tp_stats        stats;
        struct work_struct        del_work;

        /* Session receive handler for data packets.
         * Each pseudowire implementation should implement this callback in order to
         * handle incoming packets.  Packets are passed to the pseudowire handler after
         * reordering, if data sequence numbers are enabled for the session.
         */
        void (*recv_skb)(struct l2tp_session *session, struct sk_buff *skb, int data_len);

        /* Session close handler.
         * Each pseudowire implementation may implement this callback in order to carry
         * out pseudowire-specific shutdown actions.
         * The callback is called by core after unlisting the session and purging its
         * reorder queue.
         */
        void (*session_close)(struct l2tp_session *session);

        /* Session show handler.
         * Pseudowire-specific implementation of debugfs session rendering.
         * The callback is called by l2tp_debugfs.c after rendering core session
         * information.
         */
        void (*show)(struct seq_file *m, void *priv);

        u8                        priv[];                /* private data */
};

/* L2TP tunnel configuration */
struct l2tp_tunnel_cfg {
        enum l2tp_encap_type        encap;

        /* Used only for kernel-created sockets */
        struct in_addr                local_ip;
        struct in_addr                peer_ip;
#if IS_ENABLED(CONFIG_IPV6)
        struct in6_addr                *local_ip6;
        struct in6_addr                *peer_ip6;
#endif
        u16                        local_udp_port;
        u16                        peer_udp_port;
        unsigned int                use_udp_checksums:1,
                                udp6_zero_tx_checksums:1,
                                udp6_zero_rx_checksums:1;
};

/* Represents a tunnel instance.
 * Tracks runtime state including IO statistics.
 * Holds the tunnel socket (either passed from userspace or directly created by the kernel).
 * Maintains a list of sessions belonging to the tunnel instance.
 * Is linked into a per-net list of tunnels.
 */
#define L2TP_TUNNEL_NAME_MAX 20
struct l2tp_tunnel {
        unsigned long                dead;

        struct rcu_head rcu;
        spinlock_t                list_lock;        /* write-protection for session_list */
        bool                        acpt_newsess;        /* indicates whether this tunnel accepts
                                                 * new sessions. Protected by list_lock.
                                                 */
        struct list_head        session_list;        /* list of sessions */
        u32                        tunnel_id;
        u32                        peer_tunnel_id;
        int                        version;        /* 2=>L2TPv2, 3=>L2TPv3 */

        char                        name[L2TP_TUNNEL_NAME_MAX]; /* for logging */
        enum l2tp_encap_type        encap;
        struct l2tp_stats        stats;

        struct net                *l2tp_net;        /* the net we belong to */

        refcount_t                ref_count;
        struct sock                *sock;                /* parent socket */
        int                        fd;                /* parent fd, if tunnel socket was created
                                                 * by userspace
                                                 */

        struct work_struct        del_work;
};

/* Pseudowire ops callbacks for use with the l2tp genetlink interface */
struct l2tp_nl_cmd_ops {
        /* The pseudowire session create callback is responsible for creating a session
         * instance for a specific pseudowire type.
         * It must call l2tp_session_create and l2tp_session_register to register the
         * session instance, as well as carry out any pseudowire-specific initialisation.
         * It must return >= 0 on success, or an appropriate negative errno value on failure.
         */
        int (*session_create)(struct net *net, struct l2tp_tunnel *tunnel,
                              u32 session_id, u32 peer_session_id,
                              struct l2tp_session_cfg *cfg);

        /* The pseudowire session delete callback is responsible for initiating the deletion
         * of a session instance.
         * It must call l2tp_session_delete, as well as carry out any pseudowire-specific
         * teardown actions.
         */
        void (*session_delete)(struct l2tp_session *session);
};

static inline void *l2tp_session_priv(struct l2tp_session *session)
{
        return &session->priv[0];
}

/* Tunnel and session refcounts */
void l2tp_tunnel_put(struct l2tp_tunnel *tunnel);
void l2tp_session_put(struct l2tp_session *session);

/* Tunnel and session lookup.
 * These functions take a reference on the instances they return, so
 * the caller must ensure that the reference is dropped appropriately.
 */
struct l2tp_tunnel *l2tp_tunnel_get(const struct net *net, u32 tunnel_id);
struct l2tp_tunnel *l2tp_tunnel_get_next(const struct net *net, unsigned long *key);

struct l2tp_session *l2tp_v3_session_get(const struct net *net, struct sock *sk, u32 session_id);
struct l2tp_session *l2tp_v2_session_get(const struct net *net, u16 tunnel_id, u16 session_id);
struct l2tp_session *l2tp_session_get(const struct net *net, struct sock *sk, int pver,
                                      u32 tunnel_id, u32 session_id);
struct l2tp_session *l2tp_session_get_next(const struct net *net, struct sock *sk, int pver,
                                           u32 tunnel_id, unsigned long *key);
struct l2tp_session *l2tp_session_get_by_ifname(const struct net *net,
                                                const char *ifname);

/* Tunnel and session lifetime management.
 * Creation of a new instance is a two-step process: create, then register.
 * Destruction is triggered using the *_delete functions, and completes asynchronously.
 */
int l2tp_tunnel_create(int fd, int version, u32 tunnel_id,
                       u32 peer_tunnel_id, struct l2tp_tunnel_cfg *cfg,
                       struct l2tp_tunnel **tunnelp);
int l2tp_tunnel_register(struct l2tp_tunnel *tunnel, struct net *net,
                         struct l2tp_tunnel_cfg *cfg);
void l2tp_tunnel_delete(struct l2tp_tunnel *tunnel);

struct l2tp_session *l2tp_session_create(int priv_size,
                                         struct l2tp_tunnel *tunnel,
                                         u32 session_id, u32 peer_session_id,
                                         struct l2tp_session_cfg *cfg);
int l2tp_session_register(struct l2tp_session *session,
                          struct l2tp_tunnel *tunnel);
void l2tp_session_delete(struct l2tp_session *session);

/* Receive path helpers.  If data sequencing is enabled for the session these
 * functions handle queuing and reordering prior to passing packets to the
 * pseudowire code to be passed to userspace.
 */
void l2tp_recv_common(struct l2tp_session *session, struct sk_buff *skb,
                      unsigned char *ptr, unsigned char *optr, u16 hdrflags,
                      int length);
int l2tp_udp_encap_recv(struct sock *sk, struct sk_buff *skb);

/* Transmit path helpers for sending packets over the tunnel socket. */
void l2tp_session_set_header_len(struct l2tp_session *session, int version,
                                 enum l2tp_encap_type encap);
int l2tp_xmit_skb(struct l2tp_session *session, struct sk_buff *skb);

/* Pseudowire management.
 * Pseudowires should register with l2tp core on module init, and unregister
 * on module exit.
 */
int l2tp_nl_register_ops(enum l2tp_pwtype pw_type, const struct l2tp_nl_cmd_ops *ops);
void l2tp_nl_unregister_ops(enum l2tp_pwtype pw_type);

/* IOCTL helper for IP encap modules. */
int l2tp_ioctl(struct sock *sk, int cmd, int *karg);

struct l2tp_tunnel *l2tp_sk_to_tunnel(const struct sock *sk);

static inline int l2tp_get_l2specific_len(struct l2tp_session *session)
{
        switch (session->l2specific_type) {
        case L2TP_L2SPECTYPE_DEFAULT:
                return 4;
        case L2TP_L2SPECTYPE_NONE:
        default:
                return 0;
        }
}

static inline u32 l2tp_tunnel_dst_mtu(const struct l2tp_tunnel *tunnel)
{
        struct dst_entry *dst;
        u32 mtu;

        dst = sk_dst_get(tunnel->sock);
        if (!dst)
                return 0;

        mtu = dst_mtu(dst);
        dst_release(dst);

        return mtu;
}

#ifdef CONFIG_XFRM
static inline bool l2tp_tunnel_uses_xfrm(const struct l2tp_tunnel *tunnel)
{
        struct sock *sk = tunnel->sock;

        return sk && (rcu_access_pointer(sk->sk_policy[0]) ||
                      rcu_access_pointer(sk->sk_policy[1]));
}
#else
static inline bool l2tp_tunnel_uses_xfrm(const struct l2tp_tunnel *tunnel)
{
        return false;
}
#endif

static inline int l2tp_v3_ensure_opt_in_linear(struct l2tp_session *session, struct sk_buff *skb,
                                               unsigned char **ptr, unsigned char **optr)
{
        int opt_len = session->peer_cookie_len + l2tp_get_l2specific_len(session);

        if (opt_len > 0) {
                int off = *ptr - *optr;

                if (!pskb_may_pull(skb, off + opt_len))
                        return -1;

                if (skb->data != *optr) {
                        *optr = skb->data;
                        *ptr = skb->data + off;
                }
        }

        return 0;
}

#define MODULE_ALIAS_L2TP_PWTYPE(type) \
        MODULE_ALIAS("net-l2tp-type-" __stringify(type))

#endif /* _L2TP_CORE_H_ */


















    5 


    1 






































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#include <linux/errno.h>
#include <linux/ip.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/types.h>
#include <net/checksum.h>
#include <net/ip.h>
#include <net/ip6_fib.h>
#include <net/lwtunnel.h>
#include <net/protocol.h>
#include <uapi/linux/ila.h>
#include "ila.h"

void ila_init_saved_csum(struct ila_params *p)
{
        if (!p->locator_match.v64)
                return;

        p->csum_diff = compute_csum_diff8(
                                (__be32 *)&p->locator,
                                (__be32 *)&p->locator_match);
}

static __wsum get_csum_diff_iaddr(struct ila_addr *iaddr, struct ila_params *p)
{
        if (p->locator_match.v64)
                return p->csum_diff;
        else
                return compute_csum_diff8((__be32 *)&p->locator,
                                          (__be32 *)&iaddr->loc);
}

static __wsum get_csum_diff(struct ipv6hdr *ip6h, struct ila_params *p)
{
        return get_csum_diff_iaddr(ila_a2i(&ip6h->daddr), p);
}

static void ila_csum_do_neutral_fmt(struct ila_addr *iaddr,
                                    struct ila_params *p)
{
        __sum16 *adjust = (__force __sum16 *)&iaddr->ident.v16[3];
        __wsum diff, fval;

        diff = get_csum_diff_iaddr(iaddr, p);

        fval = (__force __wsum)(ila_csum_neutral_set(iaddr->ident) ?
                        CSUM_NEUTRAL_FLAG : ~CSUM_NEUTRAL_FLAG);

        diff = csum_add(diff, fval);

        *adjust = ~csum_fold(csum_add(diff, csum_unfold(*adjust)));

        /* Flip the csum-neutral bit. Either we are doing a SIR->ILA
         * translation with ILA_CSUM_NEUTRAL_MAP as the csum_method
         * and the C-bit is not set, or we are doing an ILA-SIR
         * tranlsation and the C-bit is set.
         */
        iaddr->ident.csum_neutral ^= 1;
}

static void ila_csum_do_neutral_nofmt(struct ila_addr *iaddr,
                                      struct ila_params *p)
{
        __sum16 *adjust = (__force __sum16 *)&iaddr->ident.v16[3];
        __wsum diff;

        diff = get_csum_diff_iaddr(iaddr, p);

        *adjust = ~csum_fold(csum_add(diff, csum_unfold(*adjust)));
}

static void ila_csum_adjust_transport(struct sk_buff *skb,
                                      struct ila_params *p)
{
        size_t nhoff = sizeof(struct ipv6hdr);
        struct ipv6hdr *ip6h = ipv6_hdr(skb);
        __wsum diff;

        switch (ip6h->nexthdr) {
        case NEXTHDR_TCP:
                if (likely(pskb_may_pull(skb, nhoff + sizeof(struct tcphdr)))) {
                        struct tcphdr *th = (struct tcphdr *)
                                        (skb_network_header(skb) + nhoff);

                        diff = get_csum_diff(ip6h, p);
                        inet_proto_csum_replace_by_diff(&th->check, skb,
                                                        diff, true);
                }
                break;
        case NEXTHDR_UDP:
                if (likely(pskb_may_pull(skb, nhoff + sizeof(struct udphdr)))) {
                        struct udphdr *uh = (struct udphdr *)
                                        (skb_network_header(skb) + nhoff);

                        if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) {
                                diff = get_csum_diff(ip6h, p);
                                inet_proto_csum_replace_by_diff(&uh->check, skb,
                                                                diff, true);
                                if (!uh->check)
                                        uh->check = CSUM_MANGLED_0;
                        }
                }
                break;
        case NEXTHDR_ICMP:
                if (likely(pskb_may_pull(skb,
                                         nhoff + sizeof(struct icmp6hdr)))) {
                        struct icmp6hdr *ih = (struct icmp6hdr *)
                                        (skb_network_header(skb) + nhoff);

                        diff = get_csum_diff(ip6h, p);
                        inet_proto_csum_replace_by_diff(&ih->icmp6_cksum, skb,
                                                        diff, true);
                }
                break;
        }
}

void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p,
                             bool sir2ila)
{
        struct ipv6hdr *ip6h = ipv6_hdr(skb);
        struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);

        switch (p->csum_mode) {
        case ILA_CSUM_ADJUST_TRANSPORT:
                ila_csum_adjust_transport(skb, p);
                break;
        case ILA_CSUM_NEUTRAL_MAP:
                if (sir2ila) {
                        if (WARN_ON(ila_csum_neutral_set(iaddr->ident))) {
                                /* Checksum flag should never be
                                 * set in a formatted SIR address.
                                 */
                                break;
                        }
                } else if (!ila_csum_neutral_set(iaddr->ident)) {
                        /* ILA to SIR translation and C-bit isn't
                         * set so we're good.
                         */
                        break;
                }
                ila_csum_do_neutral_fmt(iaddr, p);
                break;
        case ILA_CSUM_NEUTRAL_MAP_AUTO:
                ila_csum_do_neutral_nofmt(iaddr, p);
                break;
        case ILA_CSUM_NO_ACTION:
                break;
        }

        /* Now change destination address */
        iaddr->loc = p->locator;
}








































































































































































































































































































































































































































































































































  753 





  752 





















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
/*
 *  linux/include/linux/console.h
 *
 *  Copyright (C) 1993        Hamish Macdonald
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file COPYING in the main directory of this archive
 * for more details.
 *
 * Changed:
 * 10-Mar-94: Arno Griffioen: Conversion for vt100 emulator port from PC LINUX
 */

#ifndef _LINUX_CONSOLE_H_
#define _LINUX_CONSOLE_H_ 1

#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/irq_work.h>
#include <linux/rculist.h>
#include <linux/rcuwait.h>
#include <linux/types.h>
#include <linux/vesa.h>

struct vc_data;
struct console_font_op;
struct console_font;
struct module;
struct tty_struct;
struct notifier_block;

enum con_scroll {
        SM_UP,
        SM_DOWN,
};

enum vc_intensity;

/**
 * struct consw - callbacks for consoles
 *
 * @owner:      the module to get references of when this console is used
 * @con_startup: set up the console and return its name (like VGA, EGA, ...)
 * @con_init:   initialize the console on @vc. @init is true for the very first
 *                call on this @vc.
 * @con_deinit: deinitialize the console from @vc.
 * @con_clear:  erase @count characters at [@x, @y] on @vc. @count >= 1.
 * @con_putc:   emit one character with attributes @ca to [@x, @y] on @vc.
 *                (optional -- @con_putcs would be called instead)
 * @con_putcs:  emit @count characters with attributes @s to [@x, @y] on @vc.
 * @con_cursor: enable/disable cursor depending on @enable
 * @con_scroll: move lines from @top to @bottom in direction @dir by @lines.
 *                Return true if no generic handling should be done.
 *                Invoked by csi_M and printing to the console.
 * @con_switch: notifier about the console switch; it is supposed to return
 *                true if a redraw is needed.
 * @con_blank:  blank/unblank the console. The target mode is passed in @blank.
 *                @mode_switch is set if changing from/to text/graphics. The hook
 *                is supposed to return true if a redraw is needed.
 * @con_font_set: set console @vc font to @font with height @vpitch. @flags can
 *                be %KD_FONT_FLAG_DONT_RECALC. (optional)
 * @con_font_get: fetch the current font on @vc of height @vpitch into @font.
 *                (optional)
 * @con_font_default: set default font on @vc. @name can be %NULL or font name
 *                to search for. @font can be filled back. (optional)
 * @con_resize:        resize the @vc console to @width x @height. @from_user is true
 *                when this change comes from the user space.
 * @con_set_palette: sets the palette of the console @vc to @table (optional)
 * @con_scrolldelta: the contents of the console should be scrolled by @lines.
 *                     Invoked by user. (optional)
 * @con_set_origin: set origin (see &vc_data::vc_origin) of the @vc. If not
 *                provided or returns false, the origin is set to
 *                @vc->vc_screenbuf. (optional)
 * @con_save_screen: save screen content into @vc->vc_screenbuf. Called e.g.
 *                upon entering graphics. (optional)
 * @con_build_attr: build attributes based on @color, @intensity and other
 *                parameters. The result is used for both normal and erase
 *                characters. (optional)
 * @con_invert_region: invert a region of length @count on @vc starting at @p.
 *                (optional)
 * @con_debug_enter: prepare the console for the debugger. This includes, but
 *                is not limited to, unblanking the console, loading an
 *                appropriate palette, and allowing debugger generated output.
 *                (optional)
 * @con_debug_leave: restore the console to its pre-debug state as closely as
 *                possible. (optional)
 */
struct consw {
        struct module *owner;
        const char *(*con_startup)(void);
        void        (*con_init)(struct vc_data *vc, bool init);
        void        (*con_deinit)(struct vc_data *vc);
        void        (*con_clear)(struct vc_data *vc, unsigned int y,
                             unsigned int x, unsigned int count);
        void        (*con_putc)(struct vc_data *vc, u16 ca, unsigned int y,
                            unsigned int x);
        void        (*con_putcs)(struct vc_data *vc, const u16 *s,
                             unsigned int count, unsigned int ypos,
                             unsigned int xpos);
        void        (*con_cursor)(struct vc_data *vc, bool enable);
        bool        (*con_scroll)(struct vc_data *vc, unsigned int top,
                        unsigned int bottom, enum con_scroll dir,
                        unsigned int lines);
        bool        (*con_switch)(struct vc_data *vc);
        bool        (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank,
                             bool mode_switch);
        int        (*con_font_set)(struct vc_data *vc,
                                const struct console_font *font,
                                unsigned int vpitch, unsigned int flags);
        int        (*con_font_get)(struct vc_data *vc, struct console_font *font,
                        unsigned int vpitch);
        int        (*con_font_default)(struct vc_data *vc,
                        struct console_font *font, const char *name);
        int     (*con_resize)(struct vc_data *vc, unsigned int width,
                              unsigned int height, bool from_user);
        void        (*con_set_palette)(struct vc_data *vc,
                        const unsigned char *table);
        void        (*con_scrolldelta)(struct vc_data *vc, int lines);
        bool        (*con_set_origin)(struct vc_data *vc);
        void        (*con_save_screen)(struct vc_data *vc);
        u8        (*con_build_attr)(struct vc_data *vc, u8 color,
                        enum vc_intensity intensity,
                        bool blink, bool underline, bool reverse, bool italic);
        void        (*con_invert_region)(struct vc_data *vc, u16 *p, int count);
        void        (*con_debug_enter)(struct vc_data *vc);
        void        (*con_debug_leave)(struct vc_data *vc);
};

extern const struct consw *conswitchp;

extern const struct consw dummy_con;        /* dummy console buffer */
extern const struct consw vga_con;        /* VGA text console */
extern const struct consw newport_con;        /* SGI Newport console  */

struct screen_info;
#ifdef CONFIG_VGA_CONSOLE
void vgacon_register_screen(struct screen_info *si);
#else
static inline void vgacon_register_screen(struct screen_info *si) { }
#endif

int con_is_bound(const struct consw *csw);
int do_unregister_con_driver(const struct consw *csw);
int do_take_over_console(const struct consw *sw, int first, int last, int deflt);
void give_up_console(const struct consw *sw);
#ifdef CONFIG_VT
void con_debug_enter(struct vc_data *vc);
void con_debug_leave(void);
#else
static inline void con_debug_enter(struct vc_data *vc) { }
static inline void con_debug_leave(void) { }
#endif

/*
 * The interface for a console, or any other device that wants to capture
 * console messages (printer driver?)
 */

/**
 * enum cons_flags - General console flags
 * @CON_PRINTBUFFER:        Used by newly registered consoles to avoid duplicate
 *                        output of messages that were already shown by boot
 *                        consoles or read by userspace via syslog() syscall.
 * @CON_CONSDEV:        Indicates that the console driver is backing
 *                        /dev/console.
 * @CON_ENABLED:        Indicates if a console is allowed to print records. If
 *                        false, the console also will not advance to later
 *                        records.
 * @CON_BOOT:                Marks the console driver as early console driver which
 *                        is used during boot before the real driver becomes
 *                        available. It will be automatically unregistered
 *                        when the real console driver is registered unless
 *                        "keep_bootcon" parameter is used.
 * @CON_ANYTIME:        A misnomed historical flag which tells the core code
 *                        that the legacy @console::write callback can be invoked
 *                        on a CPU which is marked OFFLINE. That is misleading as
 *                        it suggests that there is no contextual limit for
 *                        invoking the callback. The original motivation was
 *                        readiness of the per-CPU areas.
 * @CON_BRL:                Indicates a braille device which is exempt from
 *                        receiving the printk spam for obvious reasons.
 * @CON_EXTENDED:        The console supports the extended output format of
 *                        /dev/kmesg which requires a larger output buffer.
 * @CON_SUSPENDED:        Indicates if a console is suspended. If true, the
 *                        printing callbacks must not be called.
 * @CON_NBCON:                Console can operate outside of the legacy style console_lock
 *                        constraints.
 */
enum cons_flags {
        CON_PRINTBUFFER                = BIT(0),
        CON_CONSDEV                = BIT(1),
        CON_ENABLED                = BIT(2),
        CON_BOOT                = BIT(3),
        CON_ANYTIME                = BIT(4),
        CON_BRL                        = BIT(5),
        CON_EXTENDED                = BIT(6),
        CON_SUSPENDED                = BIT(7),
        CON_NBCON                = BIT(8),
};

/**
 * struct nbcon_state - console state for nbcon consoles
 * @atom:        Compound of the state fields for atomic operations
 *
 * @req_prio:                The priority of a handover request
 * @prio:                The priority of the current owner
 * @unsafe:                Console is busy in a non takeover region
 * @unsafe_takeover:        A hostile takeover in an unsafe state happened in the
 *                        past. The console cannot be safe until re-initialized.
 * @cpu:                The CPU on which the owner runs
 *
 * To be used for reading and preparing of the value stored in the nbcon
 * state variable @console::nbcon_state.
 *
 * The @prio and @req_prio fields are particularly important to allow
 * spin-waiting to timeout and give up without the risk of a waiter being
 * assigned the lock after giving up.
 */
struct nbcon_state {
        union {
                unsigned int        atom;
                struct {
                        unsigned int prio                :  2;
                        unsigned int req_prio                :  2;
                        unsigned int unsafe                :  1;
                        unsigned int unsafe_takeover        :  1;
                        unsigned int cpu                : 24;
                };
        };
};

/*
 * The nbcon_state struct is used to easily create and interpret values that
 * are stored in the @console::nbcon_state variable. Ensure this struct stays
 * within the size boundaries of the atomic variable's underlying type in
 * order to avoid any accidental truncation.
 */
static_assert(sizeof(struct nbcon_state) <= sizeof(int));

/**
 * enum nbcon_prio - console owner priority for nbcon consoles
 * @NBCON_PRIO_NONE:                Unused
 * @NBCON_PRIO_NORMAL:                Normal (non-emergency) usage
 * @NBCON_PRIO_EMERGENCY:        Emergency output (WARN/OOPS...)
 * @NBCON_PRIO_PANIC:                Panic output
 * @NBCON_PRIO_MAX:                The number of priority levels
 *
 * A higher priority context can takeover the console when it is
 * in the safe state. The final attempt to flush consoles in panic()
 * can be allowed to do so even in an unsafe state (Hope and pray).
 */
enum nbcon_prio {
        NBCON_PRIO_NONE = 0,
        NBCON_PRIO_NORMAL,
        NBCON_PRIO_EMERGENCY,
        NBCON_PRIO_PANIC,
        NBCON_PRIO_MAX,
};

struct console;
struct printk_buffers;

/**
 * struct nbcon_context - Context for console acquire/release
 * @console:                        The associated console
 * @spinwait_max_us:                Limit for spin-wait acquire
 * @prio:                        Priority of the context
 * @allow_unsafe_takeover:        Allow performing takeover even if unsafe. Can
 *                                be used only with NBCON_PRIO_PANIC @prio. It
 *                                might cause a system freeze when the console
 *                                is used later.
 * @backlog:                        Ringbuffer has pending records
 * @pbufs:                        Pointer to the text buffer for this context
 * @seq:                        The sequence number to print for this context
 */
struct nbcon_context {
        /* members set by caller */
        struct console                *console;
        unsigned int                spinwait_max_us;
        enum nbcon_prio                prio;
        unsigned int                allow_unsafe_takeover        : 1;

        /* members set by emit */
        unsigned int                backlog                        : 1;

        /* members set by acquire */
        struct printk_buffers        *pbufs;
        u64                        seq;
};

/**
 * struct nbcon_write_context - Context handed to the nbcon write callbacks
 * @ctxt:                The core console context
 * @outbuf:                Pointer to the text buffer for output
 * @len:                Length to write
 * @unsafe_takeover:        If a hostile takeover in an unsafe state has occurred
 */
struct nbcon_write_context {
        struct nbcon_context        __private ctxt;
        char                        *outbuf;
        unsigned int                len;
        bool                        unsafe_takeover;
};

/**
 * struct console - The console descriptor structure
 * @name:                The name of the console driver
 * @write:                Legacy write callback to output messages (Optional)
 * @read:                Read callback for console input (Optional)
 * @device:                The underlying TTY device driver (Optional)
 * @unblank:                Callback to unblank the console (Optional)
 * @setup:                Callback for initializing the console (Optional)
 * @exit:                Callback for teardown of the console (Optional)
 * @match:                Callback for matching a console (Optional)
 * @flags:                Console flags. See enum cons_flags
 * @index:                Console index, e.g. port number
 * @cflag:                TTY control mode flags
 * @ispeed:                TTY input speed
 * @ospeed:                TTY output speed
 * @seq:                Sequence number of the next ringbuffer record to print
 * @dropped:                Number of unreported dropped ringbuffer records
 * @data:                Driver private data
 * @node:                hlist node for the console list
 *
 * @nbcon_state:        State for nbcon consoles
 * @nbcon_seq:                Sequence number of the next record for nbcon to print
 * @nbcon_device_ctxt:        Context available for non-printing operations
 * @nbcon_prev_seq:        Seq num the previous nbcon owner was assigned to print
 * @pbufs:                Pointer to nbcon private buffer
 * @kthread:                Printer kthread for this console
 * @rcuwait:                RCU-safe wait object for @kthread waking
 * @irq_work:                Defer @kthread waking to IRQ work context
 */
struct console {
        char                        name[16];
        void                        (*write)(struct console *co, const char *s, unsigned int count);
        int                        (*read)(struct console *co, char *s, unsigned int count);
        struct tty_driver        *(*device)(struct console *co, int *index);
        void                        (*unblank)(void);
        int                        (*setup)(struct console *co, char *options);
        int                        (*exit)(struct console *co);
        int                        (*match)(struct console *co, char *name, int idx, char *options);
        short                        flags;
        short                        index;
        int                        cflag;
        uint                        ispeed;
        uint                        ospeed;
        u64                        seq;
        unsigned long                dropped;
        void                        *data;
        struct hlist_node        node;

        /* nbcon console specific members */

        /**
         * @write_atomic:
         *
         * NBCON callback to write out text in any context. (Optional)
         *
         * This callback is called with the console already acquired. However,
         * a higher priority context is allowed to take it over by default.
         *
         * The callback must call nbcon_enter_unsafe() and nbcon_exit_unsafe()
         * around any code where the takeover is not safe, for example, when
         * manipulating the serial port registers.
         *
         * nbcon_enter_unsafe() will fail if the context has lost the console
         * ownership in the meantime. In this case, the callback is no longer
         * allowed to go forward. It must back out immediately and carefully.
         * The buffer content is also no longer trusted since it no longer
         * belongs to the context.
         *
         * The callback should allow the takeover whenever it is safe. It
         * increases the chance to see messages when the system is in trouble.
         * If the driver must reacquire ownership in order to finalize or
         * revert hardware changes, nbcon_reacquire_nobuf() can be used.
         * However, on reacquire the buffer content is no longer available. A
         * reacquire cannot be used to resume printing.
         *
         * The callback can be called from any context (including NMI).
         * Therefore it must avoid usage of any locking and instead rely
         * on the console ownership for synchronization.
         */
        void (*write_atomic)(struct console *con, struct nbcon_write_context *wctxt);

        /**
         * @write_thread:
         *
         * NBCON callback to write out text in task context.
         *
         * This callback must be called only in task context with both
         * device_lock() and the nbcon console acquired with
         * NBCON_PRIO_NORMAL.
         *
         * The same rules for console ownership verification and unsafe
         * sections handling applies as with write_atomic().
         *
         * The console ownership handling is necessary for synchronization
         * against write_atomic() which is synchronized only via the context.
         *
         * The device_lock() provides the primary serialization for operations
         * on the device. It might be as relaxed (mutex)[*] or as tight
         * (disabled preemption and interrupts) as needed. It allows
         * the kthread to operate in the least restrictive mode[**].
         *
         * [*] Standalone nbcon_context_try_acquire() is not safe with
         *     the preemption enabled, see nbcon_owner_matches(). But it
         *     can be safe when always called in the preemptive context
         *     under the device_lock().
         *
         * [**] The device_lock() makes sure that nbcon_context_try_acquire()
         *      would never need to spin which is important especially with
         *      PREEMPT_RT.
         */
        void (*write_thread)(struct console *con, struct nbcon_write_context *wctxt);

        /**
         * @device_lock:
         *
         * NBCON callback to begin synchronization with driver code.
         *
         * Console drivers typically must deal with access to the hardware
         * via user input/output (such as an interactive login shell) and
         * output of kernel messages via printk() calls. This callback is
         * called by the printk-subsystem whenever it needs to synchronize
         * with hardware access by the driver. It should be implemented to
         * use whatever synchronization mechanism the driver is using for
         * itself (for example, the port lock for uart serial consoles).
         *
         * The callback is always called from task context. It may use any
         * synchronization method required by the driver.
         *
         * IMPORTANT: The callback MUST disable migration. The console driver
         *        may be using a synchronization mechanism that already takes
         *        care of this (such as spinlocks). Otherwise this function must
         *        explicitly call migrate_disable().
         *
         * The flags argument is provided as a convenience to the driver. It
         * will be passed again to device_unlock(). It can be ignored if the
         * driver does not need it.
         */
        void (*device_lock)(struct console *con, unsigned long *flags);

        /**
         * @device_unlock:
         *
         * NBCON callback to finish synchronization with driver code.
         *
         * It is the counterpart to device_lock().
         *
         * This callback is always called from task context. It must
         * appropriately re-enable migration (depending on how device_lock()
         * disabled migration).
         *
         * The flags argument is the value of the same variable that was
         * passed to device_lock().
         */
        void (*device_unlock)(struct console *con, unsigned long flags);

        atomic_t                __private nbcon_state;
        atomic_long_t                __private nbcon_seq;
        struct nbcon_context        __private nbcon_device_ctxt;
        atomic_long_t           __private nbcon_prev_seq;

        struct printk_buffers        *pbufs;
        struct task_struct        *kthread;
        struct rcuwait                rcuwait;
        struct irq_work                irq_work;
};

#ifdef CONFIG_LOCKDEP
extern void lockdep_assert_console_list_lock_held(void);
#else
static inline void lockdep_assert_console_list_lock_held(void)
{
}
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern bool console_srcu_read_lock_is_held(void);
#else
static inline bool console_srcu_read_lock_is_held(void)
{
        return 1;
}
#endif

extern int console_srcu_read_lock(void);
extern void console_srcu_read_unlock(int cookie);

extern void console_list_lock(void) __acquires(console_mutex);
extern void console_list_unlock(void) __releases(console_mutex);

extern struct hlist_head console_list;

/**
 * console_srcu_read_flags - Locklessly read flags of a possibly registered
 *                                console
 * @con:        struct console pointer of console to read flags from
 *
 * Locklessly reading @con->flags provides a consistent read value because
 * there is at most one CPU modifying @con->flags and that CPU is using only
 * read-modify-write operations to do so.
 *
 * Requires console_srcu_read_lock to be held, which implies that @con might
 * be a registered console. The purpose of holding console_srcu_read_lock is
 * to guarantee that the console state is valid (CON_SUSPENDED/CON_ENABLED)
 * and that no exit/cleanup routines will run if the console is currently
 * undergoing unregistration.
 *
 * If the caller is holding the console_list_lock or it is _certain_ that
 * @con is not and will not become registered, the caller may read
 * @con->flags directly instead.
 *
 * Context: Any context.
 * Return: The current value of the @con->flags field.
 */
static inline short console_srcu_read_flags(const struct console *con)
{
        WARN_ON_ONCE(!console_srcu_read_lock_is_held());

        /*
         * The READ_ONCE() matches the WRITE_ONCE() when @flags are modified
         * for registered consoles with console_srcu_write_flags().
         */
        return data_race(READ_ONCE(con->flags));
}

/**
 * console_srcu_write_flags - Write flags for a registered console
 * @con:        struct console pointer of console to write flags to
 * @flags:        new flags value to write
 *
 * Only use this function to write flags for registered consoles. It
 * requires holding the console_list_lock.
 *
 * Context: Any context.
 */
static inline void console_srcu_write_flags(struct console *con, short flags)
{
        lockdep_assert_console_list_lock_held();

        /* This matches the READ_ONCE() in console_srcu_read_flags(). */
        WRITE_ONCE(con->flags, flags);
}

/* Variant of console_is_registered() when the console_list_lock is held. */
static inline bool console_is_registered_locked(const struct console *con)
{
        lockdep_assert_console_list_lock_held();
        return !hlist_unhashed(&con->node);
}

/*
 * console_is_registered - Check if the console is registered
 * @con:        struct console pointer of console to check
 *
 * Context: Process context. May sleep while acquiring console list lock.
 * Return: true if the console is in the console list, otherwise false.
 *
 * If false is returned for a console that was previously registered, it
 * can be assumed that the console's unregistration is fully completed,
 * including the exit() callback after console list removal.
 */
static inline bool console_is_registered(const struct console *con)
{
        bool ret;

        console_list_lock();
        ret = console_is_registered_locked(con);
        console_list_unlock();
        return ret;
}

/**
 * for_each_console_srcu() - Iterator over registered consoles
 * @con:        struct console pointer used as loop cursor
 *
 * Although SRCU guarantees the console list will be consistent, the
 * struct console fields may be updated by other CPUs while iterating.
 *
 * Requires console_srcu_read_lock to be held. Can be invoked from
 * any context.
 */
#define for_each_console_srcu(con)                                        \
        hlist_for_each_entry_srcu(con, &console_list, node,                \
                                  console_srcu_read_lock_is_held())

/**
 * for_each_console() - Iterator over registered consoles
 * @con:        struct console pointer used as loop cursor
 *
 * The console list and the &console.flags are immutable while iterating.
 *
 * Requires console_list_lock to be held.
 */
#define for_each_console(con)                                                \
        lockdep_assert_console_list_lock_held();                        \
        hlist_for_each_entry(con, &console_list, node)

#ifdef CONFIG_PRINTK
extern void nbcon_cpu_emergency_enter(void);
extern void nbcon_cpu_emergency_exit(void);
extern bool nbcon_can_proceed(struct nbcon_write_context *wctxt);
extern bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt);
extern bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt);
extern void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt);
#else
static inline void nbcon_cpu_emergency_enter(void) { }
static inline void nbcon_cpu_emergency_exit(void) { }
static inline bool nbcon_can_proceed(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt) { return false; }
static inline void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt) { }
#endif

extern int console_set_on_cmdline;
extern struct console *early_console;

enum con_flush_mode {
        CONSOLE_FLUSH_PENDING,
        CONSOLE_REPLAY_ALL,
};

extern int add_preferred_console(const char *name, const short idx, char *options);
extern void console_force_preferred_locked(struct console *con);
extern void register_console(struct console *);
extern int unregister_console(struct console *);
extern void console_lock(void);
extern int console_trylock(void);
extern void console_unlock(void);
extern void console_conditional_schedule(void);
extern void console_unblank(void);
extern void console_flush_on_panic(enum con_flush_mode mode);
extern struct tty_driver *console_device(int *);
extern void console_stop(struct console *);
extern void console_start(struct console *);
extern int is_console_locked(void);
extern int braille_register_console(struct console *, int index,
                char *console_options, char *braille_options);
extern int braille_unregister_console(struct console *);
#ifdef CONFIG_TTY
extern void console_sysfs_notify(void);
#else
static inline void console_sysfs_notify(void)
{ }
#endif
extern bool console_suspend_enabled;

/* Suspend and resume console messages over PM events */
extern void suspend_console(void);
extern void resume_console(void);

int mda_console_init(void);

void vcs_make_sysfs(int index);
void vcs_remove_sysfs(int index);

/* Some debug stub to catch some of the obvious races in the VT code */
#define WARN_CONSOLE_UNLOCKED()                                                \
        WARN_ON(!atomic_read(&ignore_console_lock_warning) &&                \
                !is_console_locked() && !oops_in_progress)
/*
 * Increment ignore_console_lock_warning if you need to quiet
 * WARN_CONSOLE_UNLOCKED() for debugging purposes.
 */
extern atomic_t ignore_console_lock_warning;

extern void console_init(void);

/* For deferred console takeover */
void dummycon_register_output_notifier(struct notifier_block *nb);
void dummycon_unregister_output_notifier(struct notifier_block *nb);

#endif /* _LINUX_CONSOLE_H */


























   22 













   12 


























    1 













  851 





  849 






  377 























































  851 
  735 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _LINUX_RCUREF_H
#define _LINUX_RCUREF_H

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/limits.h>
#include <linux/lockdep.h>
#include <linux/preempt.h>
#include <linux/rcupdate.h>

#define RCUREF_ONEREF                0x00000000U
#define RCUREF_MAXREF                0x7FFFFFFFU
#define RCUREF_SATURATED        0xA0000000U
#define RCUREF_RELEASED                0xC0000000U
#define RCUREF_DEAD                0xE0000000U
#define RCUREF_NOREF                0xFFFFFFFFU

/**
 * rcuref_init - Initialize a rcuref reference count with the given reference count
 * @ref:        Pointer to the reference count
 * @cnt:        The initial reference count typically '1'
 */
static inline void rcuref_init(rcuref_t *ref, unsigned int cnt)
{
        atomic_set(&ref->refcnt, cnt - 1);
}

/**
 * rcuref_read - Read the number of held reference counts of a rcuref
 * @ref:        Pointer to the reference count
 *
 * Return: The number of held references (0 ... N)
 */
static inline unsigned int rcuref_read(rcuref_t *ref)
{
        unsigned int c = atomic_read(&ref->refcnt);

        /* Return 0 if within the DEAD zone. */
        return c >= RCUREF_RELEASED ? 0 : c + 1;
}

extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);

/**
 * rcuref_get - Acquire one reference on a rcuref reference count
 * @ref:        Pointer to the reference count
 *
 * Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF.
 *
 * Provides no memory ordering, it is assumed the caller has guaranteed the
 * object memory to be stable (RCU, etc.). It does provide a control dependency
 * and thereby orders future stores. See documentation in lib/rcuref.c
 *
 * Return:
 *        False if the attempt to acquire a reference failed. This happens
 *        when the last reference has been put already
 *
 *        True if a reference was successfully acquired
 */
static inline __must_check bool rcuref_get(rcuref_t *ref)
{
        /*
         * Unconditionally increase the reference count. The saturation and
         * dead zones provide enough tolerance for this.
         */
        if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt)))
                return true;

        /* Handle the cases inside the saturation and dead zones */
        return rcuref_get_slowpath(ref);
}

extern __must_check bool rcuref_put_slowpath(rcuref_t *ref);

/*
 * Internal helper. Do not invoke directly.
 */
static __always_inline __must_check bool __rcuref_put(rcuref_t *ref)
{
        RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(),
                         "suspicious rcuref_put_rcusafe() usage");
        /*
         * Unconditionally decrease the reference count. The saturation and
         * dead zones provide enough tolerance for this.
         */
        if (likely(!atomic_add_negative_release(-1, &ref->refcnt)))
                return false;

        /*
         * Handle the last reference drop and cases inside the saturation
         * and dead zones.
         */
        return rcuref_put_slowpath(ref);
}

/**
 * rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe
 * @ref:        Pointer to the reference count
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides an acquire ordering on success such that free()
 * must come after.
 *
 * Can be invoked from contexts, which guarantee that no grace period can
 * happen which would free the object concurrently if the decrement drops
 * the last reference and the slowpath races against a concurrent get() and
 * put() pair. rcu_read_lock()'ed and atomic contexts qualify.
 *
 * Return:
 *        True if this was the last reference with no future references
 *        possible. This signals the caller that it can safely release the
 *        object which is protected by the reference counter.
 *
 *        False if there are still active references or the put() raced
 *        with a concurrent get()/put() pair. Caller is not allowed to
 *        release the protected object.
 */
static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref)
{
        return __rcuref_put(ref);
}

/**
 * rcuref_put -- Release one reference for a rcuref reference count
 * @ref:        Pointer to the reference count
 *
 * Can be invoked from any context.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides an acquire ordering on success such that free()
 * must come after.
 *
 * Return:
 *
 *        True if this was the last reference with no future references
 *        possible. This signals the caller that it can safely schedule the
 *        object, which is protected by the reference counter, for
 *        deconstruction.
 *
 *        False if there are still active references or the put() raced
 *        with a concurrent get()/put() pair. Caller is not allowed to
 *        deconstruct the protected object.
 */
static inline __must_check bool rcuref_put(rcuref_t *ref)
{
        bool released;

        preempt_disable();
        released = __rcuref_put(ref);
        preempt_enable();
        return released;
}

#endif






















   18 












































   18 



   18 



































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_VIRTIO_VSOCK_H
#define _LINUX_VIRTIO_VSOCK_H

#include <uapi/linux/virtio_vsock.h>
#include <linux/socket.h>
#include <net/sock.h>
#include <net/af_vsock.h>

#define VIRTIO_VSOCK_SKB_HEADROOM (sizeof(struct virtio_vsock_hdr))

struct virtio_vsock_skb_cb {
        bool reply;
        bool tap_delivered;
        u32 offset;
};

#define VIRTIO_VSOCK_SKB_CB(skb) ((struct virtio_vsock_skb_cb *)((skb)->cb))

static inline struct virtio_vsock_hdr *virtio_vsock_hdr(struct sk_buff *skb)
{
        return (struct virtio_vsock_hdr *)skb->head;
}

static inline bool virtio_vsock_skb_reply(struct sk_buff *skb)
{
        return VIRTIO_VSOCK_SKB_CB(skb)->reply;
}

static inline void virtio_vsock_skb_set_reply(struct sk_buff *skb)
{
        VIRTIO_VSOCK_SKB_CB(skb)->reply = true;
}

static inline bool virtio_vsock_skb_tap_delivered(struct sk_buff *skb)
{
        return VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered;
}

static inline void virtio_vsock_skb_set_tap_delivered(struct sk_buff *skb)
{
        VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered = true;
}

static inline void virtio_vsock_skb_clear_tap_delivered(struct sk_buff *skb)
{
        VIRTIO_VSOCK_SKB_CB(skb)->tap_delivered = false;
}

static inline void virtio_vsock_skb_rx_put(struct sk_buff *skb)
{
        u32 len;

        len = le32_to_cpu(virtio_vsock_hdr(skb)->len);

        if (len > 0)
                skb_put(skb, len);
}

static inline struct sk_buff *virtio_vsock_alloc_skb(unsigned int size, gfp_t mask)
{
        struct sk_buff *skb;

        if (size < VIRTIO_VSOCK_SKB_HEADROOM)
                return NULL;

        skb = alloc_skb(size, mask);
        if (!skb)
                return NULL;

        skb_reserve(skb, VIRTIO_VSOCK_SKB_HEADROOM);
        return skb;
}

static inline void
virtio_vsock_skb_queue_head(struct sk_buff_head *list, struct sk_buff *skb)
{
        spin_lock_bh(&list->lock);
        __skb_queue_head(list, skb);
        spin_unlock_bh(&list->lock);
}

static inline void
virtio_vsock_skb_queue_tail(struct sk_buff_head *list, struct sk_buff *skb)
{
        spin_lock_bh(&list->lock);
        __skb_queue_tail(list, skb);
        spin_unlock_bh(&list->lock);
}

static inline struct sk_buff *virtio_vsock_skb_dequeue(struct sk_buff_head *list)
{
        struct sk_buff *skb;

        spin_lock_bh(&list->lock);
        skb = __skb_dequeue(list);
        spin_unlock_bh(&list->lock);

        return skb;
}

static inline void virtio_vsock_skb_queue_purge(struct sk_buff_head *list)
{
        spin_lock_bh(&list->lock);
        __skb_queue_purge(list);
        spin_unlock_bh(&list->lock);
}

static inline size_t virtio_vsock_skb_len(struct sk_buff *skb)
{
        return (size_t)(skb_end_pointer(skb) - skb->head);
}

#define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE        (1024 * 4)
#define VIRTIO_VSOCK_MAX_BUF_SIZE                0xFFFFFFFFUL
#define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE                (1024 * 64)

enum {
        VSOCK_VQ_RX     = 0, /* for host to guest data */
        VSOCK_VQ_TX     = 1, /* for guest to host data */
        VSOCK_VQ_EVENT  = 2,
        VSOCK_VQ_MAX    = 3,
};

/* Per-socket state (accessed via vsk->trans) */
struct virtio_vsock_sock {
        struct vsock_sock *vsk;

        spinlock_t tx_lock;
        spinlock_t rx_lock;

        /* Protected by tx_lock */
        u32 tx_cnt;
        u32 peer_fwd_cnt;
        u32 peer_buf_alloc;
        size_t bytes_unsent;

        /* Protected by rx_lock */
        u32 fwd_cnt;
        u32 last_fwd_cnt;
        u32 rx_bytes;
        u32 buf_alloc;
        struct sk_buff_head rx_queue;
        u32 msg_count;
};

struct virtio_vsock_pkt_info {
        u32 remote_cid, remote_port;
        struct vsock_sock *vsk;
        struct msghdr *msg;
        u32 pkt_len;
        u16 type;
        u16 op;
        u32 flags;
        bool reply;
};

struct virtio_transport {
        /* This must be the first field */
        struct vsock_transport transport;

        /* Takes ownership of the packet */
        int (*send_pkt)(struct sk_buff *skb);

        /* Used in MSG_ZEROCOPY mode. Checks, that provided data
         * (number of buffers) could be transmitted with zerocopy
         * mode. If this callback is not implemented for the current
         * transport - this means that this transport doesn't need
         * extra checks and can perform zerocopy transmission by
         * default.
         */
        bool (*can_msgzerocopy)(int bufs_num);
};

ssize_t
virtio_transport_stream_dequeue(struct vsock_sock *vsk,
                                struct msghdr *msg,
                                size_t len,
                                int type);
int
virtio_transport_dgram_dequeue(struct vsock_sock *vsk,
                               struct msghdr *msg,
                               size_t len, int flags);

int
virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk,
                                   struct msghdr *msg,
                                   size_t len);
ssize_t
virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk,
                                   struct msghdr *msg,
                                   int flags);
s64 virtio_transport_stream_has_data(struct vsock_sock *vsk);
s64 virtio_transport_stream_has_space(struct vsock_sock *vsk);
u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk);

ssize_t virtio_transport_unsent_bytes(struct vsock_sock *vsk);

void virtio_transport_consume_skb_sent(struct sk_buff *skb,
                                       bool consume);

int virtio_transport_do_socket_init(struct vsock_sock *vsk,
                                 struct vsock_sock *psk);
int
virtio_transport_notify_poll_in(struct vsock_sock *vsk,
                                size_t target,
                                bool *data_ready_now);
int
virtio_transport_notify_poll_out(struct vsock_sock *vsk,
                                 size_t target,
                                 bool *space_available_now);

int virtio_transport_notify_recv_init(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data);
int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data);
int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk,
        size_t target, struct vsock_transport_recv_notify_data *data);
int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk,
        size_t target, ssize_t copied, bool data_read,
        struct vsock_transport_recv_notify_data *data);
int virtio_transport_notify_send_init(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data);
int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data);
int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk,
        struct vsock_transport_send_notify_data *data);
int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk,
        ssize_t written, struct vsock_transport_send_notify_data *data);
void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val);

u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk);
bool virtio_transport_stream_is_active(struct vsock_sock *vsk);
bool virtio_transport_stream_allow(u32 cid, u32 port);
int virtio_transport_dgram_bind(struct vsock_sock *vsk,
                                struct sockaddr_vm *addr);
bool virtio_transport_dgram_allow(u32 cid, u32 port);

int virtio_transport_connect(struct vsock_sock *vsk);

int virtio_transport_shutdown(struct vsock_sock *vsk, int mode);

void virtio_transport_release(struct vsock_sock *vsk);

ssize_t
virtio_transport_stream_enqueue(struct vsock_sock *vsk,
                                struct msghdr *msg,
                                size_t len);
int
virtio_transport_dgram_enqueue(struct vsock_sock *vsk,
                               struct sockaddr_vm *remote_addr,
                               struct msghdr *msg,
                               size_t len);

void virtio_transport_destruct(struct vsock_sock *vsk);

void virtio_transport_recv_pkt(struct virtio_transport *t,
                               struct sk_buff *skb);
void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb);
u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 wanted);
void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit);
void virtio_transport_deliver_tap_pkt(struct sk_buff *skb);
int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list);
int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t read_actor);
int virtio_transport_notify_set_rcvlowat(struct vsock_sock *vsk, int val);
#endif /* _LINUX_VIRTIO_VSOCK_H */












    5 
  446 

  442 
  372 



  447 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_TLB_H
#define _ASM_X86_TLB_H

#define tlb_flush tlb_flush
static inline void tlb_flush(struct mmu_gather *tlb);

#include <asm-generic/tlb.h>

static inline void tlb_flush(struct mmu_gather *tlb)
{
        unsigned long start = 0UL, end = TLB_FLUSH_ALL;
        unsigned int stride_shift = tlb_get_unmap_shift(tlb);

        if (!tlb->fullmm && !tlb->need_flush_all) {
                start = tlb->start;
                end = tlb->end;
        }

        flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables);
}

/*
 * While x86 architecture in general requires an IPI to perform TLB
 * shootdown, enablement code for several hypervisors overrides
 * .flush_tlb_others hook in pv_mmu_ops and implements it by issuing
 * a hypercall. To keep software pagetable walkers safe in this case we
 * switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the comment
 * below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h
 * for more details.
 */
static inline void __tlb_remove_table(void *table)
{
        free_page_and_swap_cache(table);
}

#endif /* _ASM_X86_TLB_H */




























































    5 


    5 
    5 







    5 









    5 
    5 





































































    5 
    5 
    5 

    5 




    5 
    5 

















































































































    5 


    5 




    5 


    5 

    5 


    5 










    5 








    5 




    5 





















































    5 


    5 
    5 
    5 

    5 





    5 













    5 



    5 











    5 


    4 


    5 
    5 

    5 

    5 







    5 








    5 










    5 








    5 

    5 





    5 
    5 



















    5 

    5 
    5 

    5 


    5 
    5 

    5 




















































    5 



    5 

    5 
    5 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/fs/ext4/page-io.c
 *
 * This contains the new page_io functions for ext4
 *
 * Written by Theodore Ts'o, 2010.
 */

#include <linux/fs.h>
#include <linux/time.h>
#include <linux/highuid.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/string.h>
#include <linux/buffer_head.h>
#include <linux/writeback.h>
#include <linux/pagevec.h>
#include <linux/mpage.h>
#include <linux/namei.h>
#include <linux/uio.h>
#include <linux/bio.h>
#include <linux/workqueue.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>

#include "ext4_jbd2.h"
#include "xattr.h"
#include "acl.h"

static struct kmem_cache *io_end_cachep;
static struct kmem_cache *io_end_vec_cachep;

int __init ext4_init_pageio(void)
{
        io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT);
        if (io_end_cachep == NULL)
                return -ENOMEM;

        io_end_vec_cachep = KMEM_CACHE(ext4_io_end_vec, 0);
        if (io_end_vec_cachep == NULL) {
                kmem_cache_destroy(io_end_cachep);
                return -ENOMEM;
        }
        return 0;
}

void ext4_exit_pageio(void)
{
        kmem_cache_destroy(io_end_cachep);
        kmem_cache_destroy(io_end_vec_cachep);
}

struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end)
{
        struct ext4_io_end_vec *io_end_vec;

        io_end_vec = kmem_cache_zalloc(io_end_vec_cachep, GFP_NOFS);
        if (!io_end_vec)
                return ERR_PTR(-ENOMEM);
        INIT_LIST_HEAD(&io_end_vec->list);
        list_add_tail(&io_end_vec->list, &io_end->list_vec);
        return io_end_vec;
}

static void ext4_free_io_end_vec(ext4_io_end_t *io_end)
{
        struct ext4_io_end_vec *io_end_vec, *tmp;

        if (list_empty(&io_end->list_vec))
                return;
        list_for_each_entry_safe(io_end_vec, tmp, &io_end->list_vec, list) {
                list_del(&io_end_vec->list);
                kmem_cache_free(io_end_vec_cachep, io_end_vec);
        }
}

struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end)
{
        BUG_ON(list_empty(&io_end->list_vec));
        return list_last_entry(&io_end->list_vec, struct ext4_io_end_vec, list);
}

/*
 * Print an buffer I/O error compatible with the fs/buffer.c.  This
 * provides compatibility with dmesg scrapers that look for a specific
 * buffer I/O error message.  We really need a unified error reporting
 * structure to userspace ala Digital Unix's uerf system, but it's
 * probably not going to happen in my lifetime, due to LKML politics...
 */
static void buffer_io_error(struct buffer_head *bh)
{
        printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n",
                       bh->b_bdev,
                        (unsigned long long)bh->b_blocknr);
}

static void ext4_finish_bio(struct bio *bio)
{
        struct folio_iter fi;

        bio_for_each_folio_all(fi, bio) {
                struct folio *folio = fi.folio;
                struct folio *io_folio = NULL;
                struct buffer_head *bh, *head;
                size_t bio_start = fi.offset;
                size_t bio_end = bio_start + fi.length;
                unsigned under_io = 0;
                unsigned long flags;

                if (fscrypt_is_bounce_folio(folio)) {
                        io_folio = folio;
                        folio = fscrypt_pagecache_folio(folio);
                }

                if (bio->bi_status) {
                        int err = blk_status_to_errno(bio->bi_status);
                        mapping_set_error(folio->mapping, err);
                }
                bh = head = folio_buffers(folio);
                /*
                 * We check all buffers in the folio under b_uptodate_lock
                 * to avoid races with other end io clearing async_write flags
                 */
                spin_lock_irqsave(&head->b_uptodate_lock, flags);
                do {
                        if (bh_offset(bh) < bio_start ||
                            bh_offset(bh) + bh->b_size > bio_end) {
                                if (buffer_async_write(bh))
                                        under_io++;
                                continue;
                        }
                        clear_buffer_async_write(bh);
                        if (bio->bi_status) {
                                set_buffer_write_io_error(bh);
                                buffer_io_error(bh);
                        }
                } while ((bh = bh->b_this_page) != head);
                spin_unlock_irqrestore(&head->b_uptodate_lock, flags);
                if (!under_io) {
                        fscrypt_free_bounce_page(&io_folio->page);
                        folio_end_writeback(folio);
                }
        }
}

static void ext4_release_io_end(ext4_io_end_t *io_end)
{
        struct bio *bio, *next_bio;

        BUG_ON(!list_empty(&io_end->list));
        BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
        WARN_ON(io_end->handle);

        for (bio = io_end->bio; bio; bio = next_bio) {
                next_bio = bio->bi_private;
                ext4_finish_bio(bio);
                bio_put(bio);
        }
        ext4_free_io_end_vec(io_end);
        kmem_cache_free(io_end_cachep, io_end);
}

/*
 * Check a range of space and convert unwritten extents to written. Note that
 * we are protected from truncate touching same part of extent tree by the
 * fact that truncate code waits for all DIO to finish (thus exclusion from
 * direct IO is achieved) and also waits for PageWriteback bits. Thus we
 * cannot get to ext4_ext_truncate() before all IOs overlapping that range are
 * completed (happens from ext4_free_ioend()).
 */
static int ext4_end_io_end(ext4_io_end_t *io_end)
{
        struct inode *inode = io_end->inode;
        handle_t *handle = io_end->handle;
        int ret = 0;

        ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %lu,list->next 0x%p,"
                   "list->prev 0x%p\n",
                   io_end, inode->i_ino, io_end->list.next, io_end->list.prev);

        io_end->handle = NULL;        /* Following call will use up the handle */
        ret = ext4_convert_unwritten_io_end_vec(handle, io_end);
        if (ret < 0 && !ext4_forced_shutdown(inode->i_sb)) {
                ext4_msg(inode->i_sb, KERN_EMERG,
                         "failed to convert unwritten extents to written "
                         "extents -- potential data loss!  "
                         "(inode %lu, error %d)", inode->i_ino, ret);
        }
        ext4_clear_io_unwritten_flag(io_end);
        ext4_release_io_end(io_end);
        return ret;
}

static void dump_completed_IO(struct inode *inode, struct list_head *head)
{
#ifdef        EXT4FS_DEBUG
        struct list_head *cur, *before, *after;
        ext4_io_end_t *io_end, *io_end0, *io_end1;

        if (list_empty(head))
                return;

        ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
        list_for_each_entry(io_end, head, list) {
                cur = &io_end->list;
                before = cur->prev;
                io_end0 = container_of(before, ext4_io_end_t, list);
                after = cur->next;
                io_end1 = container_of(after, ext4_io_end_t, list);

                ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
                            io_end, inode->i_ino, io_end0, io_end1);
        }
#endif
}

/* Add the io_end to per-inode completed end_io list. */
static void ext4_add_complete_io(ext4_io_end_t *io_end)
{
        struct ext4_inode_info *ei = EXT4_I(io_end->inode);
        struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);
        struct workqueue_struct *wq;
        unsigned long flags;

        /* Only reserved conversions from writeback should enter here */
        WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
        WARN_ON(!io_end->handle && sbi->s_journal);
        spin_lock_irqsave(&ei->i_completed_io_lock, flags);
        wq = sbi->rsv_conversion_wq;
        if (list_empty(&ei->i_rsv_conversion_list))
                queue_work(wq, &ei->i_rsv_conversion_work);
        list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
        spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
}

static int ext4_do_flush_completed_IO(struct inode *inode,
                                      struct list_head *head)
{
        ext4_io_end_t *io_end;
        struct list_head unwritten;
        unsigned long flags;
        struct ext4_inode_info *ei = EXT4_I(inode);
        int err, ret = 0;

        spin_lock_irqsave(&ei->i_completed_io_lock, flags);
        dump_completed_IO(inode, head);
        list_replace_init(head, &unwritten);
        spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);

        while (!list_empty(&unwritten)) {
                io_end = list_entry(unwritten.next, ext4_io_end_t, list);
                BUG_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
                list_del_init(&io_end->list);

                err = ext4_end_io_end(io_end);
                if (unlikely(!ret && err))
                        ret = err;
        }
        return ret;
}

/*
 * work on completed IO, to convert unwritten extents to extents
 */
void ext4_end_io_rsv_work(struct work_struct *work)
{
        struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
                                                  i_rsv_conversion_work);
        ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
}

ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags)
{
        ext4_io_end_t *io_end = kmem_cache_zalloc(io_end_cachep, flags);

        if (io_end) {
                io_end->inode = inode;
                INIT_LIST_HEAD(&io_end->list);
                INIT_LIST_HEAD(&io_end->list_vec);
                refcount_set(&io_end->count, 1);
        }
        return io_end;
}

void ext4_put_io_end_defer(ext4_io_end_t *io_end)
{
        if (refcount_dec_and_test(&io_end->count)) {
                if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) ||
                                list_empty(&io_end->list_vec)) {
                        ext4_release_io_end(io_end);
                        return;
                }
                ext4_add_complete_io(io_end);
        }
}

int ext4_put_io_end(ext4_io_end_t *io_end)
{
        int err = 0;

        if (refcount_dec_and_test(&io_end->count)) {
                if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
                        err = ext4_convert_unwritten_io_end_vec(io_end->handle,
                                                                io_end);
                        io_end->handle = NULL;
                        ext4_clear_io_unwritten_flag(io_end);
                }
                ext4_release_io_end(io_end);
        }
        return err;
}

ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
{
        refcount_inc(&io_end->count);
        return io_end;
}

/* BIO completion function for page writeback */
static void ext4_end_bio(struct bio *bio)
{
        ext4_io_end_t *io_end = bio->bi_private;
        sector_t bi_sector = bio->bi_iter.bi_sector;

        if (WARN_ONCE(!io_end, "io_end is NULL: %pg: sector %Lu len %u err %d\n",
                      bio->bi_bdev,
                      (long long) bio->bi_iter.bi_sector,
                      (unsigned) bio_sectors(bio),
                      bio->bi_status)) {
                ext4_finish_bio(bio);
                bio_put(bio);
                return;
        }
        bio->bi_end_io = NULL;

        if (bio->bi_status) {
                struct inode *inode = io_end->inode;

                ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
                             "starting block %llu)",
                             bio->bi_status, inode->i_ino,
                             (unsigned long long)
                             bi_sector >> (inode->i_blkbits - 9));
                mapping_set_error(inode->i_mapping,
                                blk_status_to_errno(bio->bi_status));
        }

        if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
                /*
                 * Link bio into list hanging from io_end. We have to do it
                 * atomically as bio completions can be racing against each
                 * other.
                 */
                bio->bi_private = xchg(&io_end->bio, bio);
                ext4_put_io_end_defer(io_end);
        } else {
                /*
                 * Drop io_end reference early. Inode can get freed once
                 * we finish the bio.
                 */
                ext4_put_io_end_defer(io_end);
                ext4_finish_bio(bio);
                bio_put(bio);
        }
}

void ext4_io_submit(struct ext4_io_submit *io)
{
        struct bio *bio = io->io_bio;

        if (bio) {
                if (io->io_wbc->sync_mode == WB_SYNC_ALL)
                        io->io_bio->bi_opf |= REQ_SYNC;
                submit_bio(io->io_bio);
        }
        io->io_bio = NULL;
}

void ext4_io_submit_init(struct ext4_io_submit *io,
                         struct writeback_control *wbc)
{
        io->io_wbc = wbc;
        io->io_bio = NULL;
        io->io_end = NULL;
}

static void io_submit_init_bio(struct ext4_io_submit *io,
                               struct buffer_head *bh)
{
        struct bio *bio;

        /*
         * bio_alloc will _always_ be able to allocate a bio if
         * __GFP_DIRECT_RECLAIM is set, see comments for bio_alloc_bioset().
         */
        bio = bio_alloc(bh->b_bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOIO);
        fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
        bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
        bio->bi_end_io = ext4_end_bio;
        bio->bi_private = ext4_get_io_end(io->io_end);
        io->io_bio = bio;
        io->io_next_block = bh->b_blocknr;
        wbc_init_bio(io->io_wbc, bio);
}

static void io_submit_add_bh(struct ext4_io_submit *io,
                             struct inode *inode,
                             struct folio *folio,
                             struct folio *io_folio,
                             struct buffer_head *bh)
{
        if (io->io_bio && (bh->b_blocknr != io->io_next_block ||
                           !fscrypt_mergeable_bio_bh(io->io_bio, bh))) {
submit_and_retry:
                ext4_io_submit(io);
        }
        if (io->io_bio == NULL) {
                io_submit_init_bio(io, bh);
                io->io_bio->bi_write_hint = inode->i_write_hint;
        }
        if (!bio_add_folio(io->io_bio, io_folio, bh->b_size, bh_offset(bh)))
                goto submit_and_retry;
        wbc_account_cgroup_owner(io->io_wbc, folio, bh->b_size);
        io->io_next_block++;
}

int ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio,
                size_t len)
{
        struct folio *io_folio = folio;
        struct inode *inode = folio->mapping->host;
        unsigned block_start;
        struct buffer_head *bh, *head;
        int ret = 0;
        int nr_to_submit = 0;
        struct writeback_control *wbc = io->io_wbc;
        bool keep_towrite = false;

        BUG_ON(!folio_test_locked(folio));
        BUG_ON(folio_test_writeback(folio));

        /*
         * Comments copied from block_write_full_folio:
         *
         * The folio straddles i_size.  It must be zeroed out on each and every
         * writepage invocation because it may be mmapped.  "A file is mapped
         * in multiples of the page size.  For a file that is not a multiple of
         * the page size, the remaining memory is zeroed when mapped, and
         * writes to that region are not written out to the file."
         */
        if (len < folio_size(folio))
                folio_zero_segment(folio, len, folio_size(folio));
        /*
         * In the first loop we prepare and mark buffers to submit. We have to
         * mark all buffers in the folio before submitting so that
         * folio_end_writeback() cannot be called from ext4_end_bio() when IO
         * on the first buffer finishes and we are still working on submitting
         * the second buffer.
         */
        bh = head = folio_buffers(folio);
        do {
                block_start = bh_offset(bh);
                if (block_start >= len) {
                        clear_buffer_dirty(bh);
                        set_buffer_uptodate(bh);
                        continue;
                }
                if (!buffer_dirty(bh) || buffer_delay(bh) ||
                    !buffer_mapped(bh) || buffer_unwritten(bh)) {
                        /* A hole? We can safely clear the dirty bit */
                        if (!buffer_mapped(bh))
                                clear_buffer_dirty(bh);
                        /*
                         * Keeping dirty some buffer we cannot write? Make sure
                         * to redirty the folio and keep TOWRITE tag so that
                         * racing WB_SYNC_ALL writeback does not skip the folio.
                         * This happens e.g. when doing writeout for
                         * transaction commit or when journalled data is not
                         * yet committed.
                         */
                        if (buffer_dirty(bh) ||
                            (buffer_jbd(bh) && buffer_jbddirty(bh))) {
                                if (!folio_test_dirty(folio))
                                        folio_redirty_for_writepage(wbc, folio);
                                keep_towrite = true;
                        }
                        continue;
                }
                if (buffer_new(bh))
                        clear_buffer_new(bh);
                set_buffer_async_write(bh);
                clear_buffer_dirty(bh);
                nr_to_submit++;
        } while ((bh = bh->b_this_page) != head);

        /* Nothing to submit? Just unlock the folio... */
        if (!nr_to_submit)
                return 0;

        bh = head = folio_buffers(folio);

        /*
         * If any blocks are being written to an encrypted file, encrypt them
         * into a bounce page.  For simplicity, just encrypt until the last
         * block which might be needed.  This may cause some unneeded blocks
         * (e.g. holes) to be unnecessarily encrypted, but this is rare and
         * can't happen in the common case of blocksize == PAGE_SIZE.
         */
        if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
                gfp_t gfp_flags = GFP_NOFS;
                unsigned int enc_bytes = round_up(len, i_blocksize(inode));
                struct page *bounce_page;

                /*
                 * Since bounce page allocation uses a mempool, we can only use
                 * a waiting mask (i.e. request guaranteed allocation) on the
                 * first page of the bio.  Otherwise it can deadlock.
                 */
                if (io->io_bio)
                        gfp_flags = GFP_NOWAIT | __GFP_NOWARN;
        retry_encrypt:
                bounce_page = fscrypt_encrypt_pagecache_blocks(&folio->page,
                                        enc_bytes, 0, gfp_flags);
                if (IS_ERR(bounce_page)) {
                        ret = PTR_ERR(bounce_page);
                        if (ret == -ENOMEM &&
                            (io->io_bio || wbc->sync_mode == WB_SYNC_ALL)) {
                                gfp_t new_gfp_flags = GFP_NOFS;
                                if (io->io_bio)
                                        ext4_io_submit(io);
                                else
                                        new_gfp_flags |= __GFP_NOFAIL;
                                memalloc_retry_wait(gfp_flags);
                                gfp_flags = new_gfp_flags;
                                goto retry_encrypt;
                        }

                        printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
                        folio_redirty_for_writepage(wbc, folio);
                        do {
                                if (buffer_async_write(bh)) {
                                        clear_buffer_async_write(bh);
                                        set_buffer_dirty(bh);
                                }
                                bh = bh->b_this_page;
                        } while (bh != head);

                        return ret;
                }
                io_folio = page_folio(bounce_page);
        }

        __folio_start_writeback(folio, keep_towrite);

        /* Now submit buffers to write */
        do {
                if (!buffer_async_write(bh))
                        continue;
                io_submit_add_bh(io, inode, folio, io_folio, bh);
        } while ((bh = bh->b_this_page) != head);

        return 0;
}



















































































    4 


    3 














    4 




    4 


    4 

    4 




    4 


    4 



    4 



    4 



    4 

    3 
    4 


































    4 





    4 








    4 

    4 
    3 






    4 
    4 










































    4 






    1 







    4 
    1 




    9 











    8 

    6 
    8 


    6 


    6 






    2 


    1 


    3 















































    9 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/nospec.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "tctx.h"
#include "poll.h"
#include "timeout.h"
#include "waitid.h"
#include "futex.h"
#include "cancel.h"

struct io_cancel {
        struct file                        *file;
        u64                                addr;
        u32                                flags;
        s32                                fd;
        u8                                opcode;
};

#define CANCEL_FLAGS        (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
                         IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
                         IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)

/*
 * Returns true if the request matches the criteria outlined by 'cd'.
 */
bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)
{
        bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA;

        if (req->ctx != cd->ctx)
                return false;

        if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)))
                match_user_data = true;

        if (cd->flags & IORING_ASYNC_CANCEL_ANY)
                goto check_seq;
        if (cd->flags & IORING_ASYNC_CANCEL_FD) {
                if (req->file != cd->file)
                        return false;
        }
        if (cd->flags & IORING_ASYNC_CANCEL_OP) {
                if (req->opcode != cd->opcode)
                        return false;
        }
        if (match_user_data && req->cqe.user_data != cd->data)
                return false;
        if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
check_seq:
                if (io_cancel_match_sequence(req, cd->seq))
                        return false;
        }

        return true;
}

static bool io_cancel_cb(struct io_wq_work *work, void *data)
{
        struct io_kiocb *req = container_of(work, struct io_kiocb, work);
        struct io_cancel_data *cd = data;

        return io_cancel_req_match(req, cd);
}

static int io_async_cancel_one(struct io_uring_task *tctx,
                               struct io_cancel_data *cd)
{
        enum io_wq_cancel cancel_ret;
        int ret = 0;
        bool all;

        if (!tctx || !tctx->io_wq)
                return -ENOENT;

        all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
        cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
        switch (cancel_ret) {
        case IO_WQ_CANCEL_OK:
                ret = 0;
                break;
        case IO_WQ_CANCEL_RUNNING:
                ret = -EALREADY;
                break;
        case IO_WQ_CANCEL_NOTFOUND:
                ret = -ENOENT;
                break;
        }

        return ret;
}

int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
                  unsigned issue_flags)
{
        struct io_ring_ctx *ctx = cd->ctx;
        int ret;

        WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring);

        ret = io_async_cancel_one(tctx, cd);
        /*
         * Fall-through even for -EALREADY, as we may have poll armed
         * that need unarming.
         */
        if (!ret)
                return 0;

        ret = io_poll_cancel(ctx, cd, issue_flags);
        if (ret != -ENOENT)
                return ret;

        ret = io_waitid_cancel(ctx, cd, issue_flags);
        if (ret != -ENOENT)
                return ret;

        ret = io_futex_cancel(ctx, cd, issue_flags);
        if (ret != -ENOENT)
                return ret;

        spin_lock(&ctx->completion_lock);
        if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
                ret = io_timeout_cancel(ctx, cd);
        spin_unlock(&ctx->completion_lock);
        return ret;
}

int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
        struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);

        if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
                return -EINVAL;
        if (sqe->off || sqe->splice_fd_in)
                return -EINVAL;

        cancel->addr = READ_ONCE(sqe->addr);
        cancel->flags = READ_ONCE(sqe->cancel_flags);
        if (cancel->flags & ~CANCEL_FLAGS)
                return -EINVAL;
        if (cancel->flags & IORING_ASYNC_CANCEL_FD) {
                if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
                        return -EINVAL;
                cancel->fd = READ_ONCE(sqe->fd);
        }
        if (cancel->flags & IORING_ASYNC_CANCEL_OP) {
                if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
                        return -EINVAL;
                cancel->opcode = READ_ONCE(sqe->len);
        }

        return 0;
}

static int __io_async_cancel(struct io_cancel_data *cd,
                             struct io_uring_task *tctx,
                             unsigned int issue_flags)
{
        bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
        struct io_ring_ctx *ctx = cd->ctx;
        struct io_tctx_node *node;
        int ret, nr = 0;

        do {
                ret = io_try_cancel(tctx, cd, issue_flags);
                if (ret == -ENOENT)
                        break;
                if (!all)
                        return ret;
                nr++;
        } while (1);

        /* slow path, try all io-wq's */
        io_ring_submit_lock(ctx, issue_flags);
        ret = -ENOENT;
        list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
                ret = io_async_cancel_one(node->task->io_uring, cd);
                if (ret != -ENOENT) {
                        if (!all)
                                break;
                        nr++;
                }
        }
        io_ring_submit_unlock(ctx, issue_flags);
        return all ? nr : ret;
}

int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
{
        struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
        struct io_cancel_data cd = {
                .ctx        = req->ctx,
                .data        = cancel->addr,
                .flags        = cancel->flags,
                .opcode        = cancel->opcode,
                .seq        = atomic_inc_return(&req->ctx->cancel_seq),
        };
        struct io_uring_task *tctx = req->tctx;
        int ret;

        if (cd.flags & IORING_ASYNC_CANCEL_FD) {
                if (req->flags & REQ_F_FIXED_FILE ||
                    cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) {
                        req->flags |= REQ_F_FIXED_FILE;
                        req->file = io_file_get_fixed(req, cancel->fd,
                                                        issue_flags);
                } else {
                        req->file = io_file_get_normal(req, cancel->fd);
                }
                if (!req->file) {
                        ret = -EBADF;
                        goto done;
                }
                cd.file = req->file;
        }

        ret = __io_async_cancel(&cd, tctx, issue_flags);
done:
        if (ret < 0)
                req_set_fail(req);
        io_req_set_res(req, ret, 0);
        return IOU_OK;
}

static int __io_sync_cancel(struct io_uring_task *tctx,
                            struct io_cancel_data *cd, int fd)
{
        struct io_ring_ctx *ctx = cd->ctx;

        /* fixed must be grabbed every time since we drop the uring_lock */
        if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
            (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
                struct io_rsrc_node *node;

                node = io_rsrc_node_lookup(&ctx->file_table.data, fd);
                if (unlikely(!node))
                        return -EBADF;
                cd->file = io_slot_file(node);
                if (!cd->file)
                        return -EBADF;
        }

        return __io_async_cancel(cd, tctx, 0);
}

int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
        __must_hold(&ctx->uring_lock)
{
        struct io_cancel_data cd = {
                .ctx        = ctx,
                .seq        = atomic_inc_return(&ctx->cancel_seq),
        };
        ktime_t timeout = KTIME_MAX;
        struct io_uring_sync_cancel_reg sc;
        struct file *file = NULL;
        DEFINE_WAIT(wait);
        int ret, i;

        if (copy_from_user(&sc, arg, sizeof(sc)))
                return -EFAULT;
        if (sc.flags & ~CANCEL_FLAGS)
                return -EINVAL;
        for (i = 0; i < ARRAY_SIZE(sc.pad); i++)
                if (sc.pad[i])
                        return -EINVAL;
        for (i = 0; i < ARRAY_SIZE(sc.pad2); i++)
                if (sc.pad2[i])
                        return -EINVAL;

        cd.data = sc.addr;
        cd.flags = sc.flags;
        cd.opcode = sc.opcode;

        /* we can grab a normal file descriptor upfront */
        if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
           !(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
                file = fget(sc.fd);
                if (!file)
                        return -EBADF;
                cd.file = file;
        }

        ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);

        /* found something, done! */
        if (ret != -EALREADY)
                goto out;

        if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) {
                struct timespec64 ts = {
                        .tv_sec                = sc.timeout.tv_sec,
                        .tv_nsec        = sc.timeout.tv_nsec
                };

                timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
        }

        /*
         * Keep looking until we get -ENOENT. we'll get woken everytime
         * every time a request completes and will retry the cancelation.
         */
        do {
                cd.seq = atomic_inc_return(&ctx->cancel_seq);

                prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE);

                ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);

                mutex_unlock(&ctx->uring_lock);
                if (ret != -EALREADY)
                        break;

                ret = io_run_task_work_sig(ctx);
                if (ret < 0)
                        break;
                ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS);
                if (!ret) {
                        ret = -ETIME;
                        break;
                }
                mutex_lock(&ctx->uring_lock);
        } while (1);

        finish_wait(&ctx->cq_wait, &wait);
        mutex_lock(&ctx->uring_lock);

        if (ret == -ENOENT || ret > 0)
                ret = 0;
out:
        if (file)
                fput(file);
        return ret;
}

























































































































































































































































































































































































































































































   30 











































































































































































































































































































   49 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * sysfs.h - definitions for the device driver filesystem
 *
 * Copyright (c) 2001,2002 Patrick Mochel
 * Copyright (c) 2004 Silicon Graphics, Inc.
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007 Tejun Heo <teheo@suse.de>
 *
 * Please see Documentation/filesystems/sysfs.rst for more information.
 */

#ifndef _SYSFS_H_
#define _SYSFS_H_

#include <linux/kernfs.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/kobject_ns.h>
#include <linux/stat.h>
#include <linux/atomic.h>

struct kobject;
struct module;
struct bin_attribute;
enum kobj_ns_type;

struct attribute {
        const char                *name;
        umode_t                        mode;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        bool                        ignore_lockdep:1;
        struct lock_class_key        *key;
        struct lock_class_key        skey;
#endif
};

/**
 *        sysfs_attr_init - initialize a dynamically allocated sysfs attribute
 *        @attr: struct attribute to initialize
 *
 *        Initialize a dynamically allocated struct attribute so we can
 *        make lockdep happy.  This is a new requirement for attributes
 *        and initially this is only needed when lockdep is enabled.
 *        Lockdep gives a nice error when your attribute is added to
 *        sysfs if you don't have this.
 */
#ifdef CONFIG_DEBUG_LOCK_ALLOC
#define sysfs_attr_init(attr)                                \
do {                                                        \
        static struct lock_class_key __key;                \
                                                        \
        (attr)->key = &__key;                                \
} while (0)
#else
#define sysfs_attr_init(attr) do {} while (0)
#endif

/**
 * struct attribute_group - data structure used to declare an attribute group.
 * @name:        Optional: Attribute group name
 *                If specified, the attribute group will be created in a
 *                new subdirectory with this name. Additionally when a
 *                group is named, @is_visible and @is_bin_visible may
 *                return SYSFS_GROUP_INVISIBLE to control visibility of
 *                the directory itself.
 * @is_visible:        Optional: Function to return permissions associated with an
 *                attribute of the group. Will be called repeatedly for
 *                each non-binary attribute in the group. Only read/write
 *                permissions as well as SYSFS_PREALLOC are accepted. Must
 *                return 0 if an attribute is not visible. The returned
 *                value will replace static permissions defined in struct
 *                attribute. Use SYSFS_GROUP_VISIBLE() when assigning this
 *                callback to specify separate _group_visible() and
 *                _attr_visible() handlers.
 * @is_bin_visible:
 *                Optional: Function to return permissions associated with a
 *                binary attribute of the group. Will be called repeatedly
 *                for each binary attribute in the group. Only read/write
 *                permissions as well as SYSFS_PREALLOC (and the
 *                visibility flags for named groups) are accepted. Must
 *                return 0 if a binary attribute is not visible. The
 *                returned value will replace static permissions defined
 *                in struct bin_attribute. If @is_visible is not set, Use
 *                SYSFS_GROUP_VISIBLE() when assigning this callback to
 *                specify separate _group_visible() and _attr_visible()
 *                handlers.
 * @attrs:        Pointer to NULL terminated list of attributes.
 * @bin_attrs:        Pointer to NULL terminated list of binary attributes.
 *                Either attrs or bin_attrs or both must be provided.
 */
struct attribute_group {
        const char                *name;
        umode_t                        (*is_visible)(struct kobject *,
                                              struct attribute *, int);
        umode_t                        (*is_bin_visible)(struct kobject *,
                                                  struct bin_attribute *, int);
        struct attribute        **attrs;
        struct bin_attribute        **bin_attrs;
};

#define SYSFS_PREALLOC                010000
#define SYSFS_GROUP_INVISIBLE        020000

/*
 * DEFINE_SYSFS_GROUP_VISIBLE(name):
 *        A helper macro to pair with the assignment of ".is_visible =
 *        SYSFS_GROUP_VISIBLE(name)", that arranges for the directory
 *        associated with a named attribute_group to optionally be hidden.
 *        This allows for static declaration of attribute_groups, and the
 *        simplification of attribute visibility lifetime that implies,
 *        without polluting sysfs with empty attribute directories.
 * Ex.
 *
 * static umode_t example_attr_visible(struct kobject *kobj,
 *                                   struct attribute *attr, int n)
 * {
 *       if (example_attr_condition)
 *               return 0;
 *       else if (ro_attr_condition)
 *               return 0444;
 *       return a->mode;
 * }
 *
 * static bool example_group_visible(struct kobject *kobj)
 * {
 *       if (example_group_condition)
 *               return false;
 *       return true;
 * }
 *
 * DEFINE_SYSFS_GROUP_VISIBLE(example);
 *
 * static struct attribute_group example_group = {
 *       .name = "example",
 *       .is_visible = SYSFS_GROUP_VISIBLE(example),
 *       .attrs = &example_attrs,
 * };
 *
 * Note that it expects <name>_attr_visible and <name>_group_visible to
 * be defined. For cases where individual attributes do not need
 * separate visibility consideration, only entire group visibility at
 * once, see DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE().
 */
#define DEFINE_SYSFS_GROUP_VISIBLE(name)                             \
        static inline umode_t sysfs_group_visible_##name(            \
                struct kobject *kobj, struct attribute *attr, int n) \
        {                                                            \
                if (n == 0 && !name##_group_visible(kobj))           \
                        return SYSFS_GROUP_INVISIBLE;                \
                return name##_attr_visible(kobj, attr, n);           \
        }

/*
 * DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(name):
 *        A helper macro to pair with SYSFS_GROUP_VISIBLE() that like
 *        DEFINE_SYSFS_GROUP_VISIBLE() controls group visibility, but does
 *        not require the implementation of a per-attribute visibility
 *        callback.
 * Ex.
 *
 * static bool example_group_visible(struct kobject *kobj)
 * {
 *       if (example_group_condition)
 *               return false;
 *       return true;
 * }
 *
 * DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(example);
 *
 * static struct attribute_group example_group = {
 *       .name = "example",
 *       .is_visible = SYSFS_GROUP_VISIBLE(example),
 *       .attrs = &example_attrs,
 * };
 */
#define DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(name)                   \
        static inline umode_t sysfs_group_visible_##name(         \
                struct kobject *kobj, struct attribute *a, int n) \
        {                                                         \
                if (n == 0 && !name##_group_visible(kobj))        \
                        return SYSFS_GROUP_INVISIBLE;             \
                return a->mode;                                   \
        }

/*
 * Same as DEFINE_SYSFS_GROUP_VISIBLE, but for groups with only binary
 * attributes. If an attribute_group defines both text and binary
 * attributes, the group visibility is determined by the function
 * specified to is_visible() not is_bin_visible()
 */
#define DEFINE_SYSFS_BIN_GROUP_VISIBLE(name)                             \
        static inline umode_t sysfs_group_visible_##name(                \
                struct kobject *kobj, struct bin_attribute *attr, int n) \
        {                                                                \
                if (n == 0 && !name##_group_visible(kobj))               \
                        return SYSFS_GROUP_INVISIBLE;                    \
                return name##_attr_visible(kobj, attr, n);               \
        }

#define DEFINE_SIMPLE_SYSFS_BIN_GROUP_VISIBLE(name)                   \
        static inline umode_t sysfs_group_visible_##name(             \
                struct kobject *kobj, struct bin_attribute *a, int n) \
        {                                                             \
                if (n == 0 && !name##_group_visible(kobj))            \
                        return SYSFS_GROUP_INVISIBLE;                 \
                return a->mode;                                       \
        }

#define SYSFS_GROUP_VISIBLE(fn) sysfs_group_visible_##fn

/*
 * Use these macros to make defining attributes easier.
 * See include/linux/device.h for examples..
 */

#define __ATTR(_name, _mode, _show, _store) {                                \
        .attr = {.name = __stringify(_name),                                \
                 .mode = VERIFY_OCTAL_PERMISSIONS(_mode) },                \
        .show        = _show,                                                \
        .store        = _store,                                                \
}

#define __ATTR_PREALLOC(_name, _mode, _show, _store) {                        \
        .attr = {.name = __stringify(_name),                                \
                 .mode = SYSFS_PREALLOC | VERIFY_OCTAL_PERMISSIONS(_mode) },\
        .show        = _show,                                                \
        .store        = _store,                                                \
}

#define __ATTR_RO(_name) {                                                \
        .attr        = { .name = __stringify(_name), .mode = 0444 },                \
        .show        = _name##_show,                                                \
}

#define __ATTR_RO_MODE(_name, _mode) {                                        \
        .attr        = { .name = __stringify(_name),                                \
                    .mode = VERIFY_OCTAL_PERMISSIONS(_mode) },                \
        .show        = _name##_show,                                                \
}

#define __ATTR_RW_MODE(_name, _mode) {                                        \
        .attr        = { .name = __stringify(_name),                                \
                    .mode = VERIFY_OCTAL_PERMISSIONS(_mode) },                \
        .show        = _name##_show,                                                \
        .store        = _name##_store,                                        \
}

#define __ATTR_WO(_name) {                                                \
        .attr        = { .name = __stringify(_name), .mode = 0200 },                \
        .store        = _name##_store,                                        \
}

#define __ATTR_RW(_name) __ATTR(_name, 0644, _name##_show, _name##_store)

#define __ATTR_NULL { .attr = { .name = NULL } }

#ifdef CONFIG_DEBUG_LOCK_ALLOC
#define __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) {        \
        .attr = {.name = __stringify(_name), .mode = _mode,        \
                        .ignore_lockdep = true },                \
        .show                = _show,                                \
        .store                = _store,                                \
}
#else
#define __ATTR_IGNORE_LOCKDEP        __ATTR
#endif

#define __ATTRIBUTE_GROUPS(_name)                                \
static const struct attribute_group *_name##_groups[] = {        \
        &_name##_group,                                                \
        NULL,                                                        \
}

#define ATTRIBUTE_GROUPS(_name)                                        \
static const struct attribute_group _name##_group = {                \
        .attrs = _name##_attrs,                                        \
};                                                                \
__ATTRIBUTE_GROUPS(_name)

#define BIN_ATTRIBUTE_GROUPS(_name)                                \
static const struct attribute_group _name##_group = {                \
        .bin_attrs = _name##_attrs,                                \
};                                                                \
__ATTRIBUTE_GROUPS(_name)

struct file;
struct vm_area_struct;
struct address_space;

struct bin_attribute {
        struct attribute        attr;
        size_t                        size;
        void                        *private;
        struct address_space *(*f_mapping)(void);
        ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *,
                        char *, loff_t, size_t);
        ssize_t (*write)(struct file *, struct kobject *, struct bin_attribute *,
                         char *, loff_t, size_t);
        loff_t (*llseek)(struct file *, struct kobject *, struct bin_attribute *,
                         loff_t, int);
        int (*mmap)(struct file *, struct kobject *, struct bin_attribute *attr,
                    struct vm_area_struct *vma);
};

/**
 *        sysfs_bin_attr_init - initialize a dynamically allocated bin_attribute
 *        @attr: struct bin_attribute to initialize
 *
 *        Initialize a dynamically allocated struct bin_attribute so we
 *        can make lockdep happy.  This is a new requirement for
 *        attributes and initially this is only needed when lockdep is
 *        enabled.  Lockdep gives a nice error when your attribute is
 *        added to sysfs if you don't have this.
 */
#define sysfs_bin_attr_init(bin_attr) sysfs_attr_init(&(bin_attr)->attr)

/* macros to create static binary attributes easier */
#define __BIN_ATTR(_name, _mode, _read, _write, _size) {                \
        .attr = { .name = __stringify(_name), .mode = _mode },                \
        .read        = _read,                                                \
        .write        = _write,                                                \
        .size        = _size,                                                \
}

#define __BIN_ATTR_RO(_name, _size) {                                        \
        .attr        = { .name = __stringify(_name), .mode = 0444 },                \
        .read        = _name##_read,                                                \
        .size        = _size,                                                \
}

#define __BIN_ATTR_WO(_name, _size) {                                        \
        .attr        = { .name = __stringify(_name), .mode = 0200 },                \
        .write        = _name##_write,                                        \
        .size        = _size,                                                \
}

#define __BIN_ATTR_RW(_name, _size)                                        \
        __BIN_ATTR(_name, 0644, _name##_read, _name##_write, _size)

#define __BIN_ATTR_NULL __ATTR_NULL

#define BIN_ATTR(_name, _mode, _read, _write, _size)                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR(_name, _mode, _read,        \
                                        _write, _size)

#define BIN_ATTR_RO(_name, _size)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_RO(_name, _size)

#define BIN_ATTR_WO(_name, _size)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_WO(_name, _size)

#define BIN_ATTR_RW(_name, _size)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_RW(_name, _size)


#define __BIN_ATTR_ADMIN_RO(_name, _size) {                                        \
        .attr        = { .name = __stringify(_name), .mode = 0400 },                \
        .read        = _name##_read,                                                \
        .size        = _size,                                                \
}

#define __BIN_ATTR_ADMIN_RW(_name, _size)                                        \
        __BIN_ATTR(_name, 0600, _name##_read, _name##_write, _size)

#define BIN_ATTR_ADMIN_RO(_name, _size)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RO(_name, _size)

#define BIN_ATTR_ADMIN_RW(_name, _size)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_ADMIN_RW(_name, _size)

#define __BIN_ATTR_SIMPLE_RO(_name, _mode) {                                \
        .attr        = { .name = __stringify(_name), .mode = _mode },        \
        .read        = sysfs_bin_attr_simple_read,                                \
}

#define BIN_ATTR_SIMPLE_RO(_name)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_SIMPLE_RO(_name, 0444)

#define BIN_ATTR_SIMPLE_ADMIN_RO(_name)                                        \
struct bin_attribute bin_attr_##_name = __BIN_ATTR_SIMPLE_RO(_name, 0400)

struct sysfs_ops {
        ssize_t        (*show)(struct kobject *, struct attribute *, char *);
        ssize_t        (*store)(struct kobject *, struct attribute *, const char *, size_t);
};

#ifdef CONFIG_SYSFS

int __must_check sysfs_create_dir_ns(struct kobject *kobj, const void *ns);
void sysfs_remove_dir(struct kobject *kobj);
int __must_check sysfs_rename_dir_ns(struct kobject *kobj, const char *new_name,
                                     const void *new_ns);
int __must_check sysfs_move_dir_ns(struct kobject *kobj,
                                   struct kobject *new_parent_kobj,
                                   const void *new_ns);
int __must_check sysfs_create_mount_point(struct kobject *parent_kobj,
                                          const char *name);
void sysfs_remove_mount_point(struct kobject *parent_kobj,
                              const char *name);

int __must_check sysfs_create_file_ns(struct kobject *kobj,
                                      const struct attribute *attr,
                                      const void *ns);
int __must_check sysfs_create_files(struct kobject *kobj,
                                   const struct attribute * const *attr);
int __must_check sysfs_chmod_file(struct kobject *kobj,
                                  const struct attribute *attr, umode_t mode);
struct kernfs_node *sysfs_break_active_protection(struct kobject *kobj,
                                                  const struct attribute *attr);
void sysfs_unbreak_active_protection(struct kernfs_node *kn);
void sysfs_remove_file_ns(struct kobject *kobj, const struct attribute *attr,
                          const void *ns);
bool sysfs_remove_file_self(struct kobject *kobj, const struct attribute *attr);
void sysfs_remove_files(struct kobject *kobj, const struct attribute * const *attr);

int __must_check sysfs_create_bin_file(struct kobject *kobj,
                                       const struct bin_attribute *attr);
void sysfs_remove_bin_file(struct kobject *kobj,
                           const struct bin_attribute *attr);

int __must_check sysfs_create_link(struct kobject *kobj, struct kobject *target,
                                   const char *name);
int __must_check sysfs_create_link_nowarn(struct kobject *kobj,
                                          struct kobject *target,
                                          const char *name);
void sysfs_remove_link(struct kobject *kobj, const char *name);

int sysfs_rename_link_ns(struct kobject *kobj, struct kobject *target,
                         const char *old_name, const char *new_name,
                         const void *new_ns);

void sysfs_delete_link(struct kobject *dir, struct kobject *targ,
                        const char *name);

int __must_check sysfs_create_group(struct kobject *kobj,
                                    const struct attribute_group *grp);
int __must_check sysfs_create_groups(struct kobject *kobj,
                                     const struct attribute_group **groups);
int __must_check sysfs_update_groups(struct kobject *kobj,
                                     const struct attribute_group **groups);
int sysfs_update_group(struct kobject *kobj,
                       const struct attribute_group *grp);
void sysfs_remove_group(struct kobject *kobj,
                        const struct attribute_group *grp);
void sysfs_remove_groups(struct kobject *kobj,
                         const struct attribute_group **groups);
int sysfs_add_file_to_group(struct kobject *kobj,
                        const struct attribute *attr, const char *group);
void sysfs_remove_file_from_group(struct kobject *kobj,
                        const struct attribute *attr, const char *group);
int sysfs_merge_group(struct kobject *kobj,
                       const struct attribute_group *grp);
void sysfs_unmerge_group(struct kobject *kobj,
                       const struct attribute_group *grp);
int sysfs_add_link_to_group(struct kobject *kobj, const char *group_name,
                            struct kobject *target, const char *link_name);
void sysfs_remove_link_from_group(struct kobject *kobj, const char *group_name,
                                  const char *link_name);
int compat_only_sysfs_link_entry_to_kobj(struct kobject *kobj,
                                         struct kobject *target_kobj,
                                         const char *target_name,
                                         const char *symlink_name);

void sysfs_notify(struct kobject *kobj, const char *dir, const char *attr);

int __must_check sysfs_init(void);

static inline void sysfs_enable_ns(struct kernfs_node *kn)
{
        return kernfs_enable_ns(kn);
}

int sysfs_file_change_owner(struct kobject *kobj, const char *name, kuid_t kuid,
                            kgid_t kgid);
int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t kgid);
int sysfs_link_change_owner(struct kobject *kobj, struct kobject *targ,
                            const char *name, kuid_t kuid, kgid_t kgid);
int sysfs_groups_change_owner(struct kobject *kobj,
                              const struct attribute_group **groups,
                              kuid_t kuid, kgid_t kgid);
int sysfs_group_change_owner(struct kobject *kobj,
                             const struct attribute_group *groups, kuid_t kuid,
                             kgid_t kgid);
__printf(2, 3)
int sysfs_emit(char *buf, const char *fmt, ...);
__printf(3, 4)
int sysfs_emit_at(char *buf, int at, const char *fmt, ...);

ssize_t sysfs_bin_attr_simple_read(struct file *file, struct kobject *kobj,
                                   struct bin_attribute *attr, char *buf,
                                   loff_t off, size_t count);

#else /* CONFIG_SYSFS */

static inline int sysfs_create_dir_ns(struct kobject *kobj, const void *ns)
{
        return 0;
}

static inline void sysfs_remove_dir(struct kobject *kobj)
{
}

static inline int sysfs_rename_dir_ns(struct kobject *kobj,
                                      const char *new_name, const void *new_ns)
{
        return 0;
}

static inline int sysfs_move_dir_ns(struct kobject *kobj,
                                    struct kobject *new_parent_kobj,
                                    const void *new_ns)
{
        return 0;
}

static inline int sysfs_create_mount_point(struct kobject *parent_kobj,
                                           const char *name)
{
        return 0;
}

static inline void sysfs_remove_mount_point(struct kobject *parent_kobj,
                                            const char *name)
{
}

static inline int sysfs_create_file_ns(struct kobject *kobj,
                                       const struct attribute *attr,
                                       const void *ns)
{
        return 0;
}

static inline int sysfs_create_files(struct kobject *kobj,
                                    const struct attribute * const *attr)
{
        return 0;
}

static inline int sysfs_chmod_file(struct kobject *kobj,
                                   const struct attribute *attr, umode_t mode)
{
        return 0;
}

static inline struct kernfs_node *
sysfs_break_active_protection(struct kobject *kobj,
                              const struct attribute *attr)
{
        return NULL;
}

static inline void sysfs_unbreak_active_protection(struct kernfs_node *kn)
{
}

static inline void sysfs_remove_file_ns(struct kobject *kobj,
                                        const struct attribute *attr,
                                        const void *ns)
{
}

static inline bool sysfs_remove_file_self(struct kobject *kobj,
                                          const struct attribute *attr)
{
        return false;
}

static inline void sysfs_remove_files(struct kobject *kobj,
                                     const struct attribute * const *attr)
{
}

static inline int sysfs_create_bin_file(struct kobject *kobj,
                                        const struct bin_attribute *attr)
{
        return 0;
}

static inline void sysfs_remove_bin_file(struct kobject *kobj,
                                         const struct bin_attribute *attr)
{
}

static inline int sysfs_create_link(struct kobject *kobj,
                                    struct kobject *target, const char *name)
{
        return 0;
}

static inline int sysfs_create_link_nowarn(struct kobject *kobj,
                                           struct kobject *target,
                                           const char *name)
{
        return 0;
}

static inline void sysfs_remove_link(struct kobject *kobj, const char *name)
{
}

static inline int sysfs_rename_link_ns(struct kobject *k, struct kobject *t,
                                       const char *old_name,
                                       const char *new_name, const void *ns)
{
        return 0;
}

static inline void sysfs_delete_link(struct kobject *k, struct kobject *t,
                                     const char *name)
{
}

static inline int sysfs_create_group(struct kobject *kobj,
                                     const struct attribute_group *grp)
{
        return 0;
}

static inline int sysfs_create_groups(struct kobject *kobj,
                                      const struct attribute_group **groups)
{
        return 0;
}

static inline int sysfs_update_groups(struct kobject *kobj,
                                      const struct attribute_group **groups)
{
        return 0;
}

static inline int sysfs_update_group(struct kobject *kobj,
                                const struct attribute_group *grp)
{
        return 0;
}

static inline void sysfs_remove_group(struct kobject *kobj,
                                      const struct attribute_group *grp)
{
}

static inline void sysfs_remove_groups(struct kobject *kobj,
                                       const struct attribute_group **groups)
{
}

static inline int sysfs_add_file_to_group(struct kobject *kobj,
                const struct attribute *attr, const char *group)
{
        return 0;
}

static inline void sysfs_remove_file_from_group(struct kobject *kobj,
                const struct attribute *attr, const char *group)
{
}

static inline int sysfs_merge_group(struct kobject *kobj,
                       const struct attribute_group *grp)
{
        return 0;
}

static inline void sysfs_unmerge_group(struct kobject *kobj,
                       const struct attribute_group *grp)
{
}

static inline int sysfs_add_link_to_group(struct kobject *kobj,
                const char *group_name, struct kobject *target,
                const char *link_name)
{
        return 0;
}

static inline void sysfs_remove_link_from_group(struct kobject *kobj,
                const char *group_name, const char *link_name)
{
}

static inline int compat_only_sysfs_link_entry_to_kobj(struct kobject *kobj,
                                                       struct kobject *target_kobj,
                                                       const char *target_name,
                                                       const char *symlink_name)
{
        return 0;
}

static inline void sysfs_notify(struct kobject *kobj, const char *dir,
                                const char *attr)
{
}

static inline int __must_check sysfs_init(void)
{
        return 0;
}

static inline void sysfs_enable_ns(struct kernfs_node *kn)
{
}

static inline int sysfs_file_change_owner(struct kobject *kobj,
                                          const char *name, kuid_t kuid,
                                          kgid_t kgid)
{
        return 0;
}

static inline int sysfs_link_change_owner(struct kobject *kobj,
                                          struct kobject *targ,
                                          const char *name, kuid_t kuid,
                                          kgid_t kgid)
{
        return 0;
}

static inline int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t kgid)
{
        return 0;
}

static inline int sysfs_groups_change_owner(struct kobject *kobj,
                          const struct attribute_group **groups,
                          kuid_t kuid, kgid_t kgid)
{
        return 0;
}

static inline int sysfs_group_change_owner(struct kobject *kobj,
                                           const struct attribute_group *groups,
                                           kuid_t kuid, kgid_t kgid)
{
        return 0;
}

__printf(2, 3)
static inline int sysfs_emit(char *buf, const char *fmt, ...)
{
        return 0;
}

__printf(3, 4)
static inline int sysfs_emit_at(char *buf, int at, const char *fmt, ...)
{
        return 0;
}

static inline ssize_t sysfs_bin_attr_simple_read(struct file *file,
                                                 struct kobject *kobj,
                                                 struct bin_attribute *attr,
                                                 char *buf, loff_t off,
                                                 size_t count)
{
        return 0;
}
#endif /* CONFIG_SYSFS */

static inline int __must_check sysfs_create_file(struct kobject *kobj,
                                                 const struct attribute *attr)
{
        return sysfs_create_file_ns(kobj, attr, NULL);
}

static inline void sysfs_remove_file(struct kobject *kobj,
                                     const struct attribute *attr)
{
        sysfs_remove_file_ns(kobj, attr, NULL);
}

static inline int sysfs_rename_link(struct kobject *kobj, struct kobject *target,
                                    const char *old_name, const char *new_name)
{
        return sysfs_rename_link_ns(kobj, target, old_name, new_name, NULL);
}

static inline void sysfs_notify_dirent(struct kernfs_node *kn)
{
        kernfs_notify(kn);
}

static inline struct kernfs_node *sysfs_get_dirent(struct kernfs_node *parent,
                                                   const char *name)
{
        return kernfs_find_and_get(parent, name);
}

static inline struct kernfs_node *sysfs_get(struct kernfs_node *kn)
{
        kernfs_get(kn);
        return kn;
}

static inline void sysfs_put(struct kernfs_node *kn)
{
        kernfs_put(kn);
}

#endif /* _SYSFS_H_ */






































   23 



   64 
   23 



   24 
   25 




























































   58 


   61 


   59 
   61 








   61 

















   61 

   61 
   58 




















   94 
















   94 


















   91 
    1 
   92 
   95 
   93 









   94 




   95 


   90 


   95 














































   61 


   63 


























    2 
    2 





   62 


   63 
   64 
   60 









    1 




















































    1 
    1 




















































































    1 


    1 

    1 












    7 








    5 
    2 











    5 




    6 














   63 


   60 






   63 































   53 








   55 








   54 



   55 
   54 



   11 




   54 








   52 





   54 
   52 






   54 

   53 







   50 




   11 









   60 




   63 
   55 















   63 
















   61 
   63 




   62 
   63 
    1 
    1 









   61 




   58 






   63 










   36 


   31 



   22 














   22 




















   23 
   23 







   23 


   23 










   23 
    8 

















   22 


























   38 





   23 








   54 

   55 






   64 










































   63 






   63 


   61 
   63 

    2 


   62 

   54 
   63 

















   50 













   53 






   52 
   53 
   53 


   52 







































































































































    3 














    2 





    2 

    2 
    2 














    4 










    4 
    4 


    4 


    4 


    1 




    1 















    4 
    4 






    1 








    1 
    1 











   24 











   24 
   24 
   24 
   23 










   16 


   25 





   25 

   15 

















































   24 
   25 




   25 





   24 











   25 

   25 




   25 

   25 
















   23 


   23 














   24 
   24 


   24 













   25 


   25 
    1 

    1 

    1 

   25 


   24 







   23 



























   24 







   24 








   23 

   23 

   23 


   24 


   23 









    4 


    4 


















    4 












    1 
    2 














    4 



























    4 



    4 

    2 
    4 
    4 


























































































    3 


    3 





    3 
    1 
    3 




    3 
    3 












    4 

    3 
    3 
    4 












































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
// SPDX-License-Identifier: GPL-2.0-or-later
/* Keyring handling
 *
 * Copyright (C) 2004-2005, 2008, 2013 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/export.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <linux/seq_file.h>
#include <linux/err.h>
#include <linux/user_namespace.h>
#include <linux/nsproxy.h>
#include <keys/keyring-type.h>
#include <keys/user-type.h>
#include <linux/assoc_array_priv.h>
#include <linux/uaccess.h>
#include <net/net_namespace.h>
#include "internal.h"

/*
 * When plumbing the depths of the key tree, this sets a hard limit
 * set on how deep we're willing to go.
 */
#define KEYRING_SEARCH_MAX_DEPTH 6

/*
 * We mark pointers we pass to the associative array with bit 1 set if
 * they're keyrings and clear otherwise.
 */
#define KEYRING_PTR_SUBTYPE        0x2UL

static inline bool keyring_ptr_is_keyring(const struct assoc_array_ptr *x)
{
        return (unsigned long)x & KEYRING_PTR_SUBTYPE;
}
static inline struct key *keyring_ptr_to_key(const struct assoc_array_ptr *x)
{
        void *object = assoc_array_ptr_to_leaf(x);
        return (struct key *)((unsigned long)object & ~KEYRING_PTR_SUBTYPE);
}
static inline void *keyring_key_to_ptr(struct key *key)
{
        if (key->type == &key_type_keyring)
                return (void *)((unsigned long)key | KEYRING_PTR_SUBTYPE);
        return key;
}

static DEFINE_RWLOCK(keyring_name_lock);

/*
 * Clean up the bits of user_namespace that belong to us.
 */
void key_free_user_ns(struct user_namespace *ns)
{
        write_lock(&keyring_name_lock);
        list_del_init(&ns->keyring_name_list);
        write_unlock(&keyring_name_lock);

        key_put(ns->user_keyring_register);
#ifdef CONFIG_PERSISTENT_KEYRINGS
        key_put(ns->persistent_keyring_register);
#endif
}

/*
 * The keyring key type definition.  Keyrings are simply keys of this type and
 * can be treated as ordinary keys in addition to having their own special
 * operations.
 */
static int keyring_preparse(struct key_preparsed_payload *prep);
static void keyring_free_preparse(struct key_preparsed_payload *prep);
static int keyring_instantiate(struct key *keyring,
                               struct key_preparsed_payload *prep);
static void keyring_revoke(struct key *keyring);
static void keyring_destroy(struct key *keyring);
static void keyring_describe(const struct key *keyring, struct seq_file *m);
static long keyring_read(const struct key *keyring,
                         char *buffer, size_t buflen);

struct key_type key_type_keyring = {
        .name                = "keyring",
        .def_datalen        = 0,
        .preparse        = keyring_preparse,
        .free_preparse        = keyring_free_preparse,
        .instantiate        = keyring_instantiate,
        .revoke                = keyring_revoke,
        .destroy        = keyring_destroy,
        .describe        = keyring_describe,
        .read                = keyring_read,
};
EXPORT_SYMBOL(key_type_keyring);

/*
 * Semaphore to serialise link/link calls to prevent two link calls in parallel
 * introducing a cycle.
 */
static DEFINE_MUTEX(keyring_serialise_link_lock);

/*
 * Publish the name of a keyring so that it can be found by name (if it has
 * one and it doesn't begin with a dot).
 */
static void keyring_publish_name(struct key *keyring)
{
        struct user_namespace *ns = current_user_ns();

        if (keyring->description &&
            keyring->description[0] &&
            keyring->description[0] != '.') {
                write_lock(&keyring_name_lock);
                list_add_tail(&keyring->name_link, &ns->keyring_name_list);
                write_unlock(&keyring_name_lock);
        }
}

/*
 * Preparse a keyring payload
 */
static int keyring_preparse(struct key_preparsed_payload *prep)
{
        return prep->datalen != 0 ? -EINVAL : 0;
}

/*
 * Free a preparse of a user defined key payload
 */
static void keyring_free_preparse(struct key_preparsed_payload *prep)
{
}

/*
 * Initialise a keyring.
 *
 * Returns 0 on success, -EINVAL if given any data.
 */
static int keyring_instantiate(struct key *keyring,
                               struct key_preparsed_payload *prep)
{
        assoc_array_init(&keyring->keys);
        /* make the keyring available by name if it has one */
        keyring_publish_name(keyring);
        return 0;
}

/*
 * Multiply 64-bits by 32-bits to 96-bits and fold back to 64-bit.  Ideally we'd
 * fold the carry back too, but that requires inline asm.
 */
static u64 mult_64x32_and_fold(u64 x, u32 y)
{
        u64 hi = (u64)(u32)(x >> 32) * y;
        u64 lo = (u64)(u32)(x) * y;
        return lo + ((u64)(u32)hi << 32) + (u32)(hi >> 32);
}

/*
 * Hash a key type and description.
 */
static void hash_key_type_and_desc(struct keyring_index_key *index_key)
{
        const unsigned level_shift = ASSOC_ARRAY_LEVEL_STEP;
        const unsigned long fan_mask = ASSOC_ARRAY_FAN_MASK;
        const char *description = index_key->description;
        unsigned long hash, type;
        u32 piece;
        u64 acc;
        int n, desc_len = index_key->desc_len;

        type = (unsigned long)index_key->type;
        acc = mult_64x32_and_fold(type, desc_len + 13);
        acc = mult_64x32_and_fold(acc, 9207);
        piece = (unsigned long)index_key->domain_tag;
        acc = mult_64x32_and_fold(acc, piece);
        acc = mult_64x32_and_fold(acc, 9207);

        for (;;) {
                n = desc_len;
                if (n <= 0)
                        break;
                if (n > 4)
                        n = 4;
                piece = 0;
                memcpy(&piece, description, n);
                description += n;
                desc_len -= n;
                acc = mult_64x32_and_fold(acc, piece);
                acc = mult_64x32_and_fold(acc, 9207);
        }

        /* Fold the hash down to 32 bits if need be. */
        hash = acc;
        if (ASSOC_ARRAY_KEY_CHUNK_SIZE == 32)
                hash ^= acc >> 32;

        /* Squidge all the keyrings into a separate part of the tree to
         * ordinary keys by making sure the lowest level segment in the hash is
         * zero for keyrings and non-zero otherwise.
         */
        if (index_key->type != &key_type_keyring && (hash & fan_mask) == 0)
                hash |= (hash >> (ASSOC_ARRAY_KEY_CHUNK_SIZE - level_shift)) | 1;
        else if (index_key->type == &key_type_keyring && (hash & fan_mask) != 0)
                hash = (hash + (hash << level_shift)) & ~fan_mask;
        index_key->hash = hash;
}

/*
 * Finalise an index key to include a part of the description actually in the
 * index key, to set the domain tag and to calculate the hash.
 */
void key_set_index_key(struct keyring_index_key *index_key)
{
        static struct key_tag default_domain_tag = { .usage = REFCOUNT_INIT(1), };
        size_t n = min_t(size_t, index_key->desc_len, sizeof(index_key->desc));

        memcpy(index_key->desc, index_key->description, n);

        if (!index_key->domain_tag) {
                if (index_key->type->flags & KEY_TYPE_NET_DOMAIN)
                        index_key->domain_tag = current->nsproxy->net_ns->key_domain;
                else
                        index_key->domain_tag = &default_domain_tag;
        }

        hash_key_type_and_desc(index_key);
}

/**
 * key_put_tag - Release a ref on a tag.
 * @tag: The tag to release.
 *
 * This releases a reference the given tag and returns true if that ref was the
 * last one.
 */
bool key_put_tag(struct key_tag *tag)
{
        if (refcount_dec_and_test(&tag->usage)) {
                kfree_rcu(tag, rcu);
                return true;
        }

        return false;
}

/**
 * key_remove_domain - Kill off a key domain and gc its keys
 * @domain_tag: The domain tag to release.
 *
 * This marks a domain tag as being dead and releases a ref on it.  If that
 * wasn't the last reference, the garbage collector is poked to try and delete
 * all keys that were in the domain.
 */
void key_remove_domain(struct key_tag *domain_tag)
{
        domain_tag->removed = true;
        if (!key_put_tag(domain_tag))
                key_schedule_gc_links();
}

/*
 * Build the next index key chunk.
 *
 * We return it one word-sized chunk at a time.
 */
static unsigned long keyring_get_key_chunk(const void *data, int level)
{
        const struct keyring_index_key *index_key = data;
        unsigned long chunk = 0;
        const u8 *d;
        int desc_len = index_key->desc_len, n = sizeof(chunk);

        level /= ASSOC_ARRAY_KEY_CHUNK_SIZE;
        switch (level) {
        case 0:
                return index_key->hash;
        case 1:
                return index_key->x;
        case 2:
                return (unsigned long)index_key->type;
        case 3:
                return (unsigned long)index_key->domain_tag;
        default:
                level -= 4;
                if (desc_len <= sizeof(index_key->desc))
                        return 0;

                d = index_key->description + sizeof(index_key->desc);
                d += level * sizeof(long);
                desc_len -= sizeof(index_key->desc);
                if (desc_len > n)
                        desc_len = n;
                do {
                        chunk <<= 8;
                        chunk |= *d++;
                } while (--desc_len > 0);
                return chunk;
        }
}

static unsigned long keyring_get_object_key_chunk(const void *object, int level)
{
        const struct key *key = keyring_ptr_to_key(object);
        return keyring_get_key_chunk(&key->index_key, level);
}

static bool keyring_compare_object(const void *object, const void *data)
{
        const struct keyring_index_key *index_key = data;
        const struct key *key = keyring_ptr_to_key(object);

        return key->index_key.type == index_key->type &&
                key->index_key.domain_tag == index_key->domain_tag &&
                key->index_key.desc_len == index_key->desc_len &&
                memcmp(key->index_key.description, index_key->description,
                       index_key->desc_len) == 0;
}

/*
 * Compare the index keys of a pair of objects and determine the bit position
 * at which they differ - if they differ.
 */
static int keyring_diff_objects(const void *object, const void *data)
{
        const struct key *key_a = keyring_ptr_to_key(object);
        const struct keyring_index_key *a = &key_a->index_key;
        const struct keyring_index_key *b = data;
        unsigned long seg_a, seg_b;
        int level, i;

        level = 0;
        seg_a = a->hash;
        seg_b = b->hash;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += ASSOC_ARRAY_KEY_CHUNK_SIZE / 8;

        /* The number of bits contributed by the hash is controlled by a
         * constant in the assoc_array headers.  Everything else thereafter we
         * can deal with as being machine word-size dependent.
         */
        seg_a = a->x;
        seg_b = b->x;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        /* The next bit may not work on big endian */
        seg_a = (unsigned long)a->type;
        seg_b = (unsigned long)b->type;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        seg_a = (unsigned long)a->domain_tag;
        seg_b = (unsigned long)b->domain_tag;
        if ((seg_a ^ seg_b) != 0)
                goto differ;
        level += sizeof(unsigned long);

        i = sizeof(a->desc);
        if (a->desc_len <= i)
                goto same;

        for (; i < a->desc_len; i++) {
                seg_a = *(unsigned char *)(a->description + i);
                seg_b = *(unsigned char *)(b->description + i);
                if ((seg_a ^ seg_b) != 0)
                        goto differ_plus_i;
        }

same:
        return -1;

differ_plus_i:
        level += i;
differ:
        i = level * 8 + __ffs(seg_a ^ seg_b);
        return i;
}

/*
 * Free an object after stripping the keyring flag off of the pointer.
 */
static void keyring_free_object(void *object)
{
        key_put(keyring_ptr_to_key(object));
}

/*
 * Operations for keyring management by the index-tree routines.
 */
static const struct assoc_array_ops keyring_assoc_array_ops = {
        .get_key_chunk                = keyring_get_key_chunk,
        .get_object_key_chunk        = keyring_get_object_key_chunk,
        .compare_object                = keyring_compare_object,
        .diff_objects                = keyring_diff_objects,
        .free_object                = keyring_free_object,
};

/*
 * Clean up a keyring when it is destroyed.  Unpublish its name if it had one
 * and dispose of its data.
 *
 * The garbage collector detects the final key_put(), removes the keyring from
 * the serial number tree and then does RCU synchronisation before coming here,
 * so we shouldn't need to worry about code poking around here with the RCU
 * readlock held by this time.
 */
static void keyring_destroy(struct key *keyring)
{
        if (keyring->description) {
                write_lock(&keyring_name_lock);

                if (keyring->name_link.next != NULL &&
                    !list_empty(&keyring->name_link))
                        list_del(&keyring->name_link);

                write_unlock(&keyring_name_lock);
        }

        if (keyring->restrict_link) {
                struct key_restriction *keyres = keyring->restrict_link;

                key_put(keyres->key);
                kfree(keyres);
        }

        assoc_array_destroy(&keyring->keys, &keyring_assoc_array_ops);
}

/*
 * Describe a keyring for /proc.
 */
static void keyring_describe(const struct key *keyring, struct seq_file *m)
{
        if (keyring->description)
                seq_puts(m, keyring->description);
        else
                seq_puts(m, "[anon]");

        if (key_is_positive(keyring)) {
                if (keyring->keys.nr_leaves_on_tree != 0)
                        seq_printf(m, ": %lu", keyring->keys.nr_leaves_on_tree);
                else
                        seq_puts(m, ": empty");
        }
}

struct keyring_read_iterator_context {
        size_t                        buflen;
        size_t                        count;
        key_serial_t                *buffer;
};

static int keyring_read_iterator(const void *object, void *data)
{
        struct keyring_read_iterator_context *ctx = data;
        const struct key *key = keyring_ptr_to_key(object);

        kenter("{%s,%d},,{%zu/%zu}",
               key->type->name, key->serial, ctx->count, ctx->buflen);

        if (ctx->count >= ctx->buflen)
                return 1;

        *ctx->buffer++ = key->serial;
        ctx->count += sizeof(key->serial);
        return 0;
}

/*
 * Read a list of key IDs from the keyring's contents in binary form
 *
 * The keyring's semaphore is read-locked by the caller.  This prevents someone
 * from modifying it under us - which could cause us to read key IDs multiple
 * times.
 */
static long keyring_read(const struct key *keyring,
                         char *buffer, size_t buflen)
{
        struct keyring_read_iterator_context ctx;
        long ret;

        kenter("{%d},,%zu", key_serial(keyring), buflen);

        if (buflen & (sizeof(key_serial_t) - 1))
                return -EINVAL;

        /* Copy as many key IDs as fit into the buffer */
        if (buffer && buflen) {
                ctx.buffer = (key_serial_t *)buffer;
                ctx.buflen = buflen;
                ctx.count = 0;
                ret = assoc_array_iterate(&keyring->keys,
                                          keyring_read_iterator, &ctx);
                if (ret < 0) {
                        kleave(" = %ld [iterate]", ret);
                        return ret;
                }
        }

        /* Return the size of the buffer needed */
        ret = keyring->keys.nr_leaves_on_tree * sizeof(key_serial_t);
        if (ret <= buflen)
                kleave("= %ld [ok]", ret);
        else
                kleave("= %ld [buffer too small]", ret);
        return ret;
}

/*
 * Allocate a keyring and link into the destination keyring.
 */
struct key *keyring_alloc(const char *description, kuid_t uid, kgid_t gid,
                          const struct cred *cred, key_perm_t perm,
                          unsigned long flags,
                          struct key_restriction *restrict_link,
                          struct key *dest)
{
        struct key *keyring;
        int ret;

        keyring = key_alloc(&key_type_keyring, description,
                            uid, gid, cred, perm, flags, restrict_link);
        if (!IS_ERR(keyring)) {
                ret = key_instantiate_and_link(keyring, NULL, 0, dest, NULL);
                if (ret < 0) {
                        key_put(keyring);
                        keyring = ERR_PTR(ret);
                }
        }

        return keyring;
}
EXPORT_SYMBOL(keyring_alloc);

/**
 * restrict_link_reject - Give -EPERM to restrict link
 * @keyring: The keyring being added to.
 * @type: The type of key being added.
 * @payload: The payload of the key intended to be added.
 * @restriction_key: Keys providing additional data for evaluating restriction.
 *
 * Reject the addition of any links to a keyring.  It can be overridden by
 * passing KEY_ALLOC_BYPASS_RESTRICTION to key_instantiate_and_link() when
 * adding a key to a keyring.
 *
 * This is meant to be stored in a key_restriction structure which is passed
 * in the restrict_link parameter to keyring_alloc().
 */
int restrict_link_reject(struct key *keyring,
                         const struct key_type *type,
                         const union key_payload *payload,
                         struct key *restriction_key)
{
        return -EPERM;
}

/*
 * By default, we keys found by getting an exact match on their descriptions.
 */
bool key_default_cmp(const struct key *key,
                     const struct key_match_data *match_data)
{
        return strcmp(key->description, match_data->raw_data) == 0;
}

/*
 * Iteration function to consider each key found.
 */
static int keyring_search_iterator(const void *object, void *iterator_data)
{
        struct keyring_search_context *ctx = iterator_data;
        const struct key *key = keyring_ptr_to_key(object);
        unsigned long kflags = READ_ONCE(key->flags);
        short state = READ_ONCE(key->state);

        kenter("{%d}", key->serial);

        /* ignore keys not of this type */
        if (key->type != ctx->index_key.type) {
                kleave(" = 0 [!type]");
                return 0;
        }

        /* skip invalidated, revoked and expired keys */
        if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
                time64_t expiry = READ_ONCE(key->expiry);

                if (kflags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED))) {
                        ctx->result = ERR_PTR(-EKEYREVOKED);
                        kleave(" = %d [invrev]", ctx->skipped_ret);
                        goto skipped;
                }

                if (expiry && ctx->now >= expiry) {
                        if (!(ctx->flags & KEYRING_SEARCH_SKIP_EXPIRED))
                                ctx->result = ERR_PTR(-EKEYEXPIRED);
                        kleave(" = %d [expire]", ctx->skipped_ret);
                        goto skipped;
                }
        }

        /* keys that don't match */
        if (!ctx->match_data.cmp(key, &ctx->match_data)) {
                kleave(" = 0 [!match]");
                return 0;
        }

        /* key must have search permissions */
        if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
            key_task_permission(make_key_ref(key, ctx->possessed),
                                ctx->cred, KEY_NEED_SEARCH) < 0) {
                ctx->result = ERR_PTR(-EACCES);
                kleave(" = %d [!perm]", ctx->skipped_ret);
                goto skipped;
        }

        if (ctx->flags & KEYRING_SEARCH_DO_STATE_CHECK) {
                /* we set a different error code if we pass a negative key */
                if (state < 0) {
                        ctx->result = ERR_PTR(state);
                        kleave(" = %d [neg]", ctx->skipped_ret);
                        goto skipped;
                }
        }

        /* Found */
        ctx->result = make_key_ref(key, ctx->possessed);
        kleave(" = 1 [found]");
        return 1;

skipped:
        return ctx->skipped_ret;
}

/*
 * Search inside a keyring for a key.  We can search by walking to it
 * directly based on its index-key or we can iterate over the entire
 * tree looking for it, based on the match function.
 */
static int search_keyring(struct key *keyring, struct keyring_search_context *ctx)
{
        if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_DIRECT) {
                const void *object;

                object = assoc_array_find(&keyring->keys,
                                          &keyring_assoc_array_ops,
                                          &ctx->index_key);
                return object ? ctx->iterator(object, ctx) : 0;
        }
        return assoc_array_iterate(&keyring->keys, ctx->iterator, ctx);
}

/*
 * Search a tree of keyrings that point to other keyrings up to the maximum
 * depth.
 */
static bool search_nested_keyrings(struct key *keyring,
                                   struct keyring_search_context *ctx)
{
        struct {
                struct key *keyring;
                struct assoc_array_node *node;
                int slot;
        } stack[KEYRING_SEARCH_MAX_DEPTH];

        struct assoc_array_shortcut *shortcut;
        struct assoc_array_node *node;
        struct assoc_array_ptr *ptr;
        struct key *key;
        int sp = 0, slot;

        kenter("{%d},{%s,%s}",
               keyring->serial,
               ctx->index_key.type->name,
               ctx->index_key.description);

#define STATE_CHECKS (KEYRING_SEARCH_NO_STATE_CHECK | KEYRING_SEARCH_DO_STATE_CHECK)
        BUG_ON((ctx->flags & STATE_CHECKS) == 0 ||
               (ctx->flags & STATE_CHECKS) == STATE_CHECKS);

        if (ctx->index_key.description)
                key_set_index_key(&ctx->index_key);

        /* Check to see if this top-level keyring is what we are looking for
         * and whether it is valid or not.
         */
        if (ctx->match_data.lookup_type == KEYRING_SEARCH_LOOKUP_ITERATE ||
            keyring_compare_object(keyring, &ctx->index_key)) {
                ctx->skipped_ret = 2;
                switch (ctx->iterator(keyring_key_to_ptr(keyring), ctx)) {
                case 1:
                        goto found;
                case 2:
                        return false;
                default:
                        break;
                }
        }

        ctx->skipped_ret = 0;

        /* Start processing a new keyring */
descend_to_keyring:
        kdebug("descend to %d", keyring->serial);
        if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED)))
                goto not_this_keyring;

        /* Search through the keys in this keyring before its searching its
         * subtrees.
         */
        if (search_keyring(keyring, ctx))
                goto found;

        /* Then manually iterate through the keyrings nested in this one.
         *
         * Start from the root node of the index tree.  Because of the way the
         * hash function has been set up, keyrings cluster on the leftmost
         * branch of the root node (root slot 0) or in the root node itself.
         * Non-keyrings avoid the leftmost branch of the root entirely (root
         * slots 1-15).
         */
        if (!(ctx->flags & KEYRING_SEARCH_RECURSE))
                goto not_this_keyring;

        ptr = READ_ONCE(keyring->keys.root);
        if (!ptr)
                goto not_this_keyring;

        if (assoc_array_ptr_is_shortcut(ptr)) {
                /* If the root is a shortcut, either the keyring only contains
                 * keyring pointers (everything clusters behind root slot 0) or
                 * doesn't contain any keyring pointers.
                 */
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                if ((shortcut->index_key[0] & ASSOC_ARRAY_FAN_MASK) != 0)
                        goto not_this_keyring;

                ptr = READ_ONCE(shortcut->next_node);
                node = assoc_array_ptr_to_node(ptr);
                goto begin_node;
        }

        node = assoc_array_ptr_to_node(ptr);
        ptr = node->slots[0];
        if (!assoc_array_ptr_is_meta(ptr))
                goto begin_node;

descend_to_node:
        /* Descend to a more distal node in this keyring's content tree and go
         * through that.
         */
        kdebug("descend");
        if (assoc_array_ptr_is_shortcut(ptr)) {
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                ptr = READ_ONCE(shortcut->next_node);
                BUG_ON(!assoc_array_ptr_is_node(ptr));
        }
        node = assoc_array_ptr_to_node(ptr);

begin_node:
        kdebug("begin_node");
        slot = 0;
ascend_to_node:
        /* Go through the slots in a node */
        for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) {
                ptr = READ_ONCE(node->slots[slot]);

                if (assoc_array_ptr_is_meta(ptr)) {
                        if (node->back_pointer ||
                            assoc_array_ptr_is_shortcut(ptr))
                                goto descend_to_node;
                }

                if (!keyring_ptr_is_keyring(ptr))
                        continue;

                key = keyring_ptr_to_key(ptr);

                if (sp >= KEYRING_SEARCH_MAX_DEPTH) {
                        if (ctx->flags & KEYRING_SEARCH_DETECT_TOO_DEEP) {
                                ctx->result = ERR_PTR(-ELOOP);
                                return false;
                        }
                        goto not_this_keyring;
                }

                /* Search a nested keyring */
                if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM) &&
                    key_task_permission(make_key_ref(key, ctx->possessed),
                                        ctx->cred, KEY_NEED_SEARCH) < 0)
                        continue;

                /* stack the current position */
                stack[sp].keyring = keyring;
                stack[sp].node = node;
                stack[sp].slot = slot;
                sp++;

                /* begin again with the new keyring */
                keyring = key;
                goto descend_to_keyring;
        }

        /* We've dealt with all the slots in the current node, so now we need
         * to ascend to the parent and continue processing there.
         */
        ptr = READ_ONCE(node->back_pointer);
        slot = node->parent_slot;

        if (ptr && assoc_array_ptr_is_shortcut(ptr)) {
                shortcut = assoc_array_ptr_to_shortcut(ptr);
                ptr = READ_ONCE(shortcut->back_pointer);
                slot = shortcut->parent_slot;
        }
        if (!ptr)
                goto not_this_keyring;
        node = assoc_array_ptr_to_node(ptr);
        slot++;

        /* If we've ascended to the root (zero backpointer), we must have just
         * finished processing the leftmost branch rather than the root slots -
         * so there can't be any more keyrings for us to find.
         */
        if (node->back_pointer) {
                kdebug("ascend %d", slot);
                goto ascend_to_node;
        }

        /* The keyring we're looking at was disqualified or didn't contain a
         * matching key.
         */
not_this_keyring:
        kdebug("not_this_keyring %d", sp);
        if (sp <= 0) {
                kleave(" = false");
                return false;
        }

        /* Resume the processing of a keyring higher up in the tree */
        sp--;
        keyring = stack[sp].keyring;
        node = stack[sp].node;
        slot = stack[sp].slot + 1;
        kdebug("ascend to %d [%d]", keyring->serial, slot);
        goto ascend_to_node;

        /* We found a viable match */
found:
        key = key_ref_to_ptr(ctx->result);
        key_check(key);
        if (!(ctx->flags & KEYRING_SEARCH_NO_UPDATE_TIME)) {
                key->last_used_at = ctx->now;
                keyring->last_used_at = ctx->now;
                while (sp > 0)
                        stack[--sp].keyring->last_used_at = ctx->now;
        }
        kleave(" = true");
        return true;
}

/**
 * keyring_search_rcu - Search a keyring tree for a matching key under RCU
 * @keyring_ref: A pointer to the keyring with possession indicator.
 * @ctx: The keyring search context.
 *
 * Search the supplied keyring tree for a key that matches the criteria given.
 * The root keyring and any linked keyrings must grant Search permission to the
 * caller to be searchable and keys can only be found if they too grant Search
 * to the caller. The possession flag on the root keyring pointer controls use
 * of the possessor bits in permissions checking of the entire tree.  In
 * addition, the LSM gets to forbid keyring searches and key matches.
 *
 * The search is performed as a breadth-then-depth search up to the prescribed
 * limit (KEYRING_SEARCH_MAX_DEPTH).  The caller must hold the RCU read lock to
 * prevent keyrings from being destroyed or rearranged whilst they are being
 * searched.
 *
 * Keys are matched to the type provided and are then filtered by the match
 * function, which is given the description to use in any way it sees fit.  The
 * match function may use any attributes of a key that it wishes to
 * determine the match.  Normally the match function from the key type would be
 * used.
 *
 * RCU can be used to prevent the keyring key lists from disappearing without
 * the need to take lots of locks.
 *
 * Returns a pointer to the found key and increments the key usage count if
 * successful; -EAGAIN if no matching keys were found, or if expired or revoked
 * keys were found; -ENOKEY if only negative keys were found; -ENOTDIR if the
 * specified keyring wasn't a keyring.
 *
 * In the case of a successful return, the possession attribute from
 * @keyring_ref is propagated to the returned key reference.
 */
key_ref_t keyring_search_rcu(key_ref_t keyring_ref,
                             struct keyring_search_context *ctx)
{
        struct key *keyring;
        long err;

        ctx->iterator = keyring_search_iterator;
        ctx->possessed = is_key_possessed(keyring_ref);
        ctx->result = ERR_PTR(-EAGAIN);

        keyring = key_ref_to_ptr(keyring_ref);
        key_check(keyring);

        if (keyring->type != &key_type_keyring)
                return ERR_PTR(-ENOTDIR);

        if (!(ctx->flags & KEYRING_SEARCH_NO_CHECK_PERM)) {
                err = key_task_permission(keyring_ref, ctx->cred, KEY_NEED_SEARCH);
                if (err < 0)
                        return ERR_PTR(err);
        }

        ctx->now = ktime_get_real_seconds();
        if (search_nested_keyrings(keyring, ctx))
                __key_get(key_ref_to_ptr(ctx->result));
        return ctx->result;
}

/**
 * keyring_search - Search the supplied keyring tree for a matching key
 * @keyring: The root of the keyring tree to be searched.
 * @type: The type of keyring we want to find.
 * @description: The name of the keyring we want to find.
 * @recurse: True to search the children of @keyring also
 *
 * As keyring_search_rcu() above, but using the current task's credentials and
 * type's default matching function and preferred search method.
 */
key_ref_t keyring_search(key_ref_t keyring,
                         struct key_type *type,
                         const char *description,
                         bool recurse)
{
        struct keyring_search_context ctx = {
                .index_key.type                = type,
                .index_key.description        = description,
                .index_key.desc_len        = strlen(description),
                .cred                        = current_cred(),
                .match_data.cmp                = key_default_cmp,
                .match_data.raw_data        = description,
                .match_data.lookup_type        = KEYRING_SEARCH_LOOKUP_DIRECT,
                .flags                        = KEYRING_SEARCH_DO_STATE_CHECK,
        };
        key_ref_t key;
        int ret;

        if (recurse)
                ctx.flags |= KEYRING_SEARCH_RECURSE;
        if (type->match_preparse) {
                ret = type->match_preparse(&ctx.match_data);
                if (ret < 0)
                        return ERR_PTR(ret);
        }

        rcu_read_lock();
        key = keyring_search_rcu(keyring, &ctx);
        rcu_read_unlock();

        if (type->match_free)
                type->match_free(&ctx.match_data);
        return key;
}
EXPORT_SYMBOL(keyring_search);

static struct key_restriction *keyring_restriction_alloc(
        key_restrict_link_func_t check)
{
        struct key_restriction *keyres =
                kzalloc(sizeof(struct key_restriction), GFP_KERNEL);

        if (!keyres)
                return ERR_PTR(-ENOMEM);

        keyres->check = check;

        return keyres;
}

/*
 * Semaphore to serialise restriction setup to prevent reference count
 * cycles through restriction key pointers.
 */
static DECLARE_RWSEM(keyring_serialise_restrict_sem);

/*
 * Check for restriction cycles that would prevent keyring garbage collection.
 * keyring_serialise_restrict_sem must be held.
 */
static bool keyring_detect_restriction_cycle(const struct key *dest_keyring,
                                             struct key_restriction *keyres)
{
        while (keyres && keyres->key &&
               keyres->key->type == &key_type_keyring) {
                if (keyres->key == dest_keyring)
                        return true;

                keyres = keyres->key->restrict_link;
        }

        return false;
}

/**
 * keyring_restrict - Look up and apply a restriction to a keyring
 * @keyring_ref: The keyring to be restricted
 * @type: The key type that will provide the restriction checker.
 * @restriction: The restriction options to apply to the keyring
 *
 * Look up a keyring and apply a restriction to it.  The restriction is managed
 * by the specific key type, but can be configured by the options specified in
 * the restriction string.
 */
int keyring_restrict(key_ref_t keyring_ref, const char *type,
                     const char *restriction)
{
        struct key *keyring;
        struct key_type *restrict_type = NULL;
        struct key_restriction *restrict_link;
        int ret = 0;

        keyring = key_ref_to_ptr(keyring_ref);
        key_check(keyring);

        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        if (!type) {
                restrict_link = keyring_restriction_alloc(restrict_link_reject);
        } else {
                restrict_type = key_type_lookup(type);

                if (IS_ERR(restrict_type))
                        return PTR_ERR(restrict_type);

                if (!restrict_type->lookup_restriction) {
                        ret = -ENOENT;
                        goto error;
                }

                restrict_link = restrict_type->lookup_restriction(restriction);
        }

        if (IS_ERR(restrict_link)) {
                ret = PTR_ERR(restrict_link);
                goto error;
        }

        down_write(&keyring->sem);
        down_write(&keyring_serialise_restrict_sem);

        if (keyring->restrict_link) {
                ret = -EEXIST;
        } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) {
                ret = -EDEADLK;
        } else {
                keyring->restrict_link = restrict_link;
                notify_key(keyring, NOTIFY_KEY_SETATTR, 0);
        }

        up_write(&keyring_serialise_restrict_sem);
        up_write(&keyring->sem);

        if (ret < 0) {
                key_put(restrict_link->key);
                kfree(restrict_link);
        }

error:
        if (restrict_type)
                key_type_put(restrict_type);

        return ret;
}
EXPORT_SYMBOL(keyring_restrict);

/*
 * Search the given keyring for a key that might be updated.
 *
 * The caller must guarantee that the keyring is a keyring and that the
 * permission is granted to modify the keyring as no check is made here.  The
 * caller must also hold a lock on the keyring semaphore.
 *
 * Returns a pointer to the found key with usage count incremented if
 * successful and returns NULL if not found.  Revoked and invalidated keys are
 * skipped over.
 *
 * If successful, the possession indicator is propagated from the keyring ref
 * to the returned key reference.
 */
key_ref_t find_key_to_update(key_ref_t keyring_ref,
                             const struct keyring_index_key *index_key)
{
        struct key *keyring, *key;
        const void *object;

        keyring = key_ref_to_ptr(keyring_ref);

        kenter("{%d},{%s,%s}",
               keyring->serial, index_key->type->name, index_key->description);

        object = assoc_array_find(&keyring->keys, &keyring_assoc_array_ops,
                                  index_key);

        if (object)
                goto found;

        kleave(" = NULL");
        return NULL;

found:
        key = keyring_ptr_to_key(object);
        if (key->flags & ((1 << KEY_FLAG_INVALIDATED) |
                          (1 << KEY_FLAG_REVOKED))) {
                kleave(" = NULL [x]");
                return NULL;
        }
        __key_get(key);
        kleave(" = {%d}", key->serial);
        return make_key_ref(key, is_key_possessed(keyring_ref));
}

/*
 * Find a keyring with the specified name.
 *
 * Only keyrings that have nonzero refcount, are not revoked, and are owned by a
 * user in the current user namespace are considered.  If @uid_keyring is %true,
 * the keyring additionally must have been allocated as a user or user session
 * keyring; otherwise, it must grant Search permission directly to the caller.
 *
 * Returns a pointer to the keyring with the keyring's refcount having being
 * incremented on success.  -ENOKEY is returned if a key could not be found.
 */
struct key *find_keyring_by_name(const char *name, bool uid_keyring)
{
        struct user_namespace *ns = current_user_ns();
        struct key *keyring;

        if (!name)
                return ERR_PTR(-EINVAL);

        read_lock(&keyring_name_lock);

        /* Search this hash bucket for a keyring with a matching name that
         * grants Search permission and that hasn't been revoked
         */
        list_for_each_entry(keyring, &ns->keyring_name_list, name_link) {
                if (!kuid_has_mapping(ns, keyring->user->uid))
                        continue;

                if (test_bit(KEY_FLAG_REVOKED, &keyring->flags))
                        continue;

                if (strcmp(keyring->description, name) != 0)
                        continue;

                if (uid_keyring) {
                        if (!test_bit(KEY_FLAG_UID_KEYRING,
                                      &keyring->flags))
                                continue;
                } else {
                        if (key_permission(make_key_ref(keyring, 0),
                                           KEY_NEED_SEARCH) < 0)
                                continue;
                }

                /* we've got a match but we might end up racing with
                 * key_cleanup() if the keyring is currently 'dead'
                 * (ie. it has a zero usage count) */
                if (!refcount_inc_not_zero(&keyring->usage))
                        continue;
                keyring->last_used_at = ktime_get_real_seconds();
                goto out;
        }

        keyring = ERR_PTR(-ENOKEY);
out:
        read_unlock(&keyring_name_lock);
        return keyring;
}

static int keyring_detect_cycle_iterator(const void *object,
                                         void *iterator_data)
{
        struct keyring_search_context *ctx = iterator_data;
        const struct key *key = keyring_ptr_to_key(object);

        kenter("{%d}", key->serial);

        /* We might get a keyring with matching index-key that is nonetheless a
         * different keyring. */
        if (key != ctx->match_data.raw_data)
                return 0;

        ctx->result = ERR_PTR(-EDEADLK);
        return 1;
}

/*
 * See if a cycle will be created by inserting acyclic tree B in acyclic
 * tree A at the topmost level (ie: as a direct child of A).
 *
 * Since we are adding B to A at the top level, checking for cycles should just
 * be a matter of seeing if node A is somewhere in tree B.
 */
static int keyring_detect_cycle(struct key *A, struct key *B)
{
        struct keyring_search_context ctx = {
                .index_key                = A->index_key,
                .match_data.raw_data        = A,
                .match_data.lookup_type = KEYRING_SEARCH_LOOKUP_DIRECT,
                .iterator                = keyring_detect_cycle_iterator,
                .flags                        = (KEYRING_SEARCH_NO_STATE_CHECK |
                                           KEYRING_SEARCH_NO_UPDATE_TIME |
                                           KEYRING_SEARCH_NO_CHECK_PERM |
                                           KEYRING_SEARCH_DETECT_TOO_DEEP |
                                           KEYRING_SEARCH_RECURSE),
        };

        rcu_read_lock();
        search_nested_keyrings(B, &ctx);
        rcu_read_unlock();
        return PTR_ERR(ctx.result) == -EAGAIN ? 0 : PTR_ERR(ctx.result);
}

/*
 * Lock keyring for link.
 */
int __key_link_lock(struct key *keyring,
                    const struct keyring_index_key *index_key)
        __acquires(&keyring->sem)
        __acquires(&keyring_serialise_link_lock)
{
        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);

        /* Serialise link/link calls to prevent parallel calls causing a cycle
         * when linking two keyring in opposite orders.
         */
        if (index_key->type == &key_type_keyring)
                mutex_lock(&keyring_serialise_link_lock);

        return 0;
}

/*
 * Lock keyrings for move (link/unlink combination).
 */
int __key_move_lock(struct key *l_keyring, struct key *u_keyring,
                    const struct keyring_index_key *index_key)
        __acquires(&l_keyring->sem)
        __acquires(&u_keyring->sem)
        __acquires(&keyring_serialise_link_lock)
{
        if (l_keyring->type != &key_type_keyring ||
            u_keyring->type != &key_type_keyring)
                return -ENOTDIR;

        /* We have to be very careful here to take the keyring locks in the
         * right order, lest we open ourselves to deadlocking against another
         * move operation.
         */
        if (l_keyring < u_keyring) {
                down_write(&l_keyring->sem);
                down_write_nested(&u_keyring->sem, 1);
        } else {
                down_write(&u_keyring->sem);
                down_write_nested(&l_keyring->sem, 1);
        }

        /* Serialise link/link calls to prevent parallel calls causing a cycle
         * when linking two keyring in opposite orders.
         */
        if (index_key->type == &key_type_keyring)
                mutex_lock(&keyring_serialise_link_lock);

        return 0;
}

/*
 * Preallocate memory so that a key can be linked into to a keyring.
 */
int __key_link_begin(struct key *keyring,
                     const struct keyring_index_key *index_key,
                     struct assoc_array_edit **_edit)
{
        struct assoc_array_edit *edit;
        int ret;

        kenter("%d,%s,%s,",
               keyring->serial, index_key->type->name, index_key->description);

        BUG_ON(index_key->desc_len == 0);
        BUG_ON(*_edit != NULL);

        *_edit = NULL;

        ret = -EKEYREVOKED;
        if (test_bit(KEY_FLAG_REVOKED, &keyring->flags))
                goto error;

        /* Create an edit script that will insert/replace the key in the
         * keyring tree.
         */
        edit = assoc_array_insert(&keyring->keys,
                                  &keyring_assoc_array_ops,
                                  index_key,
                                  NULL);
        if (IS_ERR(edit)) {
                ret = PTR_ERR(edit);
                goto error;
        }

        /* If we're not replacing a link in-place then we're going to need some
         * extra quota.
         */
        if (!edit->dead_leaf) {
                ret = key_payload_reserve(keyring,
                                          keyring->datalen + KEYQUOTA_LINK_BYTES);
                if (ret < 0)
                        goto error_cancel;
        }

        *_edit = edit;
        kleave(" = 0");
        return 0;

error_cancel:
        assoc_array_cancel_edit(edit);
error:
        kleave(" = %d", ret);
        return ret;
}

/*
 * Check already instantiated keys aren't going to be a problem.
 *
 * The caller must have called __key_link_begin(). Don't need to call this for
 * keys that were created since __key_link_begin() was called.
 */
int __key_link_check_live_key(struct key *keyring, struct key *key)
{
        if (key->type == &key_type_keyring)
                /* check that we aren't going to create a cycle by linking one
                 * keyring to another */
                return keyring_detect_cycle(keyring, key);
        return 0;
}

/*
 * Link a key into to a keyring.
 *
 * Must be called with __key_link_begin() having being called.  Discards any
 * already extant link to matching key if there is one, so that each keyring
 * holds at most one link to any given key of a particular type+description
 * combination.
 */
void __key_link(struct key *keyring, struct key *key,
                struct assoc_array_edit **_edit)
{
        __key_get(key);
        assoc_array_insert_set_object(*_edit, keyring_key_to_ptr(key));
        assoc_array_apply_edit(*_edit);
        *_edit = NULL;
        notify_key(keyring, NOTIFY_KEY_LINKED, key_serial(key));
}

/*
 * Finish linking a key into to a keyring.
 *
 * Must be called with __key_link_begin() having being called.
 */
void __key_link_end(struct key *keyring,
                    const struct keyring_index_key *index_key,
                    struct assoc_array_edit *edit)
        __releases(&keyring->sem)
        __releases(&keyring_serialise_link_lock)
{
        BUG_ON(index_key->type == NULL);
        kenter("%d,%s,", keyring->serial, index_key->type->name);

        if (edit) {
                if (!edit->dead_leaf) {
                        key_payload_reserve(keyring,
                                keyring->datalen - KEYQUOTA_LINK_BYTES);
                }
                assoc_array_cancel_edit(edit);
        }
        up_write(&keyring->sem);

        if (index_key->type == &key_type_keyring)
                mutex_unlock(&keyring_serialise_link_lock);
}

/*
 * Check addition of keys to restricted keyrings.
 */
static int __key_link_check_restriction(struct key *keyring, struct key *key)
{
        if (!keyring->restrict_link || !keyring->restrict_link->check)
                return 0;
        return keyring->restrict_link->check(keyring, key->type, &key->payload,
                                             keyring->restrict_link->key);
}

/**
 * key_link - Link a key to a keyring
 * @keyring: The keyring to make the link in.
 * @key: The key to link to.
 *
 * Make a link in a keyring to a key, such that the keyring holds a reference
 * on that key and the key can potentially be found by searching that keyring.
 *
 * This function will write-lock the keyring's semaphore and will consume some
 * of the user's key data quota to hold the link.
 *
 * Returns 0 if successful, -ENOTDIR if the keyring isn't a keyring,
 * -EKEYREVOKED if the keyring has been revoked, -ENFILE if the keyring is
 * full, -EDQUOT if there is insufficient key data quota remaining to add
 * another link or -ENOMEM if there's insufficient memory.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be made (the keyring should have Write permission and the key Link
 * permission).
 */
int key_link(struct key *keyring, struct key *key)
{
        struct assoc_array_edit *edit = NULL;
        int ret;

        kenter("{%d,%d}", keyring->serial, refcount_read(&keyring->usage));

        key_check(keyring);
        key_check(key);

        ret = __key_link_lock(keyring, &key->index_key);
        if (ret < 0)
                goto error;

        ret = __key_link_begin(keyring, &key->index_key, &edit);
        if (ret < 0)
                goto error_end;

        kdebug("begun {%d,%d}", keyring->serial, refcount_read(&keyring->usage));
        ret = __key_link_check_restriction(keyring, key);
        if (ret == 0)
                ret = __key_link_check_live_key(keyring, key);
        if (ret == 0)
                __key_link(keyring, key, &edit);

error_end:
        __key_link_end(keyring, &key->index_key, edit);
error:
        kleave(" = %d {%d,%d}", ret, keyring->serial, refcount_read(&keyring->usage));
        return ret;
}
EXPORT_SYMBOL(key_link);

/*
 * Lock a keyring for unlink.
 */
static int __key_unlink_lock(struct key *keyring)
        __acquires(&keyring->sem)
{
        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);
        return 0;
}

/*
 * Begin the process of unlinking a key from a keyring.
 */
static int __key_unlink_begin(struct key *keyring, struct key *key,
                              struct assoc_array_edit **_edit)
{
        struct assoc_array_edit *edit;

        BUG_ON(*_edit != NULL);

        edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops,
                                  &key->index_key);
        if (IS_ERR(edit))
                return PTR_ERR(edit);

        if (!edit)
                return -ENOENT;

        *_edit = edit;
        return 0;
}

/*
 * Apply an unlink change.
 */
static void __key_unlink(struct key *keyring, struct key *key,
                         struct assoc_array_edit **_edit)
{
        assoc_array_apply_edit(*_edit);
        notify_key(keyring, NOTIFY_KEY_UNLINKED, key_serial(key));
        *_edit = NULL;
        key_payload_reserve(keyring, keyring->datalen - KEYQUOTA_LINK_BYTES);
}

/*
 * Finish unlinking a key from to a keyring.
 */
static void __key_unlink_end(struct key *keyring,
                             struct key *key,
                             struct assoc_array_edit *edit)
        __releases(&keyring->sem)
{
        if (edit)
                assoc_array_cancel_edit(edit);
        up_write(&keyring->sem);
}

/**
 * key_unlink - Unlink the first link to a key from a keyring.
 * @keyring: The keyring to remove the link from.
 * @key: The key the link is to.
 *
 * Remove a link from a keyring to a key.
 *
 * This function will write-lock the keyring's semaphore.
 *
 * Returns 0 if successful, -ENOTDIR if the keyring isn't a keyring, -ENOENT if
 * the key isn't linked to by the keyring or -ENOMEM if there's insufficient
 * memory.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be removed (the keyring should have Write permission; no permissions are
 * required on the key).
 */
int key_unlink(struct key *keyring, struct key *key)
{
        struct assoc_array_edit *edit = NULL;
        int ret;

        key_check(keyring);
        key_check(key);

        ret = __key_unlink_lock(keyring);
        if (ret < 0)
                return ret;

        ret = __key_unlink_begin(keyring, key, &edit);
        if (ret == 0)
                __key_unlink(keyring, key, &edit);
        __key_unlink_end(keyring, key, edit);
        return ret;
}
EXPORT_SYMBOL(key_unlink);

/**
 * key_move - Move a key from one keyring to another
 * @key: The key to move
 * @from_keyring: The keyring to remove the link from.
 * @to_keyring: The keyring to make the link in.
 * @flags: Qualifying flags, such as KEYCTL_MOVE_EXCL.
 *
 * Make a link in @to_keyring to a key, such that the keyring holds a reference
 * on that key and the key can potentially be found by searching that keyring
 * whilst simultaneously removing a link to the key from @from_keyring.
 *
 * This function will write-lock both keyring's semaphores and will consume
 * some of the user's key data quota to hold the link on @to_keyring.
 *
 * Returns 0 if successful, -ENOTDIR if either keyring isn't a keyring,
 * -EKEYREVOKED if either keyring has been revoked, -ENFILE if the second
 * keyring is full, -EDQUOT if there is insufficient key data quota remaining
 * to add another link or -ENOMEM if there's insufficient memory.  If
 * KEYCTL_MOVE_EXCL is set, then -EEXIST will be returned if there's already a
 * matching key in @to_keyring.
 *
 * It is assumed that the caller has checked that it is permitted for a link to
 * be made (the keyring should have Write permission and the key Link
 * permission).
 */
int key_move(struct key *key,
             struct key *from_keyring,
             struct key *to_keyring,
             unsigned int flags)
{
        struct assoc_array_edit *from_edit = NULL, *to_edit = NULL;
        int ret;

        kenter("%d,%d,%d", key->serial, from_keyring->serial, to_keyring->serial);

        if (from_keyring == to_keyring)
                return 0;

        key_check(key);
        key_check(from_keyring);
        key_check(to_keyring);

        ret = __key_move_lock(from_keyring, to_keyring, &key->index_key);
        if (ret < 0)
                goto out;
        ret = __key_unlink_begin(from_keyring, key, &from_edit);
        if (ret < 0)
                goto error;
        ret = __key_link_begin(to_keyring, &key->index_key, &to_edit);
        if (ret < 0)
                goto error;

        ret = -EEXIST;
        if (to_edit->dead_leaf && (flags & KEYCTL_MOVE_EXCL))
                goto error;

        ret = __key_link_check_restriction(to_keyring, key);
        if (ret < 0)
                goto error;
        ret = __key_link_check_live_key(to_keyring, key);
        if (ret < 0)
                goto error;

        __key_unlink(from_keyring, key, &from_edit);
        __key_link(to_keyring, key, &to_edit);
error:
        __key_link_end(to_keyring, &key->index_key, to_edit);
        __key_unlink_end(from_keyring, key, from_edit);
out:
        kleave(" = %d", ret);
        return ret;
}
EXPORT_SYMBOL(key_move);

/**
 * keyring_clear - Clear a keyring
 * @keyring: The keyring to clear.
 *
 * Clear the contents of the specified keyring.
 *
 * Returns 0 if successful or -ENOTDIR if the keyring isn't a keyring.
 */
int keyring_clear(struct key *keyring)
{
        struct assoc_array_edit *edit;
        int ret;

        if (keyring->type != &key_type_keyring)
                return -ENOTDIR;

        down_write(&keyring->sem);

        edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
        if (IS_ERR(edit)) {
                ret = PTR_ERR(edit);
        } else {
                if (edit)
                        assoc_array_apply_edit(edit);
                notify_key(keyring, NOTIFY_KEY_CLEARED, 0);
                key_payload_reserve(keyring, 0);
                ret = 0;
        }

        up_write(&keyring->sem);
        return ret;
}
EXPORT_SYMBOL(keyring_clear);

/*
 * Dispose of the links from a revoked keyring.
 *
 * This is called with the key sem write-locked.
 */
static void keyring_revoke(struct key *keyring)
{
        struct assoc_array_edit *edit;

        edit = assoc_array_clear(&keyring->keys, &keyring_assoc_array_ops);
        if (!IS_ERR(edit)) {
                if (edit)
                        assoc_array_apply_edit(edit);
                key_payload_reserve(keyring, 0);
        }
}

static bool keyring_gc_select_iterator(void *object, void *iterator_data)
{
        struct key *key = keyring_ptr_to_key(object);
        time64_t *limit = iterator_data;

        if (key_is_dead(key, *limit))
                return false;
        key_get(key);
        return true;
}

static int keyring_gc_check_iterator(const void *object, void *iterator_data)
{
        const struct key *key = keyring_ptr_to_key(object);
        time64_t *limit = iterator_data;

        key_check(key);
        return key_is_dead(key, *limit);
}

/*
 * Garbage collect pointers from a keyring.
 *
 * Not called with any locks held.  The keyring's key struct will not be
 * deallocated under us as only our caller may deallocate it.
 */
void keyring_gc(struct key *keyring, time64_t limit)
{
        int result;

        kenter("%x{%s}", keyring->serial, keyring->description ?: "");

        if (keyring->flags & ((1 << KEY_FLAG_INVALIDATED) |
                              (1 << KEY_FLAG_REVOKED)))
                goto dont_gc;

        /* scan the keyring looking for dead keys */
        rcu_read_lock();
        result = assoc_array_iterate(&keyring->keys,
                                     keyring_gc_check_iterator, &limit);
        rcu_read_unlock();
        if (result == true)
                goto do_gc;

dont_gc:
        kleave(" [no gc]");
        return;

do_gc:
        down_write(&keyring->sem);
        assoc_array_gc(&keyring->keys, &keyring_assoc_array_ops,
                       keyring_gc_select_iterator, &limit);
        up_write(&keyring->sem);
        kleave(" [gc]");
}

/*
 * Garbage collect restriction pointers from a keyring.
 *
 * Keyring restrictions are associated with a key type, and must be cleaned
 * up if the key type is unregistered. The restriction is altered to always
 * reject additional keys so a keyring cannot be opened up by unregistering
 * a key type.
 *
 * Not called with any keyring locks held. The keyring's key struct will not
 * be deallocated under us as only our caller may deallocate it.
 *
 * The caller is required to hold key_types_sem and dead_type->sem. This is
 * fulfilled by key_gc_keytype() holding the locks on behalf of
 * key_garbage_collector(), which it invokes on a workqueue.
 */
void keyring_restriction_gc(struct key *keyring, struct key_type *dead_type)
{
        struct key_restriction *keyres;

        kenter("%x{%s}", keyring->serial, keyring->description ?: "");

        /*
         * keyring->restrict_link is only assigned at key allocation time
         * or with the key type locked, so the only values that could be
         * concurrently assigned to keyring->restrict_link are for key
         * types other than dead_type. Given this, it's ok to check
         * the key type before acquiring keyring->sem.
         */
        if (!dead_type || !keyring->restrict_link ||
            keyring->restrict_link->keytype != dead_type) {
                kleave(" [no restriction gc]");
                return;
        }

        /* Lock the keyring to ensure that a link is not in progress */
        down_write(&keyring->sem);

        keyres = keyring->restrict_link;

        keyres->check = restrict_link_reject;

        key_put(keyres->key);
        keyres->key = NULL;
        keyres->keytype = NULL;

        up_write(&keyring->sem);

        kleave(" [restriction gc]");
}

















































































   97 
   98 




   96 
   99 





   77 


   76 
   77 

   77 










































   16 








   85 


   85 




   85 




   84 






















   11 

   11 












   42 
   42 

















































   17 

   17 
   15 
















  163 

  164 
  164 
  163 
  164 
  163 
  164 












   70 
































































































   41 


























    5 

































































































    9 
    9 


    5 








    9 











    9 
    9 




    6 
    6 




    2 

    6 


    6 





    2 
    2 





    9 







    7 







    5 



    3 


    1 





    3 

    3 



    2 




    9 




    9 













    9 
    9 



    9 


    3 
    2 


    9 





    7 




    7 









    9 























    9 


    9 

    9 

    9 

    9 



    7 



    9 



    9 

    4 
    9 















































   12 

   12 





   12 
    8 




   12 










   11 

    3 



    3 













   12 


    3 

   11 

   12 

    3 






















    3 









    3 













    8 








    8 







    8 
    8 





















    8 



    8 
    2 

    8 

    8 





    2 


    8 








































   43 


   43 



   43 



   43 









   23 


   24 
   24 


   43 



































   43 
   43 






   41 



   43 

   43 


   42 






   43 
   43 


   42 
   21 
   21 

   21 
   21 

   43 




   42 

   42 









   12 









   12 







   12 


    3 





    3 



    3 



    3 









    3 
    2 

    2 

























    3 








    3 


    3 










    3 
   11 


   12 



































   12 
   12 



   12 





   12 
   12 






   11 






















   12 
   12 
   12 



   12 


   11 






















































































   31 





   31 
   31 






   22 




















   12 

















   12 












   12 







































































   12 

   12 

   12 




   11 









    2 




















   12 


   12 









    2 
    2 






















  381 
  376 










  378 

  330 
  351 

  378 






  379 
  377 

  381 
  381 


  381 


  378 

  381 

  380 








  379 




  251 


  251 












  379 















  378 
  380 


  380 





















    9 
    9 




    9 





    8 





    9 


    9 


    4 






    4 
    1 

    3 


    4 
    9 











    9 





    4 
    4 
    2 










    9 

















   42 


   42 
   42 

   42 


   42 







   41 



















   42 





   42 

   42 

   42 

   42 

   42 

   42 

   42 




   42 


   42 
   42 



   42 



   42 




   42 


   41 


   42 

   41 




   42 
   42 
   41 



   42 




   41 

   41 
   30 
   37 
   36 




   42 
   42 





   42 
   36 

   37 
   37 

   37 



   37 
   37 

   19 

   19 

   30 

   29 




   41 

   42 
   42 


   42 
   42 









   43 








   43 

   42 
   41 
    5 






   43 









   43 



















   42 
   38 
   38 
   38 
   38 


   38 


   42 






   42 
   41 


   42 


   43 


   43 
   41 

   37 
    5 

   42 
   43 

   43 





   42 






   43 

   43 
   43 




   43 





   43 
   43 


   42 

   43 


   34 



































  100 


  100 
  100 


   99 


   99 



   98 



  100 

   99 



   99 


   99 

   98 



  100 



  100 


   99 
    1 



  100 
   99 


   99 



   99 


  100 

  100 










   98 
   99 


   97 
   98 














   98 


   99 


   97 
   98 







   91 





   50 
   50 

   84 

   92 
   92 


   84 
   83 

   38 











   84 






   83 


















   92 























   92 
   92 
   90 

   92 
   92 




   92 





   83 











   51 













































   45 










   44 
   45 
   44 

   45 

   45 




   45 




   44 




   45 



   45 



   45 








   44 
   45 









   70 




   70 
   70 



   70 














   70 



   70 




   70 












   70 







   70 

































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Linux INET6 implementation
 *        Forwarding Information Database
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 *
 *        Changes:
 *        Yuji SEKIYA @USAGI:        Support default route on router node;
 *                                remove ip6_null_entry from the top of
 *                                routing table.
 *        Ville Nuorvala:                Fixed routing subtrees.
 */

#define pr_fmt(fmt) "IPv6: " fmt

#include <linux/bpf.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/net.h>
#include <linux/route.h>
#include <linux/netdevice.h>
#include <linux/in6.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/slab.h>

#include <net/ip.h>
#include <net/ipv6.h>
#include <net/ndisc.h>
#include <net/addrconf.h>
#include <net/lwtunnel.h>
#include <net/fib_notifier.h>

#include <net/ip_fib.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>

static struct kmem_cache *fib6_node_kmem __read_mostly;

struct fib6_cleaner {
        struct fib6_walker w;
        struct net *net;
        int (*func)(struct fib6_info *, void *arg);
        int sernum;
        void *arg;
        bool skip_notify;
};

#ifdef CONFIG_IPV6_SUBTREES
#define FWS_INIT FWS_S
#else
#define FWS_INIT FWS_L
#endif

static struct fib6_info *fib6_find_prefix(struct net *net,
                                         struct fib6_table *table,
                                         struct fib6_node *fn);
static struct fib6_node *fib6_repair_tree(struct net *net,
                                          struct fib6_table *table,
                                          struct fib6_node *fn);
static int fib6_walk(struct net *net, struct fib6_walker *w);
static int fib6_walk_continue(struct fib6_walker *w);

/*
 *        A routing update causes an increase of the serial number on the
 *        affected subtree. This allows for cached routes to be asynchronously
 *        tested when modifications are made to the destination cache as a
 *        result of redirects, path MTU changes, etc.
 */

static void fib6_gc_timer_cb(struct timer_list *t);

#define FOR_WALKERS(net, w) \
        list_for_each_entry(w, &(net)->ipv6.fib6_walkers, lh)

static void fib6_walker_link(struct net *net, struct fib6_walker *w)
{
        write_lock_bh(&net->ipv6.fib6_walker_lock);
        list_add(&w->lh, &net->ipv6.fib6_walkers);
        write_unlock_bh(&net->ipv6.fib6_walker_lock);
}

static void fib6_walker_unlink(struct net *net, struct fib6_walker *w)
{
        write_lock_bh(&net->ipv6.fib6_walker_lock);
        list_del(&w->lh);
        write_unlock_bh(&net->ipv6.fib6_walker_lock);
}

static int fib6_new_sernum(struct net *net)
{
        int new, old = atomic_read(&net->ipv6.fib6_sernum);

        do {
                new = old < INT_MAX ? old + 1 : 1;
        } while (!atomic_try_cmpxchg(&net->ipv6.fib6_sernum, &old, new));

        return new;
}

enum {
        FIB6_NO_SERNUM_CHANGE = 0,
};

void fib6_update_sernum(struct net *net, struct fib6_info *f6i)
{
        struct fib6_node *fn;

        fn = rcu_dereference_protected(f6i->fib6_node,
                        lockdep_is_held(&f6i->fib6_table->tb6_lock));
        if (fn)
                WRITE_ONCE(fn->fn_sernum, fib6_new_sernum(net));
}

/*
 *        Auxiliary address test functions for the radix tree.
 *
 *        These assume a 32bit processor (although it will work on
 *        64bit processors)
 */

/*
 *        test bit
 */
#if defined(__LITTLE_ENDIAN)
# define BITOP_BE32_SWIZZLE        (0x1F & ~7)
#else
# define BITOP_BE32_SWIZZLE        0
#endif

static __be32 addr_bit_set(const void *token, int fn_bit)
{
        const __be32 *addr = token;
        /*
         * Here,
         *        1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)
         * is optimized version of
         *        htonl(1 << ((~fn_bit)&0x1F))
         * See include/asm-generic/bitops/le.h.
         */
        return (__force __be32)(1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)) &
               addr[fn_bit >> 5];
}

struct fib6_info *fib6_info_alloc(gfp_t gfp_flags, bool with_fib6_nh)
{
        struct fib6_info *f6i;
        size_t sz = sizeof(*f6i);

        if (with_fib6_nh)
                sz += sizeof(struct fib6_nh);

        f6i = kzalloc(sz, gfp_flags);
        if (!f6i)
                return NULL;

        /* fib6_siblings is a union with nh_list, so this initializes both */
        INIT_LIST_HEAD(&f6i->fib6_siblings);
        refcount_set(&f6i->fib6_ref, 1);

        INIT_HLIST_NODE(&f6i->gc_link);

        return f6i;
}

void fib6_info_destroy_rcu(struct rcu_head *head)
{
        struct fib6_info *f6i = container_of(head, struct fib6_info, rcu);

        WARN_ON(f6i->fib6_node);

        if (f6i->nh)
                nexthop_put(f6i->nh);
        else
                fib6_nh_release(f6i->fib6_nh);

        ip_fib_metrics_put(f6i->fib6_metrics);
        kfree(f6i);
}
EXPORT_SYMBOL_GPL(fib6_info_destroy_rcu);

static struct fib6_node *node_alloc(struct net *net)
{
        struct fib6_node *fn;

        fn = kmem_cache_zalloc(fib6_node_kmem, GFP_ATOMIC);
        if (fn)
                net->ipv6.rt6_stats->fib_nodes++;

        return fn;
}

static void node_free_immediate(struct net *net, struct fib6_node *fn)
{
        kmem_cache_free(fib6_node_kmem, fn);
        net->ipv6.rt6_stats->fib_nodes--;
}

static void node_free(struct net *net, struct fib6_node *fn)
{
        kfree_rcu(fn, rcu);
        net->ipv6.rt6_stats->fib_nodes--;
}

static void fib6_free_table(struct fib6_table *table)
{
        inetpeer_invalidate_tree(&table->tb6_peers);
        kfree(table);
}

static void fib6_link_table(struct net *net, struct fib6_table *tb)
{
        unsigned int h;

        /*
         * Initialize table lock at a single place to give lockdep a key,
         * tables aren't visible prior to being linked to the list.
         */
        spin_lock_init(&tb->tb6_lock);
        h = tb->tb6_id & (FIB6_TABLE_HASHSZ - 1);

        /*
         * No protection necessary, this is the only list mutatation
         * operation, tables never disappear once they exist.
         */
        hlist_add_head_rcu(&tb->tb6_hlist, &net->ipv6.fib_table_hash[h]);
}

#ifdef CONFIG_IPV6_MULTIPLE_TABLES

static struct fib6_table *fib6_alloc_table(struct net *net, u32 id)
{
        struct fib6_table *table;

        table = kzalloc(sizeof(*table), GFP_ATOMIC);
        if (table) {
                table->tb6_id = id;
                rcu_assign_pointer(table->tb6_root.leaf,
                                   net->ipv6.fib6_null_entry);
                table->tb6_root.fn_flags = RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
                inet_peer_base_init(&table->tb6_peers);
                INIT_HLIST_HEAD(&table->tb6_gc_hlist);
        }

        return table;
}

struct fib6_table *fib6_new_table(struct net *net, u32 id)
{
        struct fib6_table *tb;

        if (id == 0)
                id = RT6_TABLE_MAIN;
        tb = fib6_get_table(net, id);
        if (tb)
                return tb;

        tb = fib6_alloc_table(net, id);
        if (tb)
                fib6_link_table(net, tb);

        return tb;
}
EXPORT_SYMBOL_GPL(fib6_new_table);

struct fib6_table *fib6_get_table(struct net *net, u32 id)
{
        struct fib6_table *tb;
        struct hlist_head *head;
        unsigned int h;

        if (id == 0)
                id = RT6_TABLE_MAIN;
        h = id & (FIB6_TABLE_HASHSZ - 1);
        rcu_read_lock();
        head = &net->ipv6.fib_table_hash[h];
        hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
                if (tb->tb6_id == id) {
                        rcu_read_unlock();
                        return tb;
                }
        }
        rcu_read_unlock();

        return NULL;
}
EXPORT_SYMBOL_GPL(fib6_get_table);

static void __net_init fib6_tables_init(struct net *net)
{
        fib6_link_table(net, net->ipv6.fib6_main_tbl);
        fib6_link_table(net, net->ipv6.fib6_local_tbl);
}
#else

struct fib6_table *fib6_new_table(struct net *net, u32 id)
{
        return fib6_get_table(net, id);
}

struct fib6_table *fib6_get_table(struct net *net, u32 id)
{
          return net->ipv6.fib6_main_tbl;
}

struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,
                                   const struct sk_buff *skb,
                                   int flags, pol_lookup_t lookup)
{
        struct rt6_info *rt;

        rt = pol_lookup_func(lookup,
                        net, net->ipv6.fib6_main_tbl, fl6, skb, flags);
        if (rt->dst.error == -EAGAIN) {
                ip6_rt_put_flags(rt, flags);
                rt = net->ipv6.ip6_null_entry;
                if (!(flags & RT6_LOOKUP_F_DST_NOREF))
                        dst_hold(&rt->dst);
        }

        return &rt->dst;
}

/* called with rcu lock held; no reference taken on fib6_info */
int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6,
                struct fib6_result *res, int flags)
{
        return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6,
                                 res, flags);
}

static void __net_init fib6_tables_init(struct net *net)
{
        fib6_link_table(net, net->ipv6.fib6_main_tbl);
}

#endif

unsigned int fib6_tables_seq_read(const struct net *net)
{
        unsigned int h, fib_seq = 0;

        rcu_read_lock();
        for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
                const struct hlist_head *head = &net->ipv6.fib_table_hash[h];
                const struct fib6_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb6_hlist)
                        fib_seq += READ_ONCE(tb->fib_seq);
        }
        rcu_read_unlock();

        return fib_seq;
}

static int call_fib6_entry_notifier(struct notifier_block *nb,
                                    enum fib_event_type event_type,
                                    struct fib6_info *rt,
                                    struct netlink_ext_ack *extack)
{
        struct fib6_entry_notifier_info info = {
                .info.extack = extack,
                .rt = rt,
        };

        return call_fib6_notifier(nb, event_type, &info.info);
}

static int call_fib6_multipath_entry_notifier(struct notifier_block *nb,
                                              enum fib_event_type event_type,
                                              struct fib6_info *rt,
                                              unsigned int nsiblings,
                                              struct netlink_ext_ack *extack)
{
        struct fib6_entry_notifier_info info = {
                .info.extack = extack,
                .rt = rt,
                .nsiblings = nsiblings,
        };

        return call_fib6_notifier(nb, event_type, &info.info);
}

int call_fib6_entry_notifiers(struct net *net,
                              enum fib_event_type event_type,
                              struct fib6_info *rt,
                              struct netlink_ext_ack *extack)
{
        struct fib6_entry_notifier_info info = {
                .info.extack = extack,
                .rt = rt,
        };

        WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
        return call_fib6_notifiers(net, event_type, &info.info);
}

int call_fib6_multipath_entry_notifiers(struct net *net,
                                        enum fib_event_type event_type,
                                        struct fib6_info *rt,
                                        unsigned int nsiblings,
                                        struct netlink_ext_ack *extack)
{
        struct fib6_entry_notifier_info info = {
                .info.extack = extack,
                .rt = rt,
                .nsiblings = nsiblings,
        };

        WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
        return call_fib6_notifiers(net, event_type, &info.info);
}

int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt)
{
        struct fib6_entry_notifier_info info = {
                .rt = rt,
                .nsiblings = rt->fib6_nsiblings,
        };

        WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
        return call_fib6_notifiers(net, FIB_EVENT_ENTRY_REPLACE, &info.info);
}

struct fib6_dump_arg {
        struct net *net;
        struct notifier_block *nb;
        struct netlink_ext_ack *extack;
};

static int fib6_rt_dump(struct fib6_info *rt, struct fib6_dump_arg *arg)
{
        enum fib_event_type fib_event = FIB_EVENT_ENTRY_REPLACE;
        int err;

        if (!rt || rt == arg->net->ipv6.fib6_null_entry)
                return 0;

        if (rt->fib6_nsiblings)
                err = call_fib6_multipath_entry_notifier(arg->nb, fib_event,
                                                         rt,
                                                         rt->fib6_nsiblings,
                                                         arg->extack);
        else
                err = call_fib6_entry_notifier(arg->nb, fib_event, rt,
                                               arg->extack);

        return err;
}

static int fib6_node_dump(struct fib6_walker *w)
{
        int err;

        err = fib6_rt_dump(w->leaf, w->args);
        w->leaf = NULL;
        return err;
}

static int fib6_table_dump(struct net *net, struct fib6_table *tb,
                           struct fib6_walker *w)
{
        int err;

        w->root = &tb->tb6_root;
        spin_lock_bh(&tb->tb6_lock);
        err = fib6_walk(net, w);
        spin_unlock_bh(&tb->tb6_lock);
        return err;
}

/* Called with rcu_read_lock() */
int fib6_tables_dump(struct net *net, struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        struct fib6_dump_arg arg;
        struct fib6_walker *w;
        unsigned int h;
        int err = 0;

        w = kzalloc(sizeof(*w), GFP_ATOMIC);
        if (!w)
                return -ENOMEM;

        w->func = fib6_node_dump;
        arg.net = net;
        arg.nb = nb;
        arg.extack = extack;
        w->args = &arg;

        for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
                struct hlist_head *head = &net->ipv6.fib_table_hash[h];
                struct fib6_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
                        err = fib6_table_dump(net, tb, w);
                        if (err)
                                goto out;
                }
        }

out:
        kfree(w);

        /* The tree traversal function should never return a positive value. */
        return err > 0 ? -EINVAL : err;
}

static int fib6_dump_node(struct fib6_walker *w)
{
        int res;
        struct fib6_info *rt;

        for_each_fib6_walker_rt(w) {
                res = rt6_dump_route(rt, w->args, w->skip_in_node);
                if (res >= 0) {
                        /* Frame is full, suspend walking */
                        w->leaf = rt;

                        /* We'll restart from this node, so if some routes were
                         * already dumped, skip them next time.
                         */
                        w->skip_in_node += res;

                        return 1;
                }
                w->skip_in_node = 0;

                /* Multipath routes are dumped in one route with the
                 * RTA_MULTIPATH attribute. Jump 'rt' to point to the
                 * last sibling of this route (no need to dump the
                 * sibling routes again)
                 */
                if (rt->fib6_nsiblings)
                        rt = list_last_entry(&rt->fib6_siblings,
                                             struct fib6_info,
                                             fib6_siblings);
        }
        w->leaf = NULL;
        return 0;
}

static void fib6_dump_end(struct netlink_callback *cb)
{
        struct net *net = sock_net(cb->skb->sk);
        struct fib6_walker *w = (void *)cb->args[2];

        if (w) {
                if (cb->args[4]) {
                        cb->args[4] = 0;
                        fib6_walker_unlink(net, w);
                }
                cb->args[2] = 0;
                kfree(w);
        }
        cb->done = (void *)cb->args[3];
        cb->args[1] = 3;
}

static int fib6_dump_done(struct netlink_callback *cb)
{
        fib6_dump_end(cb);
        return cb->done ? cb->done(cb) : 0;
}

static int fib6_dump_table(struct fib6_table *table, struct sk_buff *skb,
                           struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        struct fib6_walker *w;
        int res;

        w = (void *)cb->args[2];
        w->root = &table->tb6_root;

        if (cb->args[4] == 0) {
                w->count = 0;
                w->skip = 0;
                w->skip_in_node = 0;

                spin_lock_bh(&table->tb6_lock);
                res = fib6_walk(net, w);
                spin_unlock_bh(&table->tb6_lock);
                if (res > 0) {
                        cb->args[4] = 1;
                        cb->args[5] = READ_ONCE(w->root->fn_sernum);
                }
        } else {
                int sernum = READ_ONCE(w->root->fn_sernum);
                if (cb->args[5] != sernum) {
                        /* Begin at the root if the tree changed */
                        cb->args[5] = sernum;
                        w->state = FWS_INIT;
                        w->node = w->root;
                        w->skip = w->count;
                        w->skip_in_node = 0;
                } else
                        w->skip = 0;

                spin_lock_bh(&table->tb6_lock);
                res = fib6_walk_continue(w);
                spin_unlock_bh(&table->tb6_lock);
                if (res <= 0) {
                        fib6_walker_unlink(net, w);
                        cb->args[4] = 0;
                }
        }

        return res;
}

static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rt6_rtnl_dump_arg arg = {
                .filter.dump_exceptions = true,
                .filter.dump_routes = true,
                .filter.rtnl_held = false,
        };
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        unsigned int e = 0, s_e;
        struct hlist_head *head;
        struct fib6_walker *w;
        struct fib6_table *tb;
        unsigned int h, s_h;
        int err = 0;

        rcu_read_lock();
        if (cb->strict_check) {
                err = ip_valid_fib_dump_req(net, nlh, &arg.filter, cb);
                if (err < 0)
                        goto unlock;
        } else if (nlmsg_len(nlh) >= sizeof(struct rtmsg)) {
                struct rtmsg *rtm = nlmsg_data(nlh);

                if (rtm->rtm_flags & RTM_F_PREFIX)
                        arg.filter.flags = RTM_F_PREFIX;
        }

        w = (void *)cb->args[2];
        if (!w) {
                /* New dump:
                 *
                 * 1. allocate and initialize walker.
                 */
                w = kzalloc(sizeof(*w), GFP_ATOMIC);
                if (!w) {
                        err = -ENOMEM;
                        goto unlock;
                }
                w->func = fib6_dump_node;
                cb->args[2] = (long)w;

                /* 2. hook callback destructor.
                 */
                cb->args[3] = (long)cb->done;
                cb->done = fib6_dump_done;

        }

        arg.skb = skb;
        arg.cb = cb;
        arg.net = net;
        w->args = &arg;

        if (arg.filter.table_id) {
                tb = fib6_get_table(net, arg.filter.table_id);
                if (!tb) {
                        if (rtnl_msg_family(cb->nlh) != PF_INET6)
                                goto unlock;

                        NL_SET_ERR_MSG_MOD(cb->extack, "FIB table does not exist");
                        err = -ENOENT;
                        goto unlock;
                }

                if (!cb->args[0]) {
                        err = fib6_dump_table(tb, skb, cb);
                        if (!err)
                                cb->args[0] = 1;
                }
                goto unlock;
        }

        s_h = cb->args[0];
        s_e = cb->args[1];

        for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_e = 0) {
                e = 0;
                head = &net->ipv6.fib_table_hash[h];
                hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
                        if (e < s_e)
                                goto next;
                        err = fib6_dump_table(tb, skb, cb);
                        if (err != 0)
                                goto out;
next:
                        e++;
                }
        }
out:
        cb->args[1] = e;
        cb->args[0] = h;

unlock:
        rcu_read_unlock();
        if (err <= 0)
                fib6_dump_end(cb);
        return err;
}

void fib6_metric_set(struct fib6_info *f6i, int metric, u32 val)
{
        if (!f6i)
                return;

        if (f6i->fib6_metrics == &dst_default_metrics) {
                struct dst_metrics *p = kzalloc(sizeof(*p), GFP_ATOMIC);

                if (!p)
                        return;

                refcount_set(&p->refcnt, 1);
                f6i->fib6_metrics = p;
        }

        f6i->fib6_metrics->metrics[metric - 1] = val;
}

/*
 *        Routing Table
 *
 *        return the appropriate node for a routing tree "add" operation
 *        by either creating and inserting or by returning an existing
 *        node.
 */

static struct fib6_node *fib6_add_1(struct net *net,
                                    struct fib6_table *table,
                                    struct fib6_node *root,
                                    struct in6_addr *addr, int plen,
                                    int offset, int allow_create,
                                    int replace_required,
                                    struct netlink_ext_ack *extack)
{
        struct fib6_node *fn, *in, *ln;
        struct fib6_node *pn = NULL;
        struct rt6key *key;
        int        bit;
        __be32        dir = 0;

        /* insert node in tree */

        fn = root;

        do {
                struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
                                            lockdep_is_held(&table->tb6_lock));
                key = (struct rt6key *)((u8 *)leaf + offset);

                /*
                 *        Prefix match
                 */
                if (plen < fn->fn_bit ||
                    !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
                        if (!allow_create) {
                                if (replace_required) {
                                        NL_SET_ERR_MSG(extack,
                                                       "Can not replace route - no match found");
                                        pr_warn("Can't replace route, no match found\n");
                                        return ERR_PTR(-ENOENT);
                                }
                                pr_warn("NLM_F_CREATE should be set when creating new route\n");
                        }
                        goto insert_above;
                }

                /*
                 *        Exact match ?
                 */

                if (plen == fn->fn_bit) {
                        /* clean up an intermediate node */
                        if (!(fn->fn_flags & RTN_RTINFO)) {
                                RCU_INIT_POINTER(fn->leaf, NULL);
                                fib6_info_release(leaf);
                        /* remove null_entry in the root node */
                        } else if (fn->fn_flags & RTN_TL_ROOT &&
                                   rcu_access_pointer(fn->leaf) ==
                                   net->ipv6.fib6_null_entry) {
                                RCU_INIT_POINTER(fn->leaf, NULL);
                        }

                        return fn;
                }

                /*
                 *        We have more bits to go
                 */

                /* Try to walk down on tree. */
                dir = addr_bit_set(addr, fn->fn_bit);
                pn = fn;
                fn = dir ?
                     rcu_dereference_protected(fn->right,
                                        lockdep_is_held(&table->tb6_lock)) :
                     rcu_dereference_protected(fn->left,
                                        lockdep_is_held(&table->tb6_lock));
        } while (fn);

        if (!allow_create) {
                /* We should not create new node because
                 * NLM_F_REPLACE was specified without NLM_F_CREATE
                 * I assume it is safe to require NLM_F_CREATE when
                 * REPLACE flag is used! Later we may want to remove the
                 * check for replace_required, because according
                 * to netlink specification, NLM_F_CREATE
                 * MUST be specified if new route is created.
                 * That would keep IPv6 consistent with IPv4
                 */
                if (replace_required) {
                        NL_SET_ERR_MSG(extack,
                                       "Can not replace route - no match found");
                        pr_warn("Can't replace route, no match found\n");
                        return ERR_PTR(-ENOENT);
                }
                pr_warn("NLM_F_CREATE should be set when creating new route\n");
        }
        /*
         *        We walked to the bottom of tree.
         *        Create new leaf node without children.
         */

        ln = node_alloc(net);

        if (!ln)
                return ERR_PTR(-ENOMEM);
        ln->fn_bit = plen;
        RCU_INIT_POINTER(ln->parent, pn);

        if (dir)
                rcu_assign_pointer(pn->right, ln);
        else
                rcu_assign_pointer(pn->left, ln);

        return ln;


insert_above:
        /*
         * split since we don't have a common prefix anymore or
         * we have a less significant route.
         * we've to insert an intermediate node on the list
         * this new node will point to the one we need to create
         * and the current
         */

        pn = rcu_dereference_protected(fn->parent,
                                       lockdep_is_held(&table->tb6_lock));

        /* find 1st bit in difference between the 2 addrs.

           See comment in __ipv6_addr_diff: bit may be an invalid value,
           but if it is >= plen, the value is ignored in any case.
         */

        bit = __ipv6_addr_diff(addr, &key->addr, sizeof(*addr));

        /*
         *                (intermediate)[in]
         *                  /           \
         *        (new leaf node)[ln] (old node)[fn]
         */
        if (plen > bit) {
                in = node_alloc(net);
                ln = node_alloc(net);

                if (!in || !ln) {
                        if (in)
                                node_free_immediate(net, in);
                        if (ln)
                                node_free_immediate(net, ln);
                        return ERR_PTR(-ENOMEM);
                }

                /*
                 * new intermediate node.
                 * RTN_RTINFO will
                 * be off since that an address that chooses one of
                 * the branches would not match less specific routes
                 * in the other branch
                 */

                in->fn_bit = bit;

                RCU_INIT_POINTER(in->parent, pn);
                in->leaf = fn->leaf;
                fib6_info_hold(rcu_dereference_protected(in->leaf,
                                lockdep_is_held(&table->tb6_lock)));

                /* update parent pointer */
                if (dir)
                        rcu_assign_pointer(pn->right, in);
                else
                        rcu_assign_pointer(pn->left, in);

                ln->fn_bit = plen;

                RCU_INIT_POINTER(ln->parent, in);
                rcu_assign_pointer(fn->parent, in);

                if (addr_bit_set(addr, bit)) {
                        rcu_assign_pointer(in->right, ln);
                        rcu_assign_pointer(in->left, fn);
                } else {
                        rcu_assign_pointer(in->left, ln);
                        rcu_assign_pointer(in->right, fn);
                }
        } else { /* plen <= bit */

                /*
                 *                (new leaf node)[ln]
                 *                  /           \
                 *             (old node)[fn] NULL
                 */

                ln = node_alloc(net);

                if (!ln)
                        return ERR_PTR(-ENOMEM);

                ln->fn_bit = plen;

                RCU_INIT_POINTER(ln->parent, pn);

                if (addr_bit_set(&key->addr, plen))
                        RCU_INIT_POINTER(ln->right, fn);
                else
                        RCU_INIT_POINTER(ln->left, fn);

                rcu_assign_pointer(fn->parent, ln);

                if (dir)
                        rcu_assign_pointer(pn->right, ln);
                else
                        rcu_assign_pointer(pn->left, ln);
        }
        return ln;
}

static void __fib6_drop_pcpu_from(struct fib6_nh *fib6_nh,
                                  const struct fib6_info *match,
                                  const struct fib6_table *table)
{
        int cpu;

        if (!fib6_nh->rt6i_pcpu)
                return;

        rcu_read_lock();
        /* release the reference to this fib entry from
         * all of its cached pcpu routes
         */
        for_each_possible_cpu(cpu) {
                struct rt6_info **ppcpu_rt;
                struct rt6_info *pcpu_rt;

                ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);

                /* Paired with xchg() in rt6_get_pcpu_route() */
                pcpu_rt = READ_ONCE(*ppcpu_rt);

                /* only dropping the 'from' reference if the cached route
                 * is using 'match'. The cached pcpu_rt->from only changes
                 * from a fib6_info to NULL (ip6_dst_destroy); it can never
                 * change from one fib6_info reference to another
                 */
                if (pcpu_rt && rcu_access_pointer(pcpu_rt->from) == match) {
                        struct fib6_info *from;

                        from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
                        fib6_info_release(from);
                }
        }
        rcu_read_unlock();
}

struct fib6_nh_pcpu_arg {
        struct fib6_info        *from;
        const struct fib6_table *table;
};

static int fib6_nh_drop_pcpu_from(struct fib6_nh *nh, void *_arg)
{
        struct fib6_nh_pcpu_arg *arg = _arg;

        __fib6_drop_pcpu_from(nh, arg->from, arg->table);
        return 0;
}

static void fib6_drop_pcpu_from(struct fib6_info *f6i,
                                const struct fib6_table *table)
{
        /* Make sure rt6_make_pcpu_route() wont add other percpu routes
         * while we are cleaning them here.
         */
        f6i->fib6_destroying = 1;
        mb(); /* paired with the cmpxchg() in rt6_make_pcpu_route() */

        if (f6i->nh) {
                struct fib6_nh_pcpu_arg arg = {
                        .from = f6i,
                        .table = table
                };

                nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_drop_pcpu_from,
                                         &arg);
        } else {
                struct fib6_nh *fib6_nh;

                fib6_nh = f6i->fib6_nh;
                __fib6_drop_pcpu_from(fib6_nh, f6i, table);
        }
}

static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
                          struct net *net)
{
        struct fib6_table *table = rt->fib6_table;

        /* Flush all cached dst in exception table */
        rt6_flush_exceptions(rt);
        fib6_drop_pcpu_from(rt, table);

        if (rt->nh && !list_empty(&rt->nh_list))
                list_del_init(&rt->nh_list);

        if (refcount_read(&rt->fib6_ref) != 1) {
                /* This route is used as dummy address holder in some split
                 * nodes. It is not leaked, but it still holds other resources,
                 * which must be released in time. So, scan ascendant nodes
                 * and replace dummy references to this route with references
                 * to still alive ones.
                 */
                while (fn) {
                        struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
                                            lockdep_is_held(&table->tb6_lock));
                        struct fib6_info *new_leaf;
                        if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {
                                new_leaf = fib6_find_prefix(net, table, fn);
                                fib6_info_hold(new_leaf);

                                rcu_assign_pointer(fn->leaf, new_leaf);
                                fib6_info_release(rt);
                        }
                        fn = rcu_dereference_protected(fn->parent,
                                    lockdep_is_held(&table->tb6_lock));
                }
        }

        fib6_clean_expires(rt);
        fib6_remove_gc_list(rt);
}

/*
 *        Insert routing information in a node.
 */

static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,
                            struct nl_info *info,
                            struct netlink_ext_ack *extack)
{
        struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
                                    lockdep_is_held(&rt->fib6_table->tb6_lock));
        struct fib6_info *iter = NULL;
        struct fib6_info __rcu **ins;
        struct fib6_info __rcu **fallback_ins = NULL;
        int replace = (info->nlh &&
                       (info->nlh->nlmsg_flags & NLM_F_REPLACE));
        int add = (!info->nlh ||
                   (info->nlh->nlmsg_flags & NLM_F_CREATE));
        int found = 0;
        bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
        bool notify_sibling_rt = false;
        u16 nlflags = NLM_F_EXCL;
        int err;

        if (info->nlh && (info->nlh->nlmsg_flags & NLM_F_APPEND))
                nlflags |= NLM_F_APPEND;

        ins = &fn->leaf;

        for (iter = leaf; iter;
             iter = rcu_dereference_protected(iter->fib6_next,
                                lockdep_is_held(&rt->fib6_table->tb6_lock))) {
                /*
                 *        Search for duplicates
                 */

                if (iter->fib6_metric == rt->fib6_metric) {
                        /*
                         *        Same priority level
                         */
                        if (info->nlh &&
                            (info->nlh->nlmsg_flags & NLM_F_EXCL))
                                return -EEXIST;

                        nlflags &= ~NLM_F_EXCL;
                        if (replace) {
                                if (rt_can_ecmp == rt6_qualify_for_ecmp(iter)) {
                                        found++;
                                        break;
                                }
                                fallback_ins = fallback_ins ?: ins;
                                goto next_iter;
                        }

                        if (rt6_duplicate_nexthop(iter, rt)) {
                                if (rt->fib6_nsiblings)
                                        rt->fib6_nsiblings = 0;
                                if (!(iter->fib6_flags & RTF_EXPIRES))
                                        return -EEXIST;
                                if (!(rt->fib6_flags & RTF_EXPIRES)) {
                                        fib6_clean_expires(iter);
                                        fib6_remove_gc_list(iter);
                                } else {
                                        fib6_set_expires(iter, rt->expires);
                                        fib6_add_gc_list(iter);
                                }

                                if (rt->fib6_pmtu)
                                        fib6_metric_set(iter, RTAX_MTU,
                                                        rt->fib6_pmtu);
                                return -EEXIST;
                        }
                        /* If we have the same destination and the same metric,
                         * but not the same gateway, then the route we try to
                         * add is sibling to this route, increment our counter
                         * of siblings, and later we will add our route to the
                         * list.
                         * Only static routes (which don't have flag
                         * RTF_EXPIRES) are used for ECMPv6.
                         *
                         * To avoid long list, we only had siblings if the
                         * route have a gateway.
                         */
                        if (rt_can_ecmp &&
                            rt6_qualify_for_ecmp(iter))
                                rt->fib6_nsiblings++;
                }

                if (iter->fib6_metric > rt->fib6_metric)
                        break;

next_iter:
                ins = &iter->fib6_next;
        }

        if (fallback_ins && !found) {
                /* No matching route with same ecmp-able-ness found, replace
                 * first matching route
                 */
                ins = fallback_ins;
                iter = rcu_dereference_protected(*ins,
                                    lockdep_is_held(&rt->fib6_table->tb6_lock));
                found++;
        }

        /* Reset round-robin state, if necessary */
        if (ins == &fn->leaf)
                fn->rr_ptr = NULL;

        /* Link this route to others same route. */
        if (rt->fib6_nsiblings) {
                unsigned int fib6_nsiblings;
                struct fib6_info *sibling, *temp_sibling;

                /* Find the first route that have the same metric */
                sibling = leaf;
                notify_sibling_rt = true;
                while (sibling) {
                        if (sibling->fib6_metric == rt->fib6_metric &&
                            rt6_qualify_for_ecmp(sibling)) {
                                list_add_tail_rcu(&rt->fib6_siblings,
                                                  &sibling->fib6_siblings);
                                break;
                        }
                        sibling = rcu_dereference_protected(sibling->fib6_next,
                                    lockdep_is_held(&rt->fib6_table->tb6_lock));
                        notify_sibling_rt = false;
                }
                /* For each sibling in the list, increment the counter of
                 * siblings. BUG() if counters does not match, list of siblings
                 * is broken!
                 */
                fib6_nsiblings = 0;
                list_for_each_entry_safe(sibling, temp_sibling,
                                         &rt->fib6_siblings, fib6_siblings) {
                        sibling->fib6_nsiblings++;
                        BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings);
                        fib6_nsiblings++;
                }
                BUG_ON(fib6_nsiblings != rt->fib6_nsiblings);
                rt6_multipath_rebalance(temp_sibling);
        }

        /*
         *        insert node
         */
        if (!replace) {
                if (!add)
                        pr_warn("NLM_F_CREATE should be set when creating new route\n");

add:
                nlflags |= NLM_F_CREATE;

                /* The route should only be notified if it is the first
                 * route in the node or if it is added as a sibling
                 * route to the first route in the node.
                 */
                if (!info->skip_notify_kernel &&
                    (notify_sibling_rt || ins == &fn->leaf)) {
                        enum fib_event_type fib_event;

                        if (notify_sibling_rt)
                                fib_event = FIB_EVENT_ENTRY_APPEND;
                        else
                                fib_event = FIB_EVENT_ENTRY_REPLACE;
                        err = call_fib6_entry_notifiers(info->nl_net,
                                                        fib_event, rt,
                                                        extack);
                        if (err) {
                                struct fib6_info *sibling, *next_sibling;

                                /* If the route has siblings, then it first
                                 * needs to be unlinked from them.
                                 */
                                if (!rt->fib6_nsiblings)
                                        return err;

                                list_for_each_entry_safe(sibling, next_sibling,
                                                         &rt->fib6_siblings,
                                                         fib6_siblings)
                                        sibling->fib6_nsiblings--;
                                rt->fib6_nsiblings = 0;
                                list_del_rcu(&rt->fib6_siblings);
                                rt6_multipath_rebalance(next_sibling);
                                return err;
                        }
                }

                rcu_assign_pointer(rt->fib6_next, iter);
                fib6_info_hold(rt);
                rcu_assign_pointer(rt->fib6_node, fn);
                rcu_assign_pointer(*ins, rt);
                if (!info->skip_notify)
                        inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
                info->nl_net->ipv6.rt6_stats->fib_rt_entries++;

                if (!(fn->fn_flags & RTN_RTINFO)) {
                        info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
                        fn->fn_flags |= RTN_RTINFO;
                }

        } else {
                int nsiblings;

                if (!found) {
                        if (add)
                                goto add;
                        pr_warn("NLM_F_REPLACE set, but no existing node found!\n");
                        return -ENOENT;
                }

                if (!info->skip_notify_kernel && ins == &fn->leaf) {
                        err = call_fib6_entry_notifiers(info->nl_net,
                                                        FIB_EVENT_ENTRY_REPLACE,
                                                        rt, extack);
                        if (err)
                                return err;
                }

                fib6_info_hold(rt);
                rcu_assign_pointer(rt->fib6_node, fn);
                rt->fib6_next = iter->fib6_next;
                rcu_assign_pointer(*ins, rt);
                if (!info->skip_notify)
                        inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);
                if (!(fn->fn_flags & RTN_RTINFO)) {
                        info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
                        fn->fn_flags |= RTN_RTINFO;
                }
                nsiblings = iter->fib6_nsiblings;
                iter->fib6_node = NULL;
                fib6_purge_rt(iter, fn, info->nl_net);
                if (rcu_access_pointer(fn->rr_ptr) == iter)
                        fn->rr_ptr = NULL;
                fib6_info_release(iter);

                if (nsiblings) {
                        /* Replacing an ECMP route, remove all siblings */
                        ins = &rt->fib6_next;
                        iter = rcu_dereference_protected(*ins,
                                    lockdep_is_held(&rt->fib6_table->tb6_lock));
                        while (iter) {
                                if (iter->fib6_metric > rt->fib6_metric)
                                        break;
                                if (rt6_qualify_for_ecmp(iter)) {
                                        *ins = iter->fib6_next;
                                        iter->fib6_node = NULL;
                                        fib6_purge_rt(iter, fn, info->nl_net);
                                        if (rcu_access_pointer(fn->rr_ptr) == iter)
                                                fn->rr_ptr = NULL;
                                        fib6_info_release(iter);
                                        nsiblings--;
                                        info->nl_net->ipv6.rt6_stats->fib_rt_entries--;
                                } else {
                                        ins = &iter->fib6_next;
                                }
                                iter = rcu_dereference_protected(*ins,
                                        lockdep_is_held(&rt->fib6_table->tb6_lock));
                        }
                        WARN_ON(nsiblings != 0);
                }
        }

        return 0;
}

static void fib6_start_gc(struct net *net, struct fib6_info *rt)
{
        if (!timer_pending(&net->ipv6.ip6_fib_timer) &&
            (rt->fib6_flags & RTF_EXPIRES))
                mod_timer(&net->ipv6.ip6_fib_timer,
                          jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
}

void fib6_force_start_gc(struct net *net)
{
        if (!timer_pending(&net->ipv6.ip6_fib_timer))
                mod_timer(&net->ipv6.ip6_fib_timer,
                          jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
}

static void __fib6_update_sernum_upto_root(struct fib6_info *rt,
                                           int sernum)
{
        struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,
                                lockdep_is_held(&rt->fib6_table->tb6_lock));

        /* paired with smp_rmb() in fib6_get_cookie_safe() */
        smp_wmb();
        while (fn) {
                WRITE_ONCE(fn->fn_sernum, sernum);
                fn = rcu_dereference_protected(fn->parent,
                                lockdep_is_held(&rt->fib6_table->tb6_lock));
        }
}

void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt)
{
        __fib6_update_sernum_upto_root(rt, fib6_new_sernum(net));
}

/* allow ipv4 to update sernum via ipv6_stub */
void fib6_update_sernum_stub(struct net *net, struct fib6_info *f6i)
{
        spin_lock_bh(&f6i->fib6_table->tb6_lock);
        fib6_update_sernum_upto_root(net, f6i);
        spin_unlock_bh(&f6i->fib6_table->tb6_lock);
}

/*
 *        Add routing information to the routing tree.
 *        <destination addr>/<source addr>
 *        with source addr info in sub-trees
 *        Need to own table->tb6_lock
 */

int fib6_add(struct fib6_node *root, struct fib6_info *rt,
             struct nl_info *info, struct netlink_ext_ack *extack)
{
        struct fib6_table *table = rt->fib6_table;
        struct fib6_node *fn;
#ifdef CONFIG_IPV6_SUBTREES
        struct fib6_node *pn = NULL;
#endif
        int err = -ENOMEM;
        int allow_create = 1;
        int replace_required = 0;

        if (info->nlh) {
                if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
                        allow_create = 0;
                if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
                        replace_required = 1;
        }
        if (!allow_create && !replace_required)
                pr_warn("RTM_NEWROUTE with no NLM_F_CREATE or NLM_F_REPLACE\n");

        fn = fib6_add_1(info->nl_net, table, root,
                        &rt->fib6_dst.addr, rt->fib6_dst.plen,
                        offsetof(struct fib6_info, fib6_dst), allow_create,
                        replace_required, extack);
        if (IS_ERR(fn)) {
                err = PTR_ERR(fn);
                fn = NULL;
                goto out;
        }

#ifdef CONFIG_IPV6_SUBTREES
        pn = fn;

        if (rt->fib6_src.plen) {
                struct fib6_node *sn;

                if (!rcu_access_pointer(fn->subtree)) {
                        struct fib6_node *sfn;

                        /*
                         * Create subtree.
                         *
                         *                fn[main tree]
                         *                |
                         *                sfn[subtree root]
                         *                   \
                         *                    sn[new leaf node]
                         */

                        /* Create subtree root node */
                        sfn = node_alloc(info->nl_net);
                        if (!sfn)
                                goto failure;

                        fib6_info_hold(info->nl_net->ipv6.fib6_null_entry);
                        rcu_assign_pointer(sfn->leaf,
                                           info->nl_net->ipv6.fib6_null_entry);
                        sfn->fn_flags = RTN_ROOT;

                        /* Now add the first leaf node to new subtree */

                        sn = fib6_add_1(info->nl_net, table, sfn,
                                        &rt->fib6_src.addr, rt->fib6_src.plen,
                                        offsetof(struct fib6_info, fib6_src),
                                        allow_create, replace_required, extack);

                        if (IS_ERR(sn)) {
                                /* If it is failed, discard just allocated
                                   root, and then (in failure) stale node
                                   in main tree.
                                 */
                                node_free_immediate(info->nl_net, sfn);
                                err = PTR_ERR(sn);
                                goto failure;
                        }

                        /* Now link new subtree to main tree */
                        rcu_assign_pointer(sfn->parent, fn);
                        rcu_assign_pointer(fn->subtree, sfn);
                } else {
                        sn = fib6_add_1(info->nl_net, table, FIB6_SUBTREE(fn),
                                        &rt->fib6_src.addr, rt->fib6_src.plen,
                                        offsetof(struct fib6_info, fib6_src),
                                        allow_create, replace_required, extack);

                        if (IS_ERR(sn)) {
                                err = PTR_ERR(sn);
                                goto failure;
                        }
                }

                if (!rcu_access_pointer(fn->leaf)) {
                        if (fn->fn_flags & RTN_TL_ROOT) {
                                /* put back null_entry for root node */
                                rcu_assign_pointer(fn->leaf,
                                            info->nl_net->ipv6.fib6_null_entry);
                        } else {
                                fib6_info_hold(rt);
                                rcu_assign_pointer(fn->leaf, rt);
                        }
                }
                fn = sn;
        }
#endif

        err = fib6_add_rt2node(fn, rt, info, extack);
        if (!err) {
                if (rt->nh)
                        list_add(&rt->nh_list, &rt->nh->f6i_list);
                __fib6_update_sernum_upto_root(rt, fib6_new_sernum(info->nl_net));

                if (rt->fib6_flags & RTF_EXPIRES)
                        fib6_add_gc_list(rt);

                fib6_start_gc(info->nl_net, rt);
        }

out:
        if (err) {
#ifdef CONFIG_IPV6_SUBTREES
                /*
                 * If fib6_add_1 has cleared the old leaf pointer in the
                 * super-tree leaf node we have to find a new one for it.
                 */
                if (pn != fn) {
                        struct fib6_info *pn_leaf =
                                rcu_dereference_protected(pn->leaf,
                                    lockdep_is_held(&table->tb6_lock));
                        if (pn_leaf == rt) {
                                pn_leaf = NULL;
                                RCU_INIT_POINTER(pn->leaf, NULL);
                                fib6_info_release(rt);
                        }
                        if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) {
                                pn_leaf = fib6_find_prefix(info->nl_net, table,
                                                           pn);
                                if (!pn_leaf)
                                        pn_leaf =
                                            info->nl_net->ipv6.fib6_null_entry;
                                fib6_info_hold(pn_leaf);
                                rcu_assign_pointer(pn->leaf, pn_leaf);
                        }
                }
#endif
                goto failure;
        } else if (fib6_requires_src(rt)) {
                fib6_routes_require_src_inc(info->nl_net);
        }
        return err;

failure:
        /* fn->leaf could be NULL and fib6_repair_tree() needs to be called if:
         * 1. fn is an intermediate node and we failed to add the new
         * route to it in both subtree creation failure and fib6_add_rt2node()
         * failure case.
         * 2. fn is the root node in the table and we fail to add the first
         * default route to it.
         */
        if (fn &&
            (!(fn->fn_flags & (RTN_RTINFO|RTN_ROOT)) ||
             (fn->fn_flags & RTN_TL_ROOT &&
              !rcu_access_pointer(fn->leaf))))
                fib6_repair_tree(info->nl_net, table, fn);
        return err;
}

/*
 *        Routing tree lookup
 *
 */

struct lookup_args {
        int                        offset;                /* key offset on fib6_info */
        const struct in6_addr        *addr;                /* search key                        */
};

static struct fib6_node *fib6_node_lookup_1(struct fib6_node *root,
                                            struct lookup_args *args)
{
        struct fib6_node *fn;
        __be32 dir;

        if (unlikely(args->offset == 0))
                return NULL;

        /*
         *        Descend on a tree
         */

        fn = root;

        for (;;) {
                struct fib6_node *next;

                dir = addr_bit_set(args->addr, fn->fn_bit);

                next = dir ? rcu_dereference(fn->right) :
                             rcu_dereference(fn->left);

                if (next) {
                        fn = next;
                        continue;
                }
                break;
        }

        while (fn) {
                struct fib6_node *subtree = FIB6_SUBTREE(fn);

                if (subtree || fn->fn_flags & RTN_RTINFO) {
                        struct fib6_info *leaf = rcu_dereference(fn->leaf);
                        struct rt6key *key;

                        if (!leaf)
                                goto backtrack;

                        key = (struct rt6key *) ((u8 *)leaf + args->offset);

                        if (ipv6_prefix_equal(&key->addr, args->addr, key->plen)) {
#ifdef CONFIG_IPV6_SUBTREES
                                if (subtree) {
                                        struct fib6_node *sfn;
                                        sfn = fib6_node_lookup_1(subtree,
                                                                 args + 1);
                                        if (!sfn)
                                                goto backtrack;
                                        fn = sfn;
                                }
#endif
                                if (fn->fn_flags & RTN_RTINFO)
                                        return fn;
                        }
                }
backtrack:
                if (fn->fn_flags & RTN_ROOT)
                        break;

                fn = rcu_dereference(fn->parent);
        }

        return NULL;
}

/* called with rcu_read_lock() held
 */
struct fib6_node *fib6_node_lookup(struct fib6_node *root,
                                   const struct in6_addr *daddr,
                                   const struct in6_addr *saddr)
{
        struct fib6_node *fn;
        struct lookup_args args[] = {
                {
                        .offset = offsetof(struct fib6_info, fib6_dst),
                        .addr = daddr,
                },
#ifdef CONFIG_IPV6_SUBTREES
                {
                        .offset = offsetof(struct fib6_info, fib6_src),
                        .addr = saddr,
                },
#endif
                {
                        .offset = 0,        /* sentinel */
                }
        };

        fn = fib6_node_lookup_1(root, daddr ? args : args + 1);
        if (!fn || fn->fn_flags & RTN_TL_ROOT)
                fn = root;

        return fn;
}

/*
 *        Get node with specified destination prefix (and source prefix,
 *        if subtrees are used)
 *        exact_match == true means we try to find fn with exact match of
 *        the passed in prefix addr
 *        exact_match == false means we try to find fn with longest prefix
 *        match of the passed in prefix addr. This is useful for finding fn
 *        for cached route as it will be stored in the exception table under
 *        the node with longest prefix length.
 */


static struct fib6_node *fib6_locate_1(struct fib6_node *root,
                                       const struct in6_addr *addr,
                                       int plen, int offset,
                                       bool exact_match)
{
        struct fib6_node *fn, *prev = NULL;

        for (fn = root; fn ; ) {
                struct fib6_info *leaf = rcu_dereference(fn->leaf);
                struct rt6key *key;

                /* This node is being deleted */
                if (!leaf) {
                        if (plen <= fn->fn_bit)
                                goto out;
                        else
                                goto next;
                }

                key = (struct rt6key *)((u8 *)leaf + offset);

                /*
                 *        Prefix match
                 */
                if (plen < fn->fn_bit ||
                    !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit))
                        goto out;

                if (plen == fn->fn_bit)
                        return fn;

                if (fn->fn_flags & RTN_RTINFO)
                        prev = fn;

next:
                /*
                 *        We have more bits to go
                 */
                if (addr_bit_set(addr, fn->fn_bit))
                        fn = rcu_dereference(fn->right);
                else
                        fn = rcu_dereference(fn->left);
        }
out:
        if (exact_match)
                return NULL;
        else
                return prev;
}

struct fib6_node *fib6_locate(struct fib6_node *root,
                              const struct in6_addr *daddr, int dst_len,
                              const struct in6_addr *saddr, int src_len,
                              bool exact_match)
{
        struct fib6_node *fn;

        fn = fib6_locate_1(root, daddr, dst_len,
                           offsetof(struct fib6_info, fib6_dst),
                           exact_match);

#ifdef CONFIG_IPV6_SUBTREES
        if (src_len) {
                WARN_ON(saddr == NULL);
                if (fn) {
                        struct fib6_node *subtree = FIB6_SUBTREE(fn);

                        if (subtree) {
                                fn = fib6_locate_1(subtree, saddr, src_len,
                                           offsetof(struct fib6_info, fib6_src),
                                           exact_match);
                        }
                }
        }
#endif

        if (fn && fn->fn_flags & RTN_RTINFO)
                return fn;

        return NULL;
}


/*
 *        Deletion
 *
 */

static struct fib6_info *fib6_find_prefix(struct net *net,
                                         struct fib6_table *table,
                                         struct fib6_node *fn)
{
        struct fib6_node *child_left, *child_right;

        if (fn->fn_flags & RTN_ROOT)
                return net->ipv6.fib6_null_entry;

        while (fn) {
                child_left = rcu_dereference_protected(fn->left,
                                    lockdep_is_held(&table->tb6_lock));
                child_right = rcu_dereference_protected(fn->right,
                                    lockdep_is_held(&table->tb6_lock));
                if (child_left)
                        return rcu_dereference_protected(child_left->leaf,
                                        lockdep_is_held(&table->tb6_lock));
                if (child_right)
                        return rcu_dereference_protected(child_right->leaf,
                                        lockdep_is_held(&table->tb6_lock));

                fn = FIB6_SUBTREE(fn);
        }
        return NULL;
}

/*
 *        Called to trim the tree of intermediate nodes when possible. "fn"
 *        is the node we want to try and remove.
 *        Need to own table->tb6_lock
 */

static struct fib6_node *fib6_repair_tree(struct net *net,
                                          struct fib6_table *table,
                                          struct fib6_node *fn)
{
        int children;
        int nstate;
        struct fib6_node *child;
        struct fib6_walker *w;
        int iter = 0;

        /* Set fn->leaf to null_entry for root node. */
        if (fn->fn_flags & RTN_TL_ROOT) {
                rcu_assign_pointer(fn->leaf, net->ipv6.fib6_null_entry);
                return fn;
        }

        for (;;) {
                struct fib6_node *fn_r = rcu_dereference_protected(fn->right,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_node *fn_l = rcu_dereference_protected(fn->left,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_node *pn = rcu_dereference_protected(fn->parent,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_node *pn_r = rcu_dereference_protected(pn->right,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_node *pn_l = rcu_dereference_protected(pn->left,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_info *fn_leaf = rcu_dereference_protected(fn->leaf,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_info *pn_leaf = rcu_dereference_protected(pn->leaf,
                                            lockdep_is_held(&table->tb6_lock));
                struct fib6_info *new_fn_leaf;

                pr_debug("fixing tree: plen=%d iter=%d\n", fn->fn_bit, iter);
                iter++;

                WARN_ON(fn->fn_flags & RTN_RTINFO);
                WARN_ON(fn->fn_flags & RTN_TL_ROOT);
                WARN_ON(fn_leaf);

                children = 0;
                child = NULL;
                if (fn_r) {
                        child = fn_r;
                        children |= 1;
                }
                if (fn_l) {
                        child = fn_l;
                        children |= 2;
                }

                if (children == 3 || FIB6_SUBTREE(fn)
#ifdef CONFIG_IPV6_SUBTREES
                    /* Subtree root (i.e. fn) may have one child */
                    || (children && fn->fn_flags & RTN_ROOT)
#endif
                    ) {
                        new_fn_leaf = fib6_find_prefix(net, table, fn);
#if RT6_DEBUG >= 2
                        if (!new_fn_leaf) {
                                WARN_ON(!new_fn_leaf);
                                new_fn_leaf = net->ipv6.fib6_null_entry;
                        }
#endif
                        fib6_info_hold(new_fn_leaf);
                        rcu_assign_pointer(fn->leaf, new_fn_leaf);
                        return pn;
                }

#ifdef CONFIG_IPV6_SUBTREES
                if (FIB6_SUBTREE(pn) == fn) {
                        WARN_ON(!(fn->fn_flags & RTN_ROOT));
                        RCU_INIT_POINTER(pn->subtree, NULL);
                        nstate = FWS_L;
                } else {
                        WARN_ON(fn->fn_flags & RTN_ROOT);
#endif
                        if (pn_r == fn)
                                rcu_assign_pointer(pn->right, child);
                        else if (pn_l == fn)
                                rcu_assign_pointer(pn->left, child);
#if RT6_DEBUG >= 2
                        else
                                WARN_ON(1);
#endif
                        if (child)
                                rcu_assign_pointer(child->parent, pn);
                        nstate = FWS_R;
#ifdef CONFIG_IPV6_SUBTREES
                }
#endif

                read_lock(&net->ipv6.fib6_walker_lock);
                FOR_WALKERS(net, w) {
                        if (!child) {
                                if (w->node == fn) {
                                        pr_debug("W %p adjusted by delnode 1, s=%d/%d\n",
                                                 w, w->state, nstate);
                                        w->node = pn;
                                        w->state = nstate;
                                }
                        } else {
                                if (w->node == fn) {
                                        w->node = child;
                                        if (children&2) {
                                                pr_debug("W %p adjusted by delnode 2, s=%d\n",
                                                         w, w->state);
                                                w->state = w->state >= FWS_R ? FWS_U : FWS_INIT;
                                        } else {
                                                pr_debug("W %p adjusted by delnode 2, s=%d\n",
                                                         w, w->state);
                                                w->state = w->state >= FWS_C ? FWS_U : FWS_INIT;
                                        }
                                }
                        }
                }
                read_unlock(&net->ipv6.fib6_walker_lock);

                node_free(net, fn);
                if (pn->fn_flags & RTN_RTINFO || FIB6_SUBTREE(pn))
                        return pn;

                RCU_INIT_POINTER(pn->leaf, NULL);
                fib6_info_release(pn_leaf);
                fn = pn;
        }
}

static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
                           struct fib6_info __rcu **rtp, struct nl_info *info)
{
        struct fib6_info *leaf, *replace_rt = NULL;
        struct fib6_walker *w;
        struct fib6_info *rt = rcu_dereference_protected(*rtp,
                                    lockdep_is_held(&table->tb6_lock));
        struct net *net = info->nl_net;
        bool notify_del = false;

        /* If the deleted route is the first in the node and it is not part of
         * a multipath route, then we need to replace it with the next route
         * in the node, if exists.
         */
        leaf = rcu_dereference_protected(fn->leaf,
                                         lockdep_is_held(&table->tb6_lock));
        if (leaf == rt && !rt->fib6_nsiblings) {
                if (rcu_access_pointer(rt->fib6_next))
                        replace_rt = rcu_dereference_protected(rt->fib6_next,
                                            lockdep_is_held(&table->tb6_lock));
                else
                        notify_del = true;
        }

        /* Unlink it */
        *rtp = rt->fib6_next;
        rt->fib6_node = NULL;
        net->ipv6.rt6_stats->fib_rt_entries--;
        net->ipv6.rt6_stats->fib_discarded_routes++;

        /* Reset round-robin state, if necessary */
        if (rcu_access_pointer(fn->rr_ptr) == rt)
                fn->rr_ptr = NULL;

        /* Remove this entry from other siblings */
        if (rt->fib6_nsiblings) {
                struct fib6_info *sibling, *next_sibling;

                /* The route is deleted from a multipath route. If this
                 * multipath route is the first route in the node, then we need
                 * to emit a delete notification. Otherwise, we need to skip
                 * the notification.
                 */
                if (rt->fib6_metric == leaf->fib6_metric &&
                    rt6_qualify_for_ecmp(leaf))
                        notify_del = true;
                list_for_each_entry_safe(sibling, next_sibling,
                                         &rt->fib6_siblings, fib6_siblings)
                        sibling->fib6_nsiblings--;
                rt->fib6_nsiblings = 0;
                list_del_rcu(&rt->fib6_siblings);
                rt6_multipath_rebalance(next_sibling);
        }

        /* Adjust walkers */
        read_lock(&net->ipv6.fib6_walker_lock);
        FOR_WALKERS(net, w) {
                if (w->state == FWS_C && w->leaf == rt) {
                        pr_debug("walker %p adjusted by delroute\n", w);
                        w->leaf = rcu_dereference_protected(rt->fib6_next,
                                            lockdep_is_held(&table->tb6_lock));
                        if (!w->leaf)
                                w->state = FWS_U;
                }
        }
        read_unlock(&net->ipv6.fib6_walker_lock);

        /* If it was last route, call fib6_repair_tree() to:
         * 1. For root node, put back null_entry as how the table was created.
         * 2. For other nodes, expunge its radix tree node.
         */
        if (!rcu_access_pointer(fn->leaf)) {
                if (!(fn->fn_flags & RTN_TL_ROOT)) {
                        fn->fn_flags &= ~RTN_RTINFO;
                        net->ipv6.rt6_stats->fib_route_nodes--;
                }
                fn = fib6_repair_tree(net, table, fn);
        }

        fib6_purge_rt(rt, fn, net);

        if (!info->skip_notify_kernel) {
                if (notify_del)
                        call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL,
                                                  rt, NULL);
                else if (replace_rt)
                        call_fib6_entry_notifiers_replace(net, replace_rt);
        }
        if (!info->skip_notify)
                inet6_rt_notify(RTM_DELROUTE, rt, info, 0);

        fib6_info_release(rt);
}

/* Need to own table->tb6_lock */
int fib6_del(struct fib6_info *rt, struct nl_info *info)
{
        struct net *net = info->nl_net;
        struct fib6_info __rcu **rtp;
        struct fib6_info __rcu **rtp_next;
        struct fib6_table *table;
        struct fib6_node *fn;

        if (rt == net->ipv6.fib6_null_entry)
                return -ENOENT;

        table = rt->fib6_table;
        fn = rcu_dereference_protected(rt->fib6_node,
                                       lockdep_is_held(&table->tb6_lock));
        if (!fn)
                return -ENOENT;

        WARN_ON(!(fn->fn_flags & RTN_RTINFO));

        /*
         *        Walk the leaf entries looking for ourself
         */

        for (rtp = &fn->leaf; *rtp; rtp = rtp_next) {
                struct fib6_info *cur = rcu_dereference_protected(*rtp,
                                        lockdep_is_held(&table->tb6_lock));
                if (rt == cur) {
                        if (fib6_requires_src(cur))
                                fib6_routes_require_src_dec(info->nl_net);
                        fib6_del_route(table, fn, rtp, info);
                        return 0;
                }
                rtp_next = &cur->fib6_next;
        }
        return -ENOENT;
}

/*
 *        Tree traversal function.
 *
 *        Certainly, it is not interrupt safe.
 *        However, it is internally reenterable wrt itself and fib6_add/fib6_del.
 *        It means, that we can modify tree during walking
 *        and use this function for garbage collection, clone pruning,
 *        cleaning tree when a device goes down etc. etc.
 *
 *        It guarantees that every node will be traversed,
 *        and that it will be traversed only once.
 *
 *        Callback function w->func may return:
 *        0 -> continue walking.
 *        positive value -> walking is suspended (used by tree dumps,
 *        and probably by gc, if it will be split to several slices)
 *        negative value -> terminate walking.
 *
 *        The function itself returns:
 *        0   -> walk is complete.
 *        >0  -> walk is incomplete (i.e. suspended)
 *        <0  -> walk is terminated by an error.
 *
 *        This function is called with tb6_lock held.
 */

static int fib6_walk_continue(struct fib6_walker *w)
{
        struct fib6_node *fn, *pn, *left, *right;

        /* w->root should always be table->tb6_root */
        WARN_ON_ONCE(!(w->root->fn_flags & RTN_TL_ROOT));

        for (;;) {
                fn = w->node;
                if (!fn)
                        return 0;

                switch (w->state) {
#ifdef CONFIG_IPV6_SUBTREES
                case FWS_S:
                        if (FIB6_SUBTREE(fn)) {
                                w->node = FIB6_SUBTREE(fn);
                                continue;
                        }
                        w->state = FWS_L;
                        fallthrough;
#endif
                case FWS_L:
                        left = rcu_dereference_protected(fn->left, 1);
                        if (left) {
                                w->node = left;
                                w->state = FWS_INIT;
                                continue;
                        }
                        w->state = FWS_R;
                        fallthrough;
                case FWS_R:
                        right = rcu_dereference_protected(fn->right, 1);
                        if (right) {
                                w->node = right;
                                w->state = FWS_INIT;
                                continue;
                        }
                        w->state = FWS_C;
                        w->leaf = rcu_dereference_protected(fn->leaf, 1);
                        fallthrough;
                case FWS_C:
                        if (w->leaf && fn->fn_flags & RTN_RTINFO) {
                                int err;

                                if (w->skip) {
                                        w->skip--;
                                        goto skip;
                                }

                                err = w->func(w);
                                if (err)
                                        return err;

                                w->count++;
                                continue;
                        }
skip:
                        w->state = FWS_U;
                        fallthrough;
                case FWS_U:
                        if (fn == w->root)
                                return 0;
                        pn = rcu_dereference_protected(fn->parent, 1);
                        left = rcu_dereference_protected(pn->left, 1);
                        right = rcu_dereference_protected(pn->right, 1);
                        w->node = pn;
#ifdef CONFIG_IPV6_SUBTREES
                        if (FIB6_SUBTREE(pn) == fn) {
                                WARN_ON(!(fn->fn_flags & RTN_ROOT));
                                w->state = FWS_L;
                                continue;
                        }
#endif
                        if (left == fn) {
                                w->state = FWS_R;
                                continue;
                        }
                        if (right == fn) {
                                w->state = FWS_C;
                                w->leaf = rcu_dereference_protected(w->node->leaf, 1);
                                continue;
                        }
#if RT6_DEBUG >= 2
                        WARN_ON(1);
#endif
                }
        }
}

static int fib6_walk(struct net *net, struct fib6_walker *w)
{
        int res;

        w->state = FWS_INIT;
        w->node = w->root;

        fib6_walker_link(net, w);
        res = fib6_walk_continue(w);
        if (res <= 0)
                fib6_walker_unlink(net, w);
        return res;
}

static int fib6_clean_node(struct fib6_walker *w)
{
        int res;
        struct fib6_info *rt;
        struct fib6_cleaner *c = container_of(w, struct fib6_cleaner, w);
        struct nl_info info = {
                .nl_net = c->net,
                .skip_notify = c->skip_notify,
        };

        if (c->sernum != FIB6_NO_SERNUM_CHANGE &&
            READ_ONCE(w->node->fn_sernum) != c->sernum)
                WRITE_ONCE(w->node->fn_sernum, c->sernum);

        if (!c->func) {
                WARN_ON_ONCE(c->sernum == FIB6_NO_SERNUM_CHANGE);
                w->leaf = NULL;
                return 0;
        }

        for_each_fib6_walker_rt(w) {
                res = c->func(rt, c->arg);
                if (res == -1) {
                        w->leaf = rt;
                        res = fib6_del(rt, &info);
                        if (res) {
#if RT6_DEBUG >= 2
                                pr_debug("%s: del failed: rt=%p@%p err=%d\n",
                                         __func__, rt,
                                         rcu_access_pointer(rt->fib6_node),
                                         res);
#endif
                                continue;
                        }
                        return 0;
                } else if (res == -2) {
                        if (WARN_ON(!rt->fib6_nsiblings))
                                continue;
                        rt = list_last_entry(&rt->fib6_siblings,
                                             struct fib6_info, fib6_siblings);
                        continue;
                }
                WARN_ON(res != 0);
        }
        w->leaf = rt;
        return 0;
}

/*
 *        Convenient frontend to tree walker.
 *
 *        func is called on each route.
 *                It may return -2 -> skip multipath route.
 *                              -1 -> delete this route.
 *                              0  -> continue walking
 */

static void fib6_clean_tree(struct net *net, struct fib6_node *root,
                            int (*func)(struct fib6_info *, void *arg),
                            int sernum, void *arg, bool skip_notify)
{
        struct fib6_cleaner c;

        c.w.root = root;
        c.w.func = fib6_clean_node;
        c.w.count = 0;
        c.w.skip = 0;
        c.w.skip_in_node = 0;
        c.func = func;
        c.sernum = sernum;
        c.arg = arg;
        c.net = net;
        c.skip_notify = skip_notify;

        fib6_walk(net, &c.w);
}

static void __fib6_clean_all(struct net *net,
                             int (*func)(struct fib6_info *, void *),
                             int sernum, void *arg, bool skip_notify)
{
        struct fib6_table *table;
        struct hlist_head *head;
        unsigned int h;

        rcu_read_lock();
        for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
                head = &net->ipv6.fib_table_hash[h];
                hlist_for_each_entry_rcu(table, head, tb6_hlist) {
                        spin_lock_bh(&table->tb6_lock);
                        fib6_clean_tree(net, &table->tb6_root,
                                        func, sernum, arg, skip_notify);
                        spin_unlock_bh(&table->tb6_lock);
                }
        }
        rcu_read_unlock();
}

void fib6_clean_all(struct net *net, int (*func)(struct fib6_info *, void *),
                    void *arg)
{
        __fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, false);
}

void fib6_clean_all_skip_notify(struct net *net,
                                int (*func)(struct fib6_info *, void *),
                                void *arg)
{
        __fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, true);
}

static void fib6_flush_trees(struct net *net)
{
        int new_sernum = fib6_new_sernum(net);

        __fib6_clean_all(net, NULL, new_sernum, NULL, false);
}

/*
 *        Garbage collection
 */

static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args)
{
        unsigned long now = jiffies;

        /*
         *        check addrconf expiration here.
         *        Routes are expired even if they are in use.
         */

        if (rt->fib6_flags & RTF_EXPIRES && rt->expires) {
                if (time_after(now, rt->expires)) {
                        pr_debug("expiring %p\n", rt);
                        return -1;
                }
                gc_args->more++;
        }

        /*        Also age clones in the exception table.
         *        Note, that clones are aged out
         *        only if they are not in use now.
         */
        rt6_age_exceptions(rt, gc_args, now);

        return 0;
}

static void fib6_gc_table(struct net *net,
                          struct fib6_table *tb6,
                          struct fib6_gc_args *gc_args)
{
        struct fib6_info *rt;
        struct hlist_node *n;
        struct nl_info info = {
                .nl_net = net,
                .skip_notify = false,
        };

        hlist_for_each_entry_safe(rt, n, &tb6->tb6_gc_hlist, gc_link)
                if (fib6_age(rt, gc_args) == -1)
                        fib6_del(rt, &info);
}

static void fib6_gc_all(struct net *net, struct fib6_gc_args *gc_args)
{
        struct fib6_table *table;
        struct hlist_head *head;
        unsigned int h;

        rcu_read_lock();
        for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
                head = &net->ipv6.fib_table_hash[h];
                hlist_for_each_entry_rcu(table, head, tb6_hlist) {
                        spin_lock_bh(&table->tb6_lock);

                        fib6_gc_table(net, table, gc_args);

                        spin_unlock_bh(&table->tb6_lock);
                }
        }
        rcu_read_unlock();
}

void fib6_run_gc(unsigned long expires, struct net *net, bool force)
{
        struct fib6_gc_args gc_args;
        unsigned long now;

        if (force) {
                spin_lock_bh(&net->ipv6.fib6_gc_lock);
        } else if (!spin_trylock_bh(&net->ipv6.fib6_gc_lock)) {
                mod_timer(&net->ipv6.ip6_fib_timer, jiffies + HZ);
                return;
        }
        gc_args.timeout = expires ? (int)expires :
                          net->ipv6.sysctl.ip6_rt_gc_interval;
        gc_args.more = 0;

        fib6_gc_all(net, &gc_args);
        now = jiffies;
        net->ipv6.ip6_rt_last_gc = now;

        if (gc_args.more)
                mod_timer(&net->ipv6.ip6_fib_timer,
                          round_jiffies(now
                                        + net->ipv6.sysctl.ip6_rt_gc_interval));
        else
                del_timer(&net->ipv6.ip6_fib_timer);
        spin_unlock_bh(&net->ipv6.fib6_gc_lock);
}

static void fib6_gc_timer_cb(struct timer_list *t)
{
        struct net *arg = from_timer(arg, t, ipv6.ip6_fib_timer);

        fib6_run_gc(0, arg, true);
}

static int __net_init fib6_net_init(struct net *net)
{
        size_t size = sizeof(struct hlist_head) * FIB6_TABLE_HASHSZ;
        int err;

        err = fib6_notifier_init(net);
        if (err)
                return err;

        /* Default to 3-tuple */
        net->ipv6.sysctl.multipath_hash_fields =
                FIB_MULTIPATH_HASH_FIELD_DEFAULT_MASK;

        spin_lock_init(&net->ipv6.fib6_gc_lock);
        rwlock_init(&net->ipv6.fib6_walker_lock);
        INIT_LIST_HEAD(&net->ipv6.fib6_walkers);
        timer_setup(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, 0);

        net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);
        if (!net->ipv6.rt6_stats)
                goto out_notifier;

        /* Avoid false sharing : Use at least a full cache line */
        size = max_t(size_t, size, L1_CACHE_BYTES);

        net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);
        if (!net->ipv6.fib_table_hash)
                goto out_rt6_stats;

        net->ipv6.fib6_main_tbl = kzalloc(sizeof(*net->ipv6.fib6_main_tbl),
                                          GFP_KERNEL);
        if (!net->ipv6.fib6_main_tbl)
                goto out_fib_table_hash;

        net->ipv6.fib6_main_tbl->tb6_id = RT6_TABLE_MAIN;
        rcu_assign_pointer(net->ipv6.fib6_main_tbl->tb6_root.leaf,
                           net->ipv6.fib6_null_entry);
        net->ipv6.fib6_main_tbl->tb6_root.fn_flags =
                RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
        inet_peer_base_init(&net->ipv6.fib6_main_tbl->tb6_peers);
        INIT_HLIST_HEAD(&net->ipv6.fib6_main_tbl->tb6_gc_hlist);

#ifdef CONFIG_IPV6_MULTIPLE_TABLES
        net->ipv6.fib6_local_tbl = kzalloc(sizeof(*net->ipv6.fib6_local_tbl),
                                           GFP_KERNEL);
        if (!net->ipv6.fib6_local_tbl)
                goto out_fib6_main_tbl;
        net->ipv6.fib6_local_tbl->tb6_id = RT6_TABLE_LOCAL;
        rcu_assign_pointer(net->ipv6.fib6_local_tbl->tb6_root.leaf,
                           net->ipv6.fib6_null_entry);
        net->ipv6.fib6_local_tbl->tb6_root.fn_flags =
                RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
        inet_peer_base_init(&net->ipv6.fib6_local_tbl->tb6_peers);
        INIT_HLIST_HEAD(&net->ipv6.fib6_local_tbl->tb6_gc_hlist);
#endif
        fib6_tables_init(net);

        return 0;

#ifdef CONFIG_IPV6_MULTIPLE_TABLES
out_fib6_main_tbl:
        kfree(net->ipv6.fib6_main_tbl);
#endif
out_fib_table_hash:
        kfree(net->ipv6.fib_table_hash);
out_rt6_stats:
        kfree(net->ipv6.rt6_stats);
out_notifier:
        fib6_notifier_exit(net);
        return -ENOMEM;
}

static void fib6_net_exit(struct net *net)
{
        unsigned int i;

        del_timer_sync(&net->ipv6.ip6_fib_timer);

        for (i = 0; i < FIB6_TABLE_HASHSZ; i++) {
                struct hlist_head *head = &net->ipv6.fib_table_hash[i];
                struct hlist_node *tmp;
                struct fib6_table *tb;

                hlist_for_each_entry_safe(tb, tmp, head, tb6_hlist) {
                        hlist_del(&tb->tb6_hlist);
                        fib6_free_table(tb);
                }
        }

        kfree(net->ipv6.fib_table_hash);
        kfree(net->ipv6.rt6_stats);
        fib6_notifier_exit(net);
}

static struct pernet_operations fib6_net_ops = {
        .init = fib6_net_init,
        .exit = fib6_net_exit,
};

static const struct rtnl_msg_handler fib6_rtnl_msg_handlers[] __initconst_or_module = {
        {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_GETROUTE,
         .dumpit = inet6_dump_fib,
         .flags = RTNL_FLAG_DUMP_UNLOCKED | RTNL_FLAG_DUMP_SPLIT_NLM_DONE},
};

int __init fib6_init(void)
{
        int ret = -ENOMEM;

        fib6_node_kmem = KMEM_CACHE(fib6_node,
                                    SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT);
        if (!fib6_node_kmem)
                goto out;

        ret = register_pernet_subsys(&fib6_net_ops);
        if (ret)
                goto out_kmem_cache_create;

        ret = rtnl_register_many(fib6_rtnl_msg_handlers);
        if (ret)
                goto out_unregister_subsys;

        __fib6_flush_trees = fib6_flush_trees;
out:
        return ret;

out_unregister_subsys:
        unregister_pernet_subsys(&fib6_net_ops);
out_kmem_cache_create:
        kmem_cache_destroy(fib6_node_kmem);
        goto out;
}

void fib6_gc_cleanup(void)
{
        unregister_pernet_subsys(&fib6_net_ops);
        kmem_cache_destroy(fib6_node_kmem);
}

#ifdef CONFIG_PROC_FS
static int ipv6_route_native_seq_show(struct seq_file *seq, void *v)
{
        struct fib6_info *rt = v;
        struct ipv6_route_iter *iter = seq->private;
        struct fib6_nh *fib6_nh = rt->fib6_nh;
        unsigned int flags = rt->fib6_flags;
        const struct net_device *dev;

        if (rt->nh)
                fib6_nh = nexthop_fib6_nh(rt->nh);

        seq_printf(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);

#ifdef CONFIG_IPV6_SUBTREES
        seq_printf(seq, "%pi6 %02x ", &rt->fib6_src.addr, rt->fib6_src.plen);
#else
        seq_puts(seq, "00000000000000000000000000000000 00 ");
#endif
        if (fib6_nh->fib_nh_gw_family) {
                flags |= RTF_GATEWAY;
                seq_printf(seq, "%pi6", &fib6_nh->fib_nh_gw6);
        } else {
                seq_puts(seq, "00000000000000000000000000000000");
        }

        dev = fib6_nh->fib_nh_dev;
        seq_printf(seq, " %08x %08x %08x %08x %8s\n",
                   rt->fib6_metric, refcount_read(&rt->fib6_ref), 0,
                   flags, dev ? dev->name : "");
        iter->w.leaf = NULL;
        return 0;
}

static int ipv6_route_yield(struct fib6_walker *w)
{
        struct ipv6_route_iter *iter = w->args;

        if (!iter->skip)
                return 1;

        do {
                iter->w.leaf = rcu_dereference_protected(
                                iter->w.leaf->fib6_next,
                                lockdep_is_held(&iter->tbl->tb6_lock));
                iter->skip--;
                if (!iter->skip && iter->w.leaf)
                        return 1;
        } while (iter->w.leaf);

        return 0;
}

static void ipv6_route_seq_setup_walk(struct ipv6_route_iter *iter,
                                      struct net *net)
{
        memset(&iter->w, 0, sizeof(iter->w));
        iter->w.func = ipv6_route_yield;
        iter->w.root = &iter->tbl->tb6_root;
        iter->w.state = FWS_INIT;
        iter->w.node = iter->w.root;
        iter->w.args = iter;
        iter->sernum = READ_ONCE(iter->w.root->fn_sernum);
        INIT_LIST_HEAD(&iter->w.lh);
        fib6_walker_link(net, &iter->w);
}

static struct fib6_table *ipv6_route_seq_next_table(struct fib6_table *tbl,
                                                    struct net *net)
{
        unsigned int h;
        struct hlist_node *node;

        if (tbl) {
                h = (tbl->tb6_id & (FIB6_TABLE_HASHSZ - 1)) + 1;
                node = rcu_dereference(hlist_next_rcu(&tbl->tb6_hlist));
        } else {
                h = 0;
                node = NULL;
        }

        while (!node && h < FIB6_TABLE_HASHSZ) {
                node = rcu_dereference(
                        hlist_first_rcu(&net->ipv6.fib_table_hash[h++]));
        }
        return hlist_entry_safe(node, struct fib6_table, tb6_hlist);
}

static void ipv6_route_check_sernum(struct ipv6_route_iter *iter)
{
        int sernum = READ_ONCE(iter->w.root->fn_sernum);

        if (iter->sernum != sernum) {
                iter->sernum = sernum;
                iter->w.state = FWS_INIT;
                iter->w.node = iter->w.root;
                WARN_ON(iter->w.skip);
                iter->w.skip = iter->w.count;
        }
}

static void *ipv6_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        int r;
        struct fib6_info *n;
        struct net *net = seq_file_net(seq);
        struct ipv6_route_iter *iter = seq->private;

        ++(*pos);
        if (!v)
                goto iter_table;

        n = rcu_dereference(((struct fib6_info *)v)->fib6_next);
        if (n)
                return n;

iter_table:
        ipv6_route_check_sernum(iter);
        spin_lock_bh(&iter->tbl->tb6_lock);
        r = fib6_walk_continue(&iter->w);
        spin_unlock_bh(&iter->tbl->tb6_lock);
        if (r > 0) {
                return iter->w.leaf;
        } else if (r < 0) {
                fib6_walker_unlink(net, &iter->w);
                return NULL;
        }
        fib6_walker_unlink(net, &iter->w);

        iter->tbl = ipv6_route_seq_next_table(iter->tbl, net);
        if (!iter->tbl)
                return NULL;

        ipv6_route_seq_setup_walk(iter, net);
        goto iter_table;
}

static void *ipv6_route_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(RCU)
{
        struct net *net = seq_file_net(seq);
        struct ipv6_route_iter *iter = seq->private;

        rcu_read_lock();
        iter->tbl = ipv6_route_seq_next_table(NULL, net);
        iter->skip = *pos;

        if (iter->tbl) {
                loff_t p = 0;

                ipv6_route_seq_setup_walk(iter, net);
                return ipv6_route_seq_next(seq, NULL, &p);
        } else {
                return NULL;
        }
}

static bool ipv6_route_iter_active(struct ipv6_route_iter *iter)
{
        struct fib6_walker *w = &iter->w;
        return w->node && !(w->state == FWS_U && w->node == w->root);
}

static void ipv6_route_native_seq_stop(struct seq_file *seq, void *v)
        __releases(RCU)
{
        struct net *net = seq_file_net(seq);
        struct ipv6_route_iter *iter = seq->private;

        if (ipv6_route_iter_active(iter))
                fib6_walker_unlink(net, &iter->w);

        rcu_read_unlock();
}

#if IS_BUILTIN(CONFIG_IPV6) && defined(CONFIG_BPF_SYSCALL)
static int ipv6_route_prog_seq_show(struct bpf_prog *prog,
                                    struct bpf_iter_meta *meta,
                                    void *v)
{
        struct bpf_iter__ipv6_route ctx;

        ctx.meta = meta;
        ctx.rt = v;
        return bpf_iter_run_prog(prog, &ctx);
}

static int ipv6_route_seq_show(struct seq_file *seq, void *v)
{
        struct ipv6_route_iter *iter = seq->private;
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;
        int ret;

        meta.seq = seq;
        prog = bpf_iter_get_info(&meta, false);
        if (!prog)
                return ipv6_route_native_seq_show(seq, v);

        ret = ipv6_route_prog_seq_show(prog, &meta, v);
        iter->w.leaf = NULL;

        return ret;
}

static void ipv6_route_seq_stop(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        if (!v) {
                meta.seq = seq;
                prog = bpf_iter_get_info(&meta, true);
                if (prog)
                        (void)ipv6_route_prog_seq_show(prog, &meta, v);
        }

        ipv6_route_native_seq_stop(seq, v);
}
#else
static int ipv6_route_seq_show(struct seq_file *seq, void *v)
{
        return ipv6_route_native_seq_show(seq, v);
}

static void ipv6_route_seq_stop(struct seq_file *seq, void *v)
{
        ipv6_route_native_seq_stop(seq, v);
}
#endif

const struct seq_operations ipv6_route_seq_ops = {
        .start        = ipv6_route_seq_start,
        .next        = ipv6_route_seq_next,
        .stop        = ipv6_route_seq_stop,
        .show        = ipv6_route_seq_show
};
#endif /* CONFIG_PROC_FS */






































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 













































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
// SPDX-License-Identifier: GPL-2.0
/*
 *
 * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved.
 *
 */

#include <linux/blkdev.h>
#include <linux/buffer_head.h>
#include <linux/fs.h>
#include <linux/kernel.h>

#include "debug.h"
#include "ntfs.h"
#include "ntfs_fs.h"

static const struct INDEX_NAMES {
        const __le16 *name;
        u8 name_len;
} s_index_names[INDEX_MUTEX_TOTAL] = {
        { I30_NAME, ARRAY_SIZE(I30_NAME) }, { SII_NAME, ARRAY_SIZE(SII_NAME) },
        { SDH_NAME, ARRAY_SIZE(SDH_NAME) }, { SO_NAME, ARRAY_SIZE(SO_NAME) },
        { SQ_NAME, ARRAY_SIZE(SQ_NAME) },   { SR_NAME, ARRAY_SIZE(SR_NAME) },
};

/*
 * cmp_fnames - Compare two names in index.
 *
 * if l1 != 0
 *   Both names are little endian on-disk ATTR_FILE_NAME structs.
 * else
 *   key1 - cpu_str, key2 - ATTR_FILE_NAME
 */
static int cmp_fnames(const void *key1, size_t l1, const void *key2, size_t l2,
                      const void *data)
{
        const struct ATTR_FILE_NAME *f2 = key2;
        const struct ntfs_sb_info *sbi = data;
        const struct ATTR_FILE_NAME *f1;
        u16 fsize2;
        bool both_case;

        if (l2 <= offsetof(struct ATTR_FILE_NAME, name))
                return -1;

        fsize2 = fname_full_size(f2);
        if (l2 < fsize2)
                return -1;

        both_case = f2->type != FILE_NAME_DOS && !sbi->options->nocase;
        if (!l1) {
                const struct le_str *s2 = (struct le_str *)&f2->name_len;

                /*
                 * If names are equal (case insensitive)
                 * try to compare it case sensitive.
                 */
                return ntfs_cmp_names_cpu(key1, s2, sbi->upcase, both_case);
        }

        f1 = key1;
        return ntfs_cmp_names(f1->name, f1->name_len, f2->name, f2->name_len,
                              sbi->upcase, both_case);
}

/*
 * cmp_uint - $SII of $Secure and $Q of Quota
 */
static int cmp_uint(const void *key1, size_t l1, const void *key2, size_t l2,
                    const void *data)
{
        const u32 *k1 = key1;
        const u32 *k2 = key2;

        if (l2 < sizeof(u32))
                return -1;

        if (*k1 < *k2)
                return -1;
        if (*k1 > *k2)
                return 1;
        return 0;
}

/*
 * cmp_sdh - $SDH of $Secure
 */
static int cmp_sdh(const void *key1, size_t l1, const void *key2, size_t l2,
                   const void *data)
{
        const struct SECURITY_KEY *k1 = key1;
        const struct SECURITY_KEY *k2 = key2;
        u32 t1, t2;

        if (l2 < sizeof(struct SECURITY_KEY))
                return -1;

        t1 = le32_to_cpu(k1->hash);
        t2 = le32_to_cpu(k2->hash);

        /* First value is a hash value itself. */
        if (t1 < t2)
                return -1;
        if (t1 > t2)
                return 1;

        /* Second value is security Id. */
        if (data) {
                t1 = le32_to_cpu(k1->sec_id);
                t2 = le32_to_cpu(k2->sec_id);
                if (t1 < t2)
                        return -1;
                if (t1 > t2)
                        return 1;
        }

        return 0;
}

/*
 * cmp_uints - $O of ObjId and "$R" for Reparse.
 */
static int cmp_uints(const void *key1, size_t l1, const void *key2, size_t l2,
                     const void *data)
{
        const __le32 *k1 = key1;
        const __le32 *k2 = key2;
        size_t count;

        if ((size_t)data == 1) {
                /*
                 * ni_delete_all -> ntfs_remove_reparse ->
                 * delete all with this reference.
                 * k1, k2 - pointers to REPARSE_KEY
                 */

                k1 += 1; // Skip REPARSE_KEY.ReparseTag
                k2 += 1; // Skip REPARSE_KEY.ReparseTag
                if (l2 <= sizeof(int))
                        return -1;
                l2 -= sizeof(int);
                if (l1 <= sizeof(int))
                        return 1;
                l1 -= sizeof(int);
        }

        if (l2 < sizeof(int))
                return -1;

        for (count = min(l1, l2) >> 2; count > 0; --count, ++k1, ++k2) {
                u32 t1 = le32_to_cpu(*k1);
                u32 t2 = le32_to_cpu(*k2);

                if (t1 > t2)
                        return 1;
                if (t1 < t2)
                        return -1;
        }

        if (l1 > l2)
                return 1;
        if (l1 < l2)
                return -1;

        return 0;
}

static inline NTFS_CMP_FUNC get_cmp_func(const struct INDEX_ROOT *root)
{
        switch (root->type) {
        case ATTR_NAME:
                if (root->rule == NTFS_COLLATION_TYPE_FILENAME)
                        return &cmp_fnames;
                break;
        case ATTR_ZERO:
                switch (root->rule) {
                case NTFS_COLLATION_TYPE_UINT:
                        return &cmp_uint;
                case NTFS_COLLATION_TYPE_SECURITY_HASH:
                        return &cmp_sdh;
                case NTFS_COLLATION_TYPE_UINTS:
                        return &cmp_uints;
                default:
                        break;
                }
                break;
        default:
                break;
        }

        return NULL;
}

struct bmp_buf {
        struct ATTRIB *b;
        struct mft_inode *mi;
        struct buffer_head *bh;
        ulong *buf;
        size_t bit;
        u32 nbits;
        u64 new_valid;
};

static int bmp_buf_get(struct ntfs_index *indx, struct ntfs_inode *ni,
                       size_t bit, struct bmp_buf *bbuf)
{
        struct ATTRIB *b;
        size_t data_size, valid_size, vbo, off = bit >> 3;
        struct ntfs_sb_info *sbi = ni->mi.sbi;
        CLST vcn = off >> sbi->cluster_bits;
        struct ATTR_LIST_ENTRY *le = NULL;
        struct buffer_head *bh;
        struct super_block *sb;
        u32 blocksize;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];

        bbuf->bh = NULL;

        b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len,
                         &vcn, &bbuf->mi);
        bbuf->b = b;
        if (!b)
                return -EINVAL;

        if (!b->non_res) {
                data_size = le32_to_cpu(b->res.data_size);

                if (off >= data_size)
                        return -EINVAL;

                bbuf->buf = (ulong *)resident_data(b);
                bbuf->bit = 0;
                bbuf->nbits = data_size * 8;

                return 0;
        }

        data_size = le64_to_cpu(b->nres.data_size);
        if (WARN_ON(off >= data_size)) {
                /* Looks like filesystem error. */
                return -EINVAL;
        }

        valid_size = le64_to_cpu(b->nres.valid_size);

        bh = ntfs_bread_run(sbi, &indx->bitmap_run, off);
        if (!bh)
                return -EIO;

        if (IS_ERR(bh))
                return PTR_ERR(bh);

        bbuf->bh = bh;

        if (buffer_locked(bh))
                __wait_on_buffer(bh);

        lock_buffer(bh);

        sb = sbi->sb;
        blocksize = sb->s_blocksize;

        vbo = off & ~(size_t)sbi->block_mask;

        bbuf->new_valid = vbo + blocksize;
        if (bbuf->new_valid <= valid_size)
                bbuf->new_valid = 0;
        else if (bbuf->new_valid > data_size)
                bbuf->new_valid = data_size;

        if (vbo >= valid_size) {
                memset(bh->b_data, 0, blocksize);
        } else if (vbo + blocksize > valid_size) {
                u32 voff = valid_size & sbi->block_mask;

                memset(bh->b_data + voff, 0, blocksize - voff);
        }

        bbuf->buf = (ulong *)bh->b_data;
        bbuf->bit = 8 * (off & ~(size_t)sbi->block_mask);
        bbuf->nbits = 8 * blocksize;

        return 0;
}

static void bmp_buf_put(struct bmp_buf *bbuf, bool dirty)
{
        struct buffer_head *bh = bbuf->bh;
        struct ATTRIB *b = bbuf->b;

        if (!bh) {
                if (b && !b->non_res && dirty)
                        bbuf->mi->dirty = true;
                return;
        }

        if (!dirty)
                goto out;

        if (bbuf->new_valid) {
                b->nres.valid_size = cpu_to_le64(bbuf->new_valid);
                bbuf->mi->dirty = true;
        }

        set_buffer_uptodate(bh);
        mark_buffer_dirty(bh);

out:
        unlock_buffer(bh);
        put_bh(bh);
}

/*
 * indx_mark_used - Mark the bit @bit as used.
 */
static int indx_mark_used(struct ntfs_index *indx, struct ntfs_inode *ni,
                          size_t bit)
{
        int err;
        struct bmp_buf bbuf;

        err = bmp_buf_get(indx, ni, bit, &bbuf);
        if (err)
                return err;

        __set_bit_le(bit - bbuf.bit, bbuf.buf);

        bmp_buf_put(&bbuf, true);

        return 0;
}

/*
 * indx_mark_free - Mark the bit @bit as free.
 */
static int indx_mark_free(struct ntfs_index *indx, struct ntfs_inode *ni,
                          size_t bit)
{
        int err;
        struct bmp_buf bbuf;

        err = bmp_buf_get(indx, ni, bit, &bbuf);
        if (err)
                return err;

        __clear_bit_le(bit - bbuf.bit, bbuf.buf);

        bmp_buf_put(&bbuf, true);

        return 0;
}

/*
 * scan_nres_bitmap
 *
 * If ntfs_readdir calls this function (indx_used_bit -> scan_nres_bitmap),
 * inode is shared locked and no ni_lock.
 * Use rw_semaphore for read/write access to bitmap_run.
 */
static int scan_nres_bitmap(struct ntfs_inode *ni, struct ATTRIB *bitmap,
                            struct ntfs_index *indx, size_t from,
                            bool (*fn)(const ulong *buf, u32 bit, u32 bits,
                                       size_t *ret),
                            size_t *ret)
{
        struct ntfs_sb_info *sbi = ni->mi.sbi;
        struct super_block *sb = sbi->sb;
        struct runs_tree *run = &indx->bitmap_run;
        struct rw_semaphore *lock = &indx->run_lock;
        u32 nbits = sb->s_blocksize * 8;
        u32 blocksize = sb->s_blocksize;
        u64 valid_size = le64_to_cpu(bitmap->nres.valid_size);
        u64 data_size = le64_to_cpu(bitmap->nres.data_size);
        sector_t eblock = bytes_to_block(sb, data_size);
        size_t vbo = from >> 3;
        sector_t blk = (vbo & sbi->cluster_mask) >> sb->s_blocksize_bits;
        sector_t vblock = vbo >> sb->s_blocksize_bits;
        sector_t blen, block;
        CLST lcn, clen, vcn, vcn_next;
        size_t idx;
        struct buffer_head *bh;
        bool ok;

        *ret = MINUS_ONE_T;

        if (vblock >= eblock)
                return 0;

        from &= nbits - 1;
        vcn = vbo >> sbi->cluster_bits;

        down_read(lock);
        ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx);
        up_read(lock);

next_run:
        if (!ok) {
                int err;
                const struct INDEX_NAMES *name = &s_index_names[indx->type];

                down_write(lock);
                err = attr_load_runs_vcn(ni, ATTR_BITMAP, name->name,
                                         name->name_len, run, vcn);
                up_write(lock);
                if (err)
                        return err;
                down_read(lock);
                ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx);
                up_read(lock);
                if (!ok)
                        return -EINVAL;
        }

        blen = (sector_t)clen * sbi->blocks_per_cluster;
        block = (sector_t)lcn * sbi->blocks_per_cluster;

        for (; blk < blen; blk++, from = 0) {
                bh = ntfs_bread(sb, block + blk);
                if (!bh)
                        return -EIO;

                vbo = (u64)vblock << sb->s_blocksize_bits;
                if (vbo >= valid_size) {
                        memset(bh->b_data, 0, blocksize);
                } else if (vbo + blocksize > valid_size) {
                        u32 voff = valid_size & sbi->block_mask;

                        memset(bh->b_data + voff, 0, blocksize - voff);
                }

                if (vbo + blocksize > data_size)
                        nbits = 8 * (data_size - vbo);

                ok = nbits > from ?
                             (*fn)((ulong *)bh->b_data, from, nbits, ret) :
                             false;
                put_bh(bh);

                if (ok) {
                        *ret += 8 * vbo;
                        return 0;
                }

                if (++vblock >= eblock) {
                        *ret = MINUS_ONE_T;
                        return 0;
                }
        }
        blk = 0;
        vcn_next = vcn + clen;
        down_read(lock);
        ok = run_get_entry(run, ++idx, &vcn, &lcn, &clen) && vcn == vcn_next;
        if (!ok)
                vcn = vcn_next;
        up_read(lock);
        goto next_run;
}

static bool scan_for_free(const ulong *buf, u32 bit, u32 bits, size_t *ret)
{
        size_t pos = find_next_zero_bit_le(buf, bits, bit);

        if (pos >= bits)
                return false;
        *ret = pos;
        return true;
}

/*
 * indx_find_free - Look for free bit.
 *
 * Return: -1 if no free bits.
 */
static int indx_find_free(struct ntfs_index *indx, struct ntfs_inode *ni,
                          size_t *bit, struct ATTRIB **bitmap)
{
        struct ATTRIB *b;
        struct ATTR_LIST_ENTRY *le = NULL;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];
        int err;

        b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len,
                         NULL, NULL);

        if (!b)
                return -ENOENT;

        *bitmap = b;
        *bit = MINUS_ONE_T;

        if (!b->non_res) {
                u32 nbits = 8 * le32_to_cpu(b->res.data_size);
                size_t pos = find_next_zero_bit_le(resident_data(b), nbits, 0);

                if (pos < nbits)
                        *bit = pos;
        } else {
                err = scan_nres_bitmap(ni, b, indx, 0, &scan_for_free, bit);

                if (err)
                        return err;
        }

        return 0;
}

static bool scan_for_used(const ulong *buf, u32 bit, u32 bits, size_t *ret)
{
        size_t pos = find_next_bit_le(buf, bits, bit);

        if (pos >= bits)
                return false;
        *ret = pos;
        return true;
}

/*
 * indx_used_bit - Look for used bit.
 *
 * Return: MINUS_ONE_T if no used bits.
 */
int indx_used_bit(struct ntfs_index *indx, struct ntfs_inode *ni, size_t *bit)
{
        struct ATTRIB *b;
        struct ATTR_LIST_ENTRY *le = NULL;
        size_t from = *bit;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];
        int err;

        b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len,
                         NULL, NULL);

        if (!b)
                return -ENOENT;

        *bit = MINUS_ONE_T;

        if (!b->non_res) {
                u32 nbits = le32_to_cpu(b->res.data_size) * 8;
                size_t pos = find_next_bit_le(resident_data(b), nbits, from);

                if (pos < nbits)
                        *bit = pos;
        } else {
                err = scan_nres_bitmap(ni, b, indx, from, &scan_for_used, bit);
                if (err)
                        return err;
        }

        return 0;
}

/*
 * hdr_find_split
 *
 * Find a point at which the index allocation buffer would like to be split.
 * NOTE: This function should never return 'END' entry NULL returns on error.
 */
static const struct NTFS_DE *hdr_find_split(const struct INDEX_HDR *hdr)
{
        size_t o;
        const struct NTFS_DE *e = hdr_first_de(hdr);
        u32 used_2 = le32_to_cpu(hdr->used) >> 1;
        u16 esize;

        if (!e || de_is_last(e))
                return NULL;

        esize = le16_to_cpu(e->size);
        for (o = le32_to_cpu(hdr->de_off) + esize; o < used_2; o += esize) {
                const struct NTFS_DE *p = e;

                e = Add2Ptr(hdr, o);

                /* We must not return END entry. */
                if (de_is_last(e))
                        return p;

                esize = le16_to_cpu(e->size);
        }

        return e;
}

/*
 * hdr_insert_head - Insert some entries at the beginning of the buffer.
 *
 * It is used to insert entries into a newly-created buffer.
 */
static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr,
                                             const void *ins, u32 ins_bytes)
{
        u32 to_move;
        struct NTFS_DE *e = hdr_first_de(hdr);
        u32 used = le32_to_cpu(hdr->used);

        if (!e)
                return NULL;

        /* Now we just make room for the inserted entries and jam it in. */
        to_move = used - le32_to_cpu(hdr->de_off);
        memmove(Add2Ptr(e, ins_bytes), e, to_move);
        memcpy(e, ins, ins_bytes);
        hdr->used = cpu_to_le32(used + ins_bytes);

        return e;
}

/*
 * index_hdr_check
 *
 * return true if INDEX_HDR is valid
 */
static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes)
{
        u32 end = le32_to_cpu(hdr->used);
        u32 tot = le32_to_cpu(hdr->total);
        u32 off = le32_to_cpu(hdr->de_off);

        if (!IS_ALIGNED(off, 8) || tot > bytes || end > tot ||
            off + sizeof(struct NTFS_DE) > end) {
                /* incorrect index buffer. */
                return false;
        }

        return true;
}

/*
 * index_buf_check
 *
 * return true if INDEX_BUFFER seems is valid
 */
static bool index_buf_check(const struct INDEX_BUFFER *ib, u32 bytes,
                            const CLST *vbn)
{
        const struct NTFS_RECORD_HEADER *rhdr = &ib->rhdr;
        u16 fo = le16_to_cpu(rhdr->fix_off);
        u16 fn = le16_to_cpu(rhdr->fix_num);

        if (bytes <= offsetof(struct INDEX_BUFFER, ihdr) ||
            rhdr->sign != NTFS_INDX_SIGNATURE ||
            fo < sizeof(struct INDEX_BUFFER)
            /* Check index buffer vbn. */
            || (vbn && *vbn != le64_to_cpu(ib->vbn)) || (fo % sizeof(short)) ||
            fo + fn * sizeof(short) >= bytes ||
            fn != ((bytes >> SECTOR_SHIFT) + 1)) {
                /* incorrect index buffer. */
                return false;
        }

        return index_hdr_check(&ib->ihdr,
                               bytes - offsetof(struct INDEX_BUFFER, ihdr));
}

void fnd_clear(struct ntfs_fnd *fnd)
{
        int i;

        for (i = fnd->level - 1; i >= 0; i--) {
                struct indx_node *n = fnd->nodes[i];

                if (!n)
                        continue;

                put_indx_node(n);
                fnd->nodes[i] = NULL;
        }
        fnd->level = 0;
        fnd->root_de = NULL;
}

static int fnd_push(struct ntfs_fnd *fnd, struct indx_node *n,
                    struct NTFS_DE *e)
{
        int i = fnd->level;

        if (i < 0 || i >= ARRAY_SIZE(fnd->nodes))
                return -EINVAL;
        fnd->nodes[i] = n;
        fnd->de[i] = e;
        fnd->level += 1;
        return 0;
}

static struct indx_node *fnd_pop(struct ntfs_fnd *fnd)
{
        struct indx_node *n;
        int i = fnd->level;

        i -= 1;
        n = fnd->nodes[i];
        fnd->nodes[i] = NULL;
        fnd->level = i;

        return n;
}

static bool fnd_is_empty(struct ntfs_fnd *fnd)
{
        if (!fnd->level)
                return !fnd->root_de;

        return !fnd->de[fnd->level - 1];
}

/*
 * hdr_find_e - Locate an entry the index buffer.
 *
 * If no matching entry is found, it returns the first entry which is greater
 * than the desired entry If the search key is greater than all the entries the
 * buffer, it returns the 'end' entry. This function does a binary search of the
 * current index buffer, for the first entry that is <= to the search value.
 *
 * Return: NULL if error.
 */
static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx,
                                  const struct INDEX_HDR *hdr, const void *key,
                                  size_t key_len, const void *ctx, int *diff)
{
        struct NTFS_DE *e, *found = NULL;
        NTFS_CMP_FUNC cmp = indx->cmp;
        int min_idx = 0, mid_idx, max_idx = 0;
        int diff2;
        int table_size = 8;
        u32 e_size, e_key_len;
        u32 end = le32_to_cpu(hdr->used);
        u32 off = le32_to_cpu(hdr->de_off);
        u32 total = le32_to_cpu(hdr->total);
        u16 offs[128];

        if (unlikely(!cmp))
                return NULL;

fill_table:
        if (end > total)
                return NULL;

        if (off + sizeof(struct NTFS_DE) > end)
                return NULL;

        e = Add2Ptr(hdr, off);
        e_size = le16_to_cpu(e->size);

        if (e_size < sizeof(struct NTFS_DE) || off + e_size > end)
                return NULL;

        if (!de_is_last(e)) {
                offs[max_idx] = off;
                off += e_size;

                max_idx++;
                if (max_idx < table_size)
                        goto fill_table;

                max_idx--;
        }

binary_search:
        e_key_len = le16_to_cpu(e->key_size);

        diff2 = (*cmp)(key, key_len, e + 1, e_key_len, ctx);
        if (diff2 > 0) {
                if (found) {
                        min_idx = mid_idx + 1;
                } else {
                        if (de_is_last(e))
                                return NULL;

                        max_idx = 0;
                        table_size = min(table_size * 2, (int)ARRAY_SIZE(offs));
                        goto fill_table;
                }
        } else if (diff2 < 0) {
                if (found)
                        max_idx = mid_idx - 1;
                else
                        max_idx--;

                found = e;
        } else {
                *diff = 0;
                return e;
        }

        if (min_idx > max_idx) {
                *diff = -1;
                return found;
        }

        mid_idx = (min_idx + max_idx) >> 1;
        e = Add2Ptr(hdr, offs[mid_idx]);

        goto binary_search;
}

/*
 * hdr_insert_de - Insert an index entry into the buffer.
 *
 * 'before' should be a pointer previously returned from hdr_find_e.
 */
static struct NTFS_DE *hdr_insert_de(const struct ntfs_index *indx,
                                     struct INDEX_HDR *hdr,
                                     const struct NTFS_DE *de,
                                     struct NTFS_DE *before, const void *ctx)
{
        int diff;
        size_t off = PtrOffset(hdr, before);
        u32 used = le32_to_cpu(hdr->used);
        u32 total = le32_to_cpu(hdr->total);
        u16 de_size = le16_to_cpu(de->size);

        /* First, check to see if there's enough room. */
        if (used + de_size > total)
                return NULL;

        /* We know there's enough space, so we know we'll succeed. */
        if (before) {
                /* Check that before is inside Index. */
                if (off >= used || off < le32_to_cpu(hdr->de_off) ||
                    off + le16_to_cpu(before->size) > total) {
                        return NULL;
                }
                goto ok;
        }
        /* No insert point is applied. Get it manually. */
        before = hdr_find_e(indx, hdr, de + 1, le16_to_cpu(de->key_size), ctx,
                            &diff);
        if (!before)
                return NULL;
        off = PtrOffset(hdr, before);

ok:
        /* Now we just make room for the entry and jam it in. */
        memmove(Add2Ptr(before, de_size), before, used - off);

        hdr->used = cpu_to_le32(used + de_size);
        memcpy(before, de, de_size);

        return before;
}

/*
 * hdr_delete_de - Remove an entry from the index buffer.
 */
static inline struct NTFS_DE *hdr_delete_de(struct INDEX_HDR *hdr,
                                            struct NTFS_DE *re)
{
        u32 used = le32_to_cpu(hdr->used);
        u16 esize = le16_to_cpu(re->size);
        u32 off = PtrOffset(hdr, re);
        int bytes = used - (off + esize);

        /* check INDEX_HDR valid before using INDEX_HDR */
        if (!check_index_header(hdr, le32_to_cpu(hdr->total)))
                return NULL;

        if (off >= used || esize < sizeof(struct NTFS_DE) ||
            bytes < sizeof(struct NTFS_DE))
                return NULL;

        hdr->used = cpu_to_le32(used - esize);
        memmove(re, Add2Ptr(re, esize), bytes);

        return re;
}

void indx_clear(struct ntfs_index *indx)
{
        run_close(&indx->alloc_run);
        run_close(&indx->bitmap_run);
}

int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi,
              const struct ATTRIB *attr, enum index_mutex_classed type)
{
        u32 t32;
        const struct INDEX_ROOT *root = resident_data(attr);

        t32 = le32_to_cpu(attr->res.data_size);
        if (t32 <= offsetof(struct INDEX_ROOT, ihdr) ||
            !index_hdr_check(&root->ihdr,
                             t32 - offsetof(struct INDEX_ROOT, ihdr))) {
                goto out;
        }

        /* Check root fields. */
        if (!root->index_block_clst)
                goto out;

        indx->type = type;
        indx->idx2vbn_bits = __ffs(root->index_block_clst);

        t32 = le32_to_cpu(root->index_block_size);
        indx->index_bits = blksize_bits(t32);

        /* Check index record size. */
        if (t32 < sbi->cluster_size) {
                /* Index record is smaller than a cluster, use 512 blocks. */
                if (t32 != root->index_block_clst * SECTOR_SIZE)
                        goto out;

                /* Check alignment to a cluster. */
                if ((sbi->cluster_size >> SECTOR_SHIFT) &
                    (root->index_block_clst - 1)) {
                        goto out;
                }

                indx->vbn2vbo_bits = SECTOR_SHIFT;
        } else {
                /* Index record must be a multiple of cluster size. */
                if (t32 != root->index_block_clst << sbi->cluster_bits)
                        goto out;

                indx->vbn2vbo_bits = sbi->cluster_bits;
        }

        init_rwsem(&indx->run_lock);

        indx->cmp = get_cmp_func(root);
        if (!indx->cmp)
                goto out;

        return 0;

out:
        ntfs_set_state(sbi, NTFS_DIRTY_DIRTY);
        return -EINVAL;
}

static struct indx_node *indx_new(struct ntfs_index *indx,
                                  struct ntfs_inode *ni, CLST vbn,
                                  const __le64 *sub_vbn)
{
        int err;
        struct NTFS_DE *e;
        struct indx_node *r;
        struct INDEX_HDR *hdr;
        struct INDEX_BUFFER *index;
        u64 vbo = (u64)vbn << indx->vbn2vbo_bits;
        u32 bytes = 1u << indx->index_bits;
        u16 fn;
        u32 eo;

        r = kzalloc(sizeof(struct indx_node), GFP_NOFS);
        if (!r)
                return ERR_PTR(-ENOMEM);

        index = kzalloc(bytes, GFP_NOFS);
        if (!index) {
                kfree(r);
                return ERR_PTR(-ENOMEM);
        }

        err = ntfs_get_bh(ni->mi.sbi, &indx->alloc_run, vbo, bytes, &r->nb);

        if (err) {
                kfree(index);
                kfree(r);
                return ERR_PTR(err);
        }

        /* Create header. */
        index->rhdr.sign = NTFS_INDX_SIGNATURE;
        index->rhdr.fix_off = cpu_to_le16(sizeof(struct INDEX_BUFFER)); // 0x28
        fn = (bytes >> SECTOR_SHIFT) + 1; // 9
        index->rhdr.fix_num = cpu_to_le16(fn);
        index->vbn = cpu_to_le64(vbn);
        hdr = &index->ihdr;
        eo = ALIGN(sizeof(struct INDEX_BUFFER) + fn * sizeof(short), 8);
        hdr->de_off = cpu_to_le32(eo);

        e = Add2Ptr(hdr, eo);

        if (sub_vbn) {
                e->flags = NTFS_IE_LAST | NTFS_IE_HAS_SUBNODES;
                e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64));
                hdr->used =
                        cpu_to_le32(eo + sizeof(struct NTFS_DE) + sizeof(u64));
                de_set_vbn_le(e, *sub_vbn);
                hdr->flags = NTFS_INDEX_HDR_HAS_SUBNODES;
        } else {
                e->size = cpu_to_le16(sizeof(struct NTFS_DE));
                hdr->used = cpu_to_le32(eo + sizeof(struct NTFS_DE));
                e->flags = NTFS_IE_LAST;
        }

        hdr->total = cpu_to_le32(bytes - offsetof(struct INDEX_BUFFER, ihdr));

        r->index = index;
        return r;
}

struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni,
                                 struct ATTRIB **attr, struct mft_inode **mi)
{
        struct ATTR_LIST_ENTRY *le = NULL;
        struct ATTRIB *a;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];
        struct INDEX_ROOT *root;

        a = ni_find_attr(ni, NULL, &le, ATTR_ROOT, in->name, in->name_len, NULL,
                         mi);
        if (!a)
                return NULL;

        if (attr)
                *attr = a;

        root = resident_data_ex(a, sizeof(struct INDEX_ROOT));

        /* length check */
        if (root &&
            offsetof(struct INDEX_ROOT, ihdr) + le32_to_cpu(root->ihdr.used) >
                    le32_to_cpu(a->res.data_size)) {
                return NULL;
        }

        return root;
}

static int indx_write(struct ntfs_index *indx, struct ntfs_inode *ni,
                      struct indx_node *node, int sync)
{
        struct INDEX_BUFFER *ib = node->index;

        return ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &node->nb, sync);
}

/*
 * indx_read
 *
 * If ntfs_readdir calls this function
 * inode is shared locked and no ni_lock.
 * Use rw_semaphore for read/write access to alloc_run.
 */
int indx_read(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn,
              struct indx_node **node)
{
        int err;
        struct INDEX_BUFFER *ib;
        struct runs_tree *run = &indx->alloc_run;
        struct rw_semaphore *lock = &indx->run_lock;
        u64 vbo = (u64)vbn << indx->vbn2vbo_bits;
        u32 bytes = 1u << indx->index_bits;
        struct indx_node *in = *node;
        const struct INDEX_NAMES *name;

        if (!in) {
                in = kzalloc(sizeof(struct indx_node), GFP_NOFS);
                if (!in)
                        return -ENOMEM;
        } else {
                nb_put(&in->nb);
        }

        ib = in->index;
        if (!ib) {
                ib = kmalloc(bytes, GFP_NOFS);
                if (!ib) {
                        err = -ENOMEM;
                        goto out;
                }
        }

        down_read(lock);
        err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb);
        up_read(lock);
        if (!err)
                goto ok;

        if (err == -E_NTFS_FIXUP)
                goto ok;

        if (err != -ENOENT)
                goto out;

        name = &s_index_names[indx->type];
        down_write(lock);
        err = attr_load_runs_range(ni, ATTR_ALLOC, name->name, name->name_len,
                                   run, vbo, vbo + bytes);
        up_write(lock);
        if (err)
                goto out;

        down_read(lock);
        err = ntfs_read_bh(ni->mi.sbi, run, vbo, &ib->rhdr, bytes, &in->nb);
        up_read(lock);
        if (err == -E_NTFS_FIXUP)
                goto ok;

        if (err)
                goto out;

ok:
        if (!index_buf_check(ib, bytes, &vbn)) {
                ntfs_inode_err(&ni->vfs_inode, "directory corrupted");
                ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_ERROR);
                err = -EINVAL;
                goto out;
        }

        if (err == -E_NTFS_FIXUP) {
                ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &in->nb, 0);
                err = 0;
        }

        /* check for index header length */
        if (offsetof(struct INDEX_BUFFER, ihdr) + le32_to_cpu(ib->ihdr.used) >
            bytes) {
                err = -EINVAL;
                goto out;
        }

        in->index = ib;
        *node = in;

out:
        if (err == -E_NTFS_CORRUPT) {
                ntfs_inode_err(&ni->vfs_inode, "directory corrupted");
                ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_ERROR);
                err = -EINVAL;
        }

        if (ib != in->index)
                kfree(ib);

        if (*node != in) {
                nb_put(&in->nb);
                kfree(in);
        }

        return err;
}

/*
 * indx_find - Scan NTFS directory for given entry.
 */
int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni,
              const struct INDEX_ROOT *root, const void *key, size_t key_len,
              const void *ctx, int *diff, struct NTFS_DE **entry,
              struct ntfs_fnd *fnd)
{
        int err;
        struct NTFS_DE *e;
        struct indx_node *node;

        if (!root)
                root = indx_get_root(&ni->dir, ni, NULL, NULL);

        if (!root) {
                /* Should not happen. */
                return -EINVAL;
        }

        /* Check cache. */
        e = fnd->level ? fnd->de[fnd->level - 1] : fnd->root_de;
        if (e && !de_is_last(e) &&
            !(*indx->cmp)(key, key_len, e + 1, le16_to_cpu(e->key_size), ctx)) {
                *entry = e;
                *diff = 0;
                return 0;
        }

        /* Soft finder reset. */
        fnd_clear(fnd);

        /* Lookup entry that is <= to the search value. */
        e = hdr_find_e(indx, &root->ihdr, key, key_len, ctx, diff);
        if (!e)
                return -EINVAL;

        fnd->root_de = e;

        for (;;) {
                node = NULL;
                if (*diff >= 0 || !de_has_vcn_ex(e))
                        break;

                /* Read next level. */
                err = indx_read(indx, ni, de_get_vbn(e), &node);
                if (err) {
                        /* io error? */
                        return err;
                }

                /* Lookup entry that is <= to the search value. */
                e = hdr_find_e(indx, &node->index->ihdr, key, key_len, ctx,
                               diff);
                if (!e) {
                        put_indx_node(node);
                        return -EINVAL;
                }

                fnd_push(fnd, node, e);
        }

        *entry = e;
        return 0;
}

int indx_find_sort(struct ntfs_index *indx, struct ntfs_inode *ni,
                   const struct INDEX_ROOT *root, struct NTFS_DE **entry,
                   struct ntfs_fnd *fnd)
{
        int err;
        struct indx_node *n = NULL;
        struct NTFS_DE *e;
        size_t iter = 0;
        int level = fnd->level;

        if (!*entry) {
                /* Start find. */
                e = hdr_first_de(&root->ihdr);
                if (!e)
                        return 0;
                fnd_clear(fnd);
                fnd->root_de = e;
        } else if (!level) {
                if (de_is_last(fnd->root_de)) {
                        *entry = NULL;
                        return 0;
                }

                e = hdr_next_de(&root->ihdr, fnd->root_de);
                if (!e)
                        return -EINVAL;
                fnd->root_de = e;
        } else {
                n = fnd->nodes[level - 1];
                e = fnd->de[level - 1];

                if (de_is_last(e))
                        goto pop_level;

                e = hdr_next_de(&n->index->ihdr, e);
                if (!e)
                        return -EINVAL;

                fnd->de[level - 1] = e;
        }

        /* Just to avoid tree cycle. */
next_iter:
        if (iter++ >= 1000)
                return -EINVAL;

        while (de_has_vcn_ex(e)) {
                if (le16_to_cpu(e->size) <
                    sizeof(struct NTFS_DE) + sizeof(u64)) {
                        if (n) {
                                fnd_pop(fnd);
                                kfree(n);
                        }
                        return -EINVAL;
                }

                /* Read next level. */
                err = indx_read(indx, ni, de_get_vbn(e), &n);
                if (err)
                        return err;

                /* Try next level. */
                e = hdr_first_de(&n->index->ihdr);
                if (!e) {
                        kfree(n);
                        return -EINVAL;
                }

                fnd_push(fnd, n, e);
        }

        if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) {
                *entry = e;
                return 0;
        }

pop_level:
        for (;;) {
                if (!de_is_last(e))
                        goto next_iter;

                /* Pop one level. */
                if (n) {
                        fnd_pop(fnd);
                        kfree(n);
                }

                level = fnd->level;

                if (level) {
                        n = fnd->nodes[level - 1];
                        e = fnd->de[level - 1];
                } else if (fnd->root_de) {
                        n = NULL;
                        e = fnd->root_de;
                        fnd->root_de = NULL;
                } else {
                        *entry = NULL;
                        return 0;
                }

                if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) {
                        *entry = e;
                        if (!fnd->root_de)
                                fnd->root_de = e;
                        return 0;
                }
        }
}

int indx_find_raw(struct ntfs_index *indx, struct ntfs_inode *ni,
                  const struct INDEX_ROOT *root, struct NTFS_DE **entry,
                  size_t *off, struct ntfs_fnd *fnd)
{
        int err;
        struct indx_node *n = NULL;
        struct NTFS_DE *e = NULL;
        struct NTFS_DE *e2;
        size_t bit;
        CLST next_used_vbn;
        CLST next_vbn;
        u32 record_size = ni->mi.sbi->record_size;

        /* Use non sorted algorithm. */
        if (!*entry) {
                /* This is the first call. */
                e = hdr_first_de(&root->ihdr);
                if (!e)
                        return 0;
                fnd_clear(fnd);
                fnd->root_de = e;

                /* The first call with setup of initial element. */
                if (*off >= record_size) {
                        next_vbn = (((*off - record_size) >> indx->index_bits))
                                   << indx->idx2vbn_bits;
                        /* Jump inside cycle 'for'. */
                        goto next;
                }

                /* Start enumeration from root. */
                *off = 0;
        } else if (!fnd->root_de)
                return -EINVAL;

        for (;;) {
                /* Check if current entry can be used. */
                if (e && le16_to_cpu(e->size) > sizeof(struct NTFS_DE))
                        goto ok;

                if (!fnd->level) {
                        /* Continue to enumerate root. */
                        if (!de_is_last(fnd->root_de)) {
                                e = hdr_next_de(&root->ihdr, fnd->root_de);
                                if (!e)
                                        return -EINVAL;
                                fnd->root_de = e;
                                continue;
                        }

                        /* Start to enumerate indexes from 0. */
                        next_vbn = 0;
                } else {
                        /* Continue to enumerate indexes. */
                        e2 = fnd->de[fnd->level - 1];

                        n = fnd->nodes[fnd->level - 1];

                        if (!de_is_last(e2)) {
                                e = hdr_next_de(&n->index->ihdr, e2);
                                if (!e)
                                        return -EINVAL;
                                fnd->de[fnd->level - 1] = e;
                                continue;
                        }

                        /* Continue with next index. */
                        next_vbn = le64_to_cpu(n->index->vbn) +
                                   root->index_block_clst;
                }

next:
                /* Release current index. */
                if (n) {
                        fnd_pop(fnd);
                        put_indx_node(n);
                        n = NULL;
                }

                /* Skip all free indexes. */
                bit = next_vbn >> indx->idx2vbn_bits;
                err = indx_used_bit(indx, ni, &bit);
                if (err == -ENOENT || bit == MINUS_ONE_T) {
                        /* No used indexes. */
                        *entry = NULL;
                        return 0;
                }

                next_used_vbn = bit << indx->idx2vbn_bits;

                /* Read buffer into memory. */
                err = indx_read(indx, ni, next_used_vbn, &n);
                if (err)
                        return err;

                e = hdr_first_de(&n->index->ihdr);
                fnd_push(fnd, n, e);
                if (!e)
                        return -EINVAL;
        }

ok:
        /* Return offset to restore enumerator if necessary. */
        if (!n) {
                /* 'e' points in root, */
                *off = PtrOffset(&root->ihdr, e);
        } else {
                /* 'e' points in index, */
                *off = (le64_to_cpu(n->index->vbn) << indx->vbn2vbo_bits) +
                       record_size + PtrOffset(&n->index->ihdr, e);
        }

        *entry = e;
        return 0;
}

/*
 * indx_create_allocate - Create "Allocation + Bitmap" attributes.
 */
static int indx_create_allocate(struct ntfs_index *indx, struct ntfs_inode *ni,
                                CLST *vbn)
{
        int err;
        struct ntfs_sb_info *sbi = ni->mi.sbi;
        struct ATTRIB *bitmap;
        struct ATTRIB *alloc;
        u32 data_size = 1u << indx->index_bits;
        u32 alloc_size = ntfs_up_cluster(sbi, data_size);
        CLST len = alloc_size >> sbi->cluster_bits;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];
        CLST alen;
        struct runs_tree run;

        run_init(&run);

        err = attr_allocate_clusters(sbi, &run, 0, 0, len, NULL, ALLOCATE_DEF,
                                     &alen, 0, NULL, NULL);
        if (err)
                goto out;

        err = ni_insert_nonresident(ni, ATTR_ALLOC, in->name, in->name_len,
                                    &run, 0, len, 0, &alloc, NULL, NULL);
        if (err)
                goto out1;

        alloc->nres.valid_size = alloc->nres.data_size = cpu_to_le64(data_size);

        err = ni_insert_resident(ni, ntfs3_bitmap_size(1), ATTR_BITMAP,
                                 in->name, in->name_len, &bitmap, NULL, NULL);
        if (err)
                goto out2;

        if (in->name == I30_NAME) {
                i_size_write(&ni->vfs_inode, data_size);
                inode_set_bytes(&ni->vfs_inode, alloc_size);
        }

        memcpy(&indx->alloc_run, &run, sizeof(run));

        *vbn = 0;

        return 0;

out2:
        mi_remove_attr(NULL, &ni->mi, alloc);

out1:
        run_deallocate(sbi, &run, false);

out:
        return err;
}

/*
 * indx_add_allocate - Add clusters to index.
 */
static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni,
                             CLST *vbn)
{
        int err;
        size_t bit;
        u64 data_size;
        u64 bmp_size, bmp_size_v;
        struct ATTRIB *bmp, *alloc;
        struct mft_inode *mi;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];

        err = indx_find_free(indx, ni, &bit, &bmp);
        if (err)
                goto out1;

        if (bit != MINUS_ONE_T) {
                bmp = NULL;
        } else {
                if (bmp->non_res) {
                        bmp_size = le64_to_cpu(bmp->nres.data_size);
                        bmp_size_v = le64_to_cpu(bmp->nres.valid_size);
                } else {
                        bmp_size = bmp_size_v = le32_to_cpu(bmp->res.data_size);
                }

                bit = bmp_size << 3;
        }

        data_size = (u64)(bit + 1) << indx->index_bits;

        if (bmp) {
                /* Increase bitmap. */
                err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len,
                                    &indx->bitmap_run,
                                    ntfs3_bitmap_size(bit + 1), NULL, true,
                                    NULL);
                if (err)
                        goto out1;
        }

        alloc = ni_find_attr(ni, NULL, NULL, ATTR_ALLOC, in->name, in->name_len,
                             NULL, &mi);
        if (!alloc) {
                err = -EINVAL;
                if (bmp)
                        goto out2;
                goto out1;
        }

        if (data_size <= le64_to_cpu(alloc->nres.data_size)) {
                /* Reuse index. */
                goto out;
        }

        /* Increase allocation. */
        err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len,
                            &indx->alloc_run, data_size, &data_size, true,
                            NULL);
        if (err) {
                if (bmp)
                        goto out2;
                goto out1;
        }

        if (in->name == I30_NAME)
                i_size_write(&ni->vfs_inode, data_size);

out:
        *vbn = bit << indx->idx2vbn_bits;

        return 0;

out2:
        /* Ops. No space? */
        attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len,
                      &indx->bitmap_run, bmp_size, &bmp_size_v, false, NULL);

out1:
        return err;
}

/*
 * indx_insert_into_root - Attempt to insert an entry into the index root.
 *
 * @undo - True if we undoing previous remove.
 * If necessary, it will twiddle the index b-tree.
 */
static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
                                 const struct NTFS_DE *new_de,
                                 struct NTFS_DE *root_de, const void *ctx,
                                 struct ntfs_fnd *fnd, bool undo)
{
        int err = 0;
        struct NTFS_DE *e, *e0, *re;
        struct mft_inode *mi;
        struct ATTRIB *attr;
        struct INDEX_HDR *hdr;
        struct indx_node *n;
        CLST new_vbn;
        __le64 *sub_vbn, t_vbn;
        u16 new_de_size;
        u32 hdr_used, hdr_total, asize, to_move;
        u32 root_size, new_root_size;
        struct ntfs_sb_info *sbi;
        int ds_root;
        struct INDEX_ROOT *root, *a_root;

        /* Get the record this root placed in. */
        root = indx_get_root(indx, ni, &attr, &mi);
        if (!root)
                return -EINVAL;

        /*
         * Try easy case:
         * hdr_insert_de will succeed if there's
         * room the root for the new entry.
         */
        hdr = &root->ihdr;
        sbi = ni->mi.sbi;
        new_de_size = le16_to_cpu(new_de->size);
        hdr_used = le32_to_cpu(hdr->used);
        hdr_total = le32_to_cpu(hdr->total);
        asize = le32_to_cpu(attr->size);
        root_size = le32_to_cpu(attr->res.data_size);

        ds_root = new_de_size + hdr_used - hdr_total;

        /* If 'undo' is set then reduce requirements. */
        if ((undo || asize + ds_root < sbi->max_bytes_per_attr) &&
            mi_resize_attr(mi, attr, ds_root)) {
                hdr->total = cpu_to_le32(hdr_total + ds_root);
                e = hdr_insert_de(indx, hdr, new_de, root_de, ctx);
                WARN_ON(!e);
                fnd_clear(fnd);
                fnd->root_de = e;

                return 0;
        }

        /* Make a copy of root attribute to restore if error. */
        a_root = kmemdup(attr, asize, GFP_NOFS);
        if (!a_root)
                return -ENOMEM;

        /*
         * Copy all the non-end entries from
         * the index root to the new buffer.
         */
        to_move = 0;
        e0 = hdr_first_de(hdr);

        /* Calculate the size to copy. */
        for (e = e0;; e = hdr_next_de(hdr, e)) {
                if (!e) {
                        err = -EINVAL;
                        goto out_free_root;
                }

                if (de_is_last(e))
                        break;
                to_move += le16_to_cpu(e->size);
        }

        if (!to_move) {
                re = NULL;
        } else {
                re = kmemdup(e0, to_move, GFP_NOFS);
                if (!re) {
                        err = -ENOMEM;
                        goto out_free_root;
                }
        }

        sub_vbn = NULL;
        if (de_has_vcn(e)) {
                t_vbn = de_get_vbn_le(e);
                sub_vbn = &t_vbn;
        }

        new_root_size = sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE) +
                        sizeof(u64);
        ds_root = new_root_size - root_size;

        if (ds_root > 0 && asize + ds_root > sbi->max_bytes_per_attr) {
                /* Make root external. */
                err = -EOPNOTSUPP;
                goto out_free_re;
        }

        if (ds_root)
                mi_resize_attr(mi, attr, ds_root);

        /* Fill first entry (vcn will be set later). */
        e = (struct NTFS_DE *)(root + 1);
        memset(e, 0, sizeof(struct NTFS_DE));
        e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64));
        e->flags = NTFS_IE_HAS_SUBNODES | NTFS_IE_LAST;

        hdr->flags = NTFS_INDEX_HDR_HAS_SUBNODES;
        hdr->used = hdr->total =
                cpu_to_le32(new_root_size - offsetof(struct INDEX_ROOT, ihdr));

        fnd->root_de = hdr_first_de(hdr);
        mi->dirty = true;

        /* Create alloc and bitmap attributes (if not). */
        err = run_is_empty(&indx->alloc_run) ?
                      indx_create_allocate(indx, ni, &new_vbn) :
                      indx_add_allocate(indx, ni, &new_vbn);

        /* Layout of record may be changed, so rescan root. */
        root = indx_get_root(indx, ni, &attr, &mi);
        if (!root) {
                /* Bug? */
                ntfs_set_state(sbi, NTFS_DIRTY_ERROR);
                err = -EINVAL;
                goto out_free_re;
        }

        if (err) {
                /* Restore root. */
                if (mi_resize_attr(mi, attr, -ds_root)) {
                        memcpy(attr, a_root, asize);
                } else {
                        /* Bug? */
                        ntfs_set_state(sbi, NTFS_DIRTY_ERROR);
                }
                goto out_free_re;
        }

        e = (struct NTFS_DE *)(root + 1);
        *(__le64 *)(e + 1) = cpu_to_le64(new_vbn);
        mi->dirty = true;

        /* Now we can create/format the new buffer and copy the entries into. */
        n = indx_new(indx, ni, new_vbn, sub_vbn);
        if (IS_ERR(n)) {
                err = PTR_ERR(n);
                goto out_free_re;
        }

        hdr = &n->index->ihdr;
        hdr_used = le32_to_cpu(hdr->used);
        hdr_total = le32_to_cpu(hdr->total);

        /* Copy root entries into new buffer. */
        hdr_insert_head(hdr, re, to_move);

        /* Update bitmap attribute. */
        indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits);

        /* Check if we can insert new entry new index buffer. */
        if (hdr_used + new_de_size > hdr_total) {
                /*
                 * This occurs if MFT record is the same or bigger than index
                 * buffer. Move all root new index and have no space to add
                 * new entry classic case when MFT record is 1K and index
                 * buffer 4K the problem should not occurs.
                 */
                kfree(re);
                indx_write(indx, ni, n, 0);

                put_indx_node(n);
                fnd_clear(fnd);
                err = indx_insert_entry(indx, ni, new_de, ctx, fnd, undo);
                goto out_free_root;
        }

        /*
         * Now root is a parent for new index buffer.
         * Insert NewEntry a new buffer.
         */
        e = hdr_insert_de(indx, hdr, new_de, NULL, ctx);
        if (!e) {
                err = -EINVAL;
                goto out_put_n;
        }
        fnd_push(fnd, n, e);

        /* Just write updates index into disk. */
        indx_write(indx, ni, n, 0);

        n = NULL;

out_put_n:
        put_indx_node(n);
out_free_re:
        kfree(re);
out_free_root:
        kfree(a_root);
        return err;
}

/*
 * indx_insert_into_buffer
 *
 * Attempt to insert an entry into an Index Allocation Buffer.
 * If necessary, it will split the buffer.
 */
static int
indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
                        struct INDEX_ROOT *root, const struct NTFS_DE *new_de,
                        const void *ctx, int level, struct ntfs_fnd *fnd)
{
        int err;
        const struct NTFS_DE *sp;
        struct NTFS_DE *e, *de_t, *up_e;
        struct indx_node *n2;
        struct indx_node *n1 = fnd->nodes[level];
        struct INDEX_HDR *hdr1 = &n1->index->ihdr;
        struct INDEX_HDR *hdr2;
        u32 to_copy, used, used1;
        CLST new_vbn;
        __le64 t_vbn, *sub_vbn;
        u16 sp_size;
        void *hdr1_saved = NULL;

        /* Try the most easy case. */
        e = fnd->level - 1 == level ? fnd->de[level] : NULL;
        e = hdr_insert_de(indx, hdr1, new_de, e, ctx);
        fnd->de[level] = e;
        if (e) {
                /* Just write updated index into disk. */
                indx_write(indx, ni, n1, 0);
                return 0;
        }

        /*
         * No space to insert into buffer. Split it.
         * To split we:
         *  - Save split point ('cause index buffers will be changed)
         * - Allocate NewBuffer and copy all entries <= sp into new buffer
         * - Remove all entries (sp including) from TargetBuffer
         * - Insert NewEntry into left or right buffer (depending on sp <=>
         *     NewEntry)
         * - Insert sp into parent buffer (or root)
         * - Make sp a parent for new buffer
         */
        sp = hdr_find_split(hdr1);
        if (!sp)
                return -EINVAL;

        sp_size = le16_to_cpu(sp->size);
        up_e = kmalloc(sp_size + sizeof(u64), GFP_NOFS);
        if (!up_e)
                return -ENOMEM;
        memcpy(up_e, sp, sp_size);

        used1 = le32_to_cpu(hdr1->used);
        hdr1_saved = kmemdup(hdr1, used1, GFP_NOFS);
        if (!hdr1_saved) {
                err = -ENOMEM;
                goto out;
        }

        if (!hdr1->flags) {
                up_e->flags |= NTFS_IE_HAS_SUBNODES;
                up_e->size = cpu_to_le16(sp_size + sizeof(u64));
                sub_vbn = NULL;
        } else {
                t_vbn = de_get_vbn_le(up_e);
                sub_vbn = &t_vbn;
        }

        /* Allocate on disk a new index allocation buffer. */
        err = indx_add_allocate(indx, ni, &new_vbn);
        if (err)
                goto out;

        /* Allocate and format memory a new index buffer. */
        n2 = indx_new(indx, ni, new_vbn, sub_vbn);
        if (IS_ERR(n2)) {
                err = PTR_ERR(n2);
                goto out;
        }

        hdr2 = &n2->index->ihdr;

        /* Make sp a parent for new buffer. */
        de_set_vbn(up_e, new_vbn);

        /* Copy all the entries <= sp into the new buffer. */
        de_t = hdr_first_de(hdr1);
        to_copy = PtrOffset(de_t, sp);
        hdr_insert_head(hdr2, de_t, to_copy);

        /* Remove all entries (sp including) from hdr1. */
        used = used1 - to_copy - sp_size;
        memmove(de_t, Add2Ptr(sp, sp_size), used - le32_to_cpu(hdr1->de_off));
        hdr1->used = cpu_to_le32(used);

        /*
         * Insert new entry into left or right buffer
         * (depending on sp <=> new_de).
         */
        hdr_insert_de(indx,
                      (*indx->cmp)(new_de + 1, le16_to_cpu(new_de->key_size),
                                   up_e + 1, le16_to_cpu(up_e->key_size),
                                   ctx) < 0 ?
                              hdr2 :
                              hdr1,
                      new_de, NULL, ctx);

        indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits);

        indx_write(indx, ni, n1, 0);
        indx_write(indx, ni, n2, 0);

        put_indx_node(n2);

        /*
         * We've finished splitting everybody, so we are ready to
         * insert the promoted entry into the parent.
         */
        if (!level) {
                /* Insert in root. */
                err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0);
        } else {
                /*
                 * The target buffer's parent is another index buffer.
                 * TODO: Remove recursion.
                 */
                err = indx_insert_into_buffer(indx, ni, root, up_e, ctx,
                                              level - 1, fnd);
        }

        if (err) {
                /*
                 * Undo critical operations.
                 */
                indx_mark_free(indx, ni, new_vbn >> indx->idx2vbn_bits);
                memcpy(hdr1, hdr1_saved, used1);
                indx_write(indx, ni, n1, 0);
        }

out:
        kfree(up_e);
        kfree(hdr1_saved);

        return err;
}

/*
 * indx_insert_entry - Insert new entry into index.
 *
 * @undo - True if we undoing previous remove.
 */
int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
                      const struct NTFS_DE *new_de, const void *ctx,
                      struct ntfs_fnd *fnd, bool undo)
{
        int err;
        int diff;
        struct NTFS_DE *e;
        struct ntfs_fnd *fnd_a = NULL;
        struct INDEX_ROOT *root;

        if (!fnd) {
                fnd_a = fnd_get();
                if (!fnd_a) {
                        err = -ENOMEM;
                        goto out1;
                }
                fnd = fnd_a;
        }

        root = indx_get_root(indx, ni, NULL, NULL);
        if (!root) {
                err = -EINVAL;
                goto out;
        }

        if (fnd_is_empty(fnd)) {
                /*
                 * Find the spot the tree where we want to
                 * insert the new entry.
                 */
                err = indx_find(indx, ni, root, new_de + 1,
                                le16_to_cpu(new_de->key_size), ctx, &diff, &e,
                                fnd);
                if (err)
                        goto out;

                if (!diff) {
                        err = -EEXIST;
                        goto out;
                }
        }

        if (!fnd->level) {
                /*
                 * The root is also a leaf, so we'll insert the
                 * new entry into it.
                 */
                err = indx_insert_into_root(indx, ni, new_de, fnd->root_de, ctx,
                                            fnd, undo);
        } else {
                /*
                 * Found a leaf buffer, so we'll insert the new entry into it.
                 */
                err = indx_insert_into_buffer(indx, ni, root, new_de, ctx,
                                              fnd->level - 1, fnd);
        }

out:
        fnd_put(fnd_a);
out1:
        return err;
}

/*
 * indx_find_buffer - Locate a buffer from the tree.
 */
static struct indx_node *indx_find_buffer(struct ntfs_index *indx,
                                          struct ntfs_inode *ni,
                                          const struct INDEX_ROOT *root,
                                          __le64 vbn, struct indx_node *n)
{
        int err;
        const struct NTFS_DE *e;
        struct indx_node *r;
        const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr;

        /* Step 1: Scan one level. */
        for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) {
                if (!e)
                        return ERR_PTR(-EINVAL);

                if (de_has_vcn(e) && vbn == de_get_vbn_le(e))
                        return n;

                if (de_is_last(e))
                        break;
        }

        /* Step2: Do recursion. */
        e = Add2Ptr(hdr, le32_to_cpu(hdr->de_off));
        for (;;) {
                if (de_has_vcn_ex(e)) {
                        err = indx_read(indx, ni, de_get_vbn(e), &n);
                        if (err)
                                return ERR_PTR(err);

                        r = indx_find_buffer(indx, ni, root, vbn, n);
                        if (r)
                                return r;
                }

                if (de_is_last(e))
                        break;

                e = Add2Ptr(e, le16_to_cpu(e->size));
        }

        return NULL;
}

/*
 * indx_shrink - Deallocate unused tail indexes.
 */
static int indx_shrink(struct ntfs_index *indx, struct ntfs_inode *ni,
                       size_t bit)
{
        int err = 0;
        u64 bpb, new_data;
        size_t nbits;
        struct ATTRIB *b;
        struct ATTR_LIST_ENTRY *le = NULL;
        const struct INDEX_NAMES *in = &s_index_names[indx->type];

        b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len,
                         NULL, NULL);

        if (!b)
                return -ENOENT;

        if (!b->non_res) {
                unsigned long pos;
                const unsigned long *bm = resident_data(b);

                nbits = (size_t)le32_to_cpu(b->res.data_size) * 8;

                if (bit >= nbits)
                        return 0;

                pos = find_next_bit_le(bm, nbits, bit);
                if (pos < nbits)
                        return 0;
        } else {
                size_t used = MINUS_ONE_T;

                nbits = le64_to_cpu(b->nres.data_size) * 8;

                if (bit >= nbits)
                        return 0;

                err = scan_nres_bitmap(ni, b, indx, bit, &scan_for_used, &used);
                if (err)
                        return err;

                if (used != MINUS_ONE_T)
                        return 0;
        }

        new_data = (u64)bit << indx->index_bits;

        err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len,
                            &indx->alloc_run, new_data, &new_data, false, NULL);
        if (err)
                return err;

        if (in->name == I30_NAME)
                i_size_write(&ni->vfs_inode, new_data);

        bpb = ntfs3_bitmap_size(bit);
        if (bpb * 8 == nbits)
                return 0;

        err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len,
                            &indx->bitmap_run, bpb, &bpb, false, NULL);

        return err;
}

static int indx_free_children(struct ntfs_index *indx, struct ntfs_inode *ni,
                              const struct NTFS_DE *e, bool trim)
{
        int err;
        struct indx_node *n = NULL;
        struct INDEX_HDR *hdr;
        CLST vbn = de_get_vbn(e);
        size_t i;

        err = indx_read(indx, ni, vbn, &n);
        if (err)
                return err;

        hdr = &n->index->ihdr;
        /* First, recurse into the children, if any. */
        if (hdr_has_subnode(hdr)) {
                for (e = hdr_first_de(hdr); e; e = hdr_next_de(hdr, e)) {
                        indx_free_children(indx, ni, e, false);
                        if (de_is_last(e))
                                break;
                }
        }

        put_indx_node(n);

        i = vbn >> indx->idx2vbn_bits;
        /*
         * We've gotten rid of the children; add this buffer to the free list.
         */
        indx_mark_free(indx, ni, i);

        if (!trim)
                return 0;

        /*
         * If there are no used indexes after current free index
         * then we can truncate allocation and bitmap.
         * Use bitmap to estimate the case.
         */
        indx_shrink(indx, ni, i + 1);
        return 0;
}

/*
 * indx_get_entry_to_replace
 *
 * Find a replacement entry for a deleted entry.
 * Always returns a node entry:
 * NTFS_IE_HAS_SUBNODES is set the flags and the size includes the sub_vcn.
 */
static int indx_get_entry_to_replace(struct ntfs_index *indx,
                                     struct ntfs_inode *ni,
                                     const struct NTFS_DE *de_next,
                                     struct NTFS_DE **de_to_replace,
                                     struct ntfs_fnd *fnd)
{
        int err;
        int level = -1;
        CLST vbn;
        struct NTFS_DE *e, *te, *re;
        struct indx_node *n;
        struct INDEX_BUFFER *ib;

        *de_to_replace = NULL;

        /* Find first leaf entry down from de_next. */
        vbn = de_get_vbn(de_next);
        for (;;) {
                n = NULL;
                err = indx_read(indx, ni, vbn, &n);
                if (err)
                        goto out;

                e = hdr_first_de(&n->index->ihdr);
                fnd_push(fnd, n, e);

                if (!de_is_last(e)) {
                        /*
                         * This buffer is non-empty, so its first entry
                         * could be used as the replacement entry.
                         */
                        level = fnd->level - 1;
                }

                if (!de_has_vcn(e))
                        break;

                /* This buffer is a node. Continue to go down. */
                vbn = de_get_vbn(e);
        }

        if (level == -1)
                goto out;

        n = fnd->nodes[level];
        te = hdr_first_de(&n->index->ihdr);
        /* Copy the candidate entry into the replacement entry buffer. */
        re = kmalloc(le16_to_cpu(te->size) + sizeof(u64), GFP_NOFS);
        if (!re) {
                err = -ENOMEM;
                goto out;
        }

        *de_to_replace = re;
        memcpy(re, te, le16_to_cpu(te->size));

        if (!de_has_vcn(re)) {
                /*
                 * The replacement entry we found doesn't have a sub_vcn.
                 * increase its size to hold one.
                 */
                le16_add_cpu(&re->size, sizeof(u64));
                re->flags |= NTFS_IE_HAS_SUBNODES;
        } else {
                /*
                 * The replacement entry we found was a node entry, which
                 * means that all its child buffers are empty. Return them
                 * to the free pool.
                 */
                indx_free_children(indx, ni, te, true);
        }

        /*
         * Expunge the replacement entry from its former location,
         * and then write that buffer.
         */
        ib = n->index;
        e = hdr_delete_de(&ib->ihdr, te);

        fnd->de[level] = e;
        indx_write(indx, ni, n, 0);

        if (ib_is_leaf(ib) && ib_is_empty(ib)) {
                /* An empty leaf. */
                return 0;
        }

out:
        fnd_clear(fnd);
        return err;
}

/*
 * indx_delete_entry - Delete an entry from the index.
 */
int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
                      const void *key, u32 key_len, const void *ctx)
{
        int err, diff;
        struct INDEX_ROOT *root;
        struct INDEX_HDR *hdr;
        struct ntfs_fnd *fnd, *fnd2;
        struct INDEX_BUFFER *ib;
        struct NTFS_DE *e, *re, *next, *prev, *me;
        struct indx_node *n, *n2d = NULL;
        __le64 sub_vbn;
        int level, level2;
        struct ATTRIB *attr;
        struct mft_inode *mi;
        u32 e_size, root_size, new_root_size;
        size_t trim_bit;
        const struct INDEX_NAMES *in;

        fnd = fnd_get();
        if (!fnd) {
                err = -ENOMEM;
                goto out2;
        }

        fnd2 = fnd_get();
        if (!fnd2) {
                err = -ENOMEM;
                goto out1;
        }

        root = indx_get_root(indx, ni, &attr, &mi);
        if (!root) {
                err = -EINVAL;
                goto out;
        }

        /* Locate the entry to remove. */
        err = indx_find(indx, ni, root, key, key_len, ctx, &diff, &e, fnd);
        if (err)
                goto out;

        if (!e || diff) {
                err = -ENOENT;
                goto out;
        }

        level = fnd->level;

        if (level) {
                n = fnd->nodes[level - 1];
                e = fnd->de[level - 1];
                ib = n->index;
                hdr = &ib->ihdr;
        } else {
                hdr = &root->ihdr;
                e = fnd->root_de;
                n = NULL;
        }

        e_size = le16_to_cpu(e->size);

        if (!de_has_vcn_ex(e)) {
                /* The entry to delete is a leaf, so we can just rip it out. */
                hdr_delete_de(hdr, e);

                if (!level) {
                        hdr->total = hdr->used;

                        /* Shrink resident root attribute. */
                        mi_resize_attr(mi, attr, 0 - e_size);
                        goto out;
                }

                indx_write(indx, ni, n, 0);

                /*
                 * Check to see if removing that entry made
                 * the leaf empty.
                 */
                if (ib_is_leaf(ib) && ib_is_empty(ib)) {
                        fnd_pop(fnd);
                        fnd_push(fnd2, n, e);
                }
        } else {
                /*
                 * The entry we wish to delete is a node buffer, so we
                 * have to find a replacement for it.
                 */
                next = de_get_next(e);

                err = indx_get_entry_to_replace(indx, ni, next, &re, fnd2);
                if (err)
                        goto out;

                if (re) {
                        de_set_vbn_le(re, de_get_vbn_le(e));
                        hdr_delete_de(hdr, e);

                        err = level ? indx_insert_into_buffer(indx, ni, root,
                                                              re, ctx,
                                                              fnd->level - 1,
                                                              fnd) :
                                      indx_insert_into_root(indx, ni, re, e,
                                                            ctx, fnd, 0);
                        kfree(re);

                        if (err)
                                goto out;
                } else {
                        /*
                         * There is no replacement for the current entry.
                         * This means that the subtree rooted at its node
                         * is empty, and can be deleted, which turn means
                         * that the node can just inherit the deleted
                         * entry sub_vcn.
                         */
                        indx_free_children(indx, ni, next, true);

                        de_set_vbn_le(next, de_get_vbn_le(e));
                        hdr_delete_de(hdr, e);
                        if (level) {
                                indx_write(indx, ni, n, 0);
                        } else {
                                hdr->total = hdr->used;

                                /* Shrink resident root attribute. */
                                mi_resize_attr(mi, attr, 0 - e_size);
                        }
                }
        }

        /* Delete a branch of tree. */
        if (!fnd2 || !fnd2->level)
                goto out;

        /* Reinit root 'cause it can be changed. */
        root = indx_get_root(indx, ni, &attr, &mi);
        if (!root) {
                err = -EINVAL;
                goto out;
        }

        n2d = NULL;
        sub_vbn = fnd2->nodes[0]->index->vbn;
        level2 = 0;
        level = fnd->level;

        hdr = level ? &fnd->nodes[level - 1]->index->ihdr : &root->ihdr;

        /* Scan current level. */
        for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) {
                if (!e) {
                        err = -EINVAL;
                        goto out;
                }

                if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e))
                        break;

                if (de_is_last(e)) {
                        e = NULL;
                        break;
                }
        }

        if (!e) {
                /* Do slow search from root. */
                struct indx_node *in;

                fnd_clear(fnd);

                in = indx_find_buffer(indx, ni, root, sub_vbn, NULL);
                if (IS_ERR(in)) {
                        err = PTR_ERR(in);
                        goto out;
                }

                if (in)
                        fnd_push(fnd, in, NULL);
        }

        /* Merge fnd2 -> fnd. */
        for (level = 0; level < fnd2->level; level++) {
                fnd_push(fnd, fnd2->nodes[level], fnd2->de[level]);
                fnd2->nodes[level] = NULL;
        }
        fnd2->level = 0;

        hdr = NULL;
        for (level = fnd->level; level; level--) {
                struct indx_node *in = fnd->nodes[level - 1];

                ib = in->index;
                if (ib_is_empty(ib)) {
                        sub_vbn = ib->vbn;
                } else {
                        hdr = &ib->ihdr;
                        n2d = in;
                        level2 = level;
                        break;
                }
        }

        if (!hdr)
                hdr = &root->ihdr;

        e = hdr_first_de(hdr);
        if (!e) {
                err = -EINVAL;
                goto out;
        }

        if (hdr != &root->ihdr || !de_is_last(e)) {
                prev = NULL;
                while (!de_is_last(e)) {
                        if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e))
                                break;
                        prev = e;
                        e = hdr_next_de(hdr, e);
                        if (!e) {
                                err = -EINVAL;
                                goto out;
                        }
                }

                if (sub_vbn != de_get_vbn_le(e)) {
                        /*
                         * Didn't find the parent entry, although this buffer
                         * is the parent trail. Something is corrupt.
                         */
                        err = -EINVAL;
                        goto out;
                }

                if (de_is_last(e)) {
                        /*
                         * Since we can't remove the end entry, we'll remove
                         * its predecessor instead. This means we have to
                         * transfer the predecessor's sub_vcn to the end entry.
                         * Note: This index block is not empty, so the
                         * predecessor must exist.
                         */
                        if (!prev) {
                                err = -EINVAL;
                                goto out;
                        }

                        if (de_has_vcn(prev)) {
                                de_set_vbn_le(e, de_get_vbn_le(prev));
                        } else if (de_has_vcn(e)) {
                                le16_sub_cpu(&e->size, sizeof(u64));
                                e->flags &= ~NTFS_IE_HAS_SUBNODES;
                                le32_sub_cpu(&hdr->used, sizeof(u64));
                        }
                        e = prev;
                }

                /*
                 * Copy the current entry into a temporary buffer (stripping
                 * off its down-pointer, if any) and delete it from the current
                 * buffer or root, as appropriate.
                 */
                e_size = le16_to_cpu(e->size);
                me = kmemdup(e, e_size, GFP_NOFS);
                if (!me) {
                        err = -ENOMEM;
                        goto out;
                }

                if (de_has_vcn(me)) {
                        me->flags &= ~NTFS_IE_HAS_SUBNODES;
                        le16_sub_cpu(&me->size, sizeof(u64));
                }

                hdr_delete_de(hdr, e);

                if (hdr == &root->ihdr) {
                        level = 0;
                        hdr->total = hdr->used;

                        /* Shrink resident root attribute. */
                        mi_resize_attr(mi, attr, 0 - e_size);
                } else {
                        indx_write(indx, ni, n2d, 0);
                        level = level2;
                }

                /* Mark unused buffers as free. */
                trim_bit = -1;
                for (; level < fnd->level; level++) {
                        ib = fnd->nodes[level]->index;
                        if (ib_is_empty(ib)) {
                                size_t k = le64_to_cpu(ib->vbn) >>
                                           indx->idx2vbn_bits;

                                indx_mark_free(indx, ni, k);
                                if (k < trim_bit)
                                        trim_bit = k;
                        }
                }

                fnd_clear(fnd);
                /*fnd->root_de = NULL;*/

                /*
                 * Re-insert the entry into the tree.
                 * Find the spot the tree where we want to insert the new entry.
                 */
                err = indx_insert_entry(indx, ni, me, ctx, fnd, 0);
                kfree(me);
                if (err)
                        goto out;

                if (trim_bit != -1)
                        indx_shrink(indx, ni, trim_bit);
        } else {
                /*
                 * This tree needs to be collapsed down to an empty root.
                 * Recreate the index root as an empty leaf and free all
                 * the bits the index allocation bitmap.
                 */
                fnd_clear(fnd);
                fnd_clear(fnd2);

                in = &s_index_names[indx->type];

                err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len,
                                    &indx->alloc_run, 0, NULL, false, NULL);
                if (in->name == I30_NAME)
                        i_size_write(&ni->vfs_inode, 0);

                err = ni_remove_attr(ni, ATTR_ALLOC, in->name, in->name_len,
                                     false, NULL);
                run_close(&indx->alloc_run);

                err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len,
                                    &indx->bitmap_run, 0, NULL, false, NULL);
                err = ni_remove_attr(ni, ATTR_BITMAP, in->name, in->name_len,
                                     false, NULL);
                run_close(&indx->bitmap_run);

                root = indx_get_root(indx, ni, &attr, &mi);
                if (!root) {
                        err = -EINVAL;
                        goto out;
                }

                root_size = le32_to_cpu(attr->res.data_size);
                new_root_size =
                        sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE);

                if (new_root_size != root_size &&
                    !mi_resize_attr(mi, attr, new_root_size - root_size)) {
                        err = -EINVAL;
                        goto out;
                }

                /* Fill first entry. */
                e = (struct NTFS_DE *)(root + 1);
                e->ref.low = 0;
                e->ref.high = 0;
                e->ref.seq = 0;
                e->size = cpu_to_le16(sizeof(struct NTFS_DE));
                e->flags = NTFS_IE_LAST; // 0x02
                e->key_size = 0;
                e->res = 0;

                hdr = &root->ihdr;
                hdr->flags = 0;
                hdr->used = hdr->total = cpu_to_le32(
                        new_root_size - offsetof(struct INDEX_ROOT, ihdr));
                mi->dirty = true;
        }

out:
        fnd_put(fnd2);
out1:
        fnd_put(fnd);
out2:
        return err;
}

/*
 * Update duplicated information in directory entry
 * 'dup' - info from MFT record
 */
int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi,
                    const struct ATTR_FILE_NAME *fname,
                    const struct NTFS_DUP_INFO *dup, int sync)
{
        int err, diff;
        struct NTFS_DE *e = NULL;
        struct ATTR_FILE_NAME *e_fname;
        struct ntfs_fnd *fnd;
        struct INDEX_ROOT *root;
        struct mft_inode *mi;
        struct ntfs_index *indx = &ni->dir;

        fnd = fnd_get();
        if (!fnd)
                return -ENOMEM;

        root = indx_get_root(indx, ni, NULL, &mi);
        if (!root) {
                err = -EINVAL;
                goto out;
        }

        /* Find entry in directory. */
        err = indx_find(indx, ni, root, fname, fname_full_size(fname), sbi,
                        &diff, &e, fnd);
        if (err)
                goto out;

        if (!e) {
                err = -EINVAL;
                goto out;
        }

        if (diff) {
                err = -EINVAL;
                goto out;
        }

        e_fname = (struct ATTR_FILE_NAME *)(e + 1);

        if (!memcmp(&e_fname->dup, dup, sizeof(*dup))) {
                /*
                 * Nothing to update in index! Try to avoid this call.
                 */
                goto out;
        }

        memcpy(&e_fname->dup, dup, sizeof(*dup));

        if (fnd->level) {
                /* Directory entry in index. */
                err = indx_write(indx, ni, fnd->nodes[fnd->level - 1], sync);
        } else {
                /* Directory entry in directory MFT record. */
                mi->dirty = true;
                if (sync)
                        err = mi_write(mi, 1);
                else
                        mark_inode_dirty(&ni->vfs_inode);
        }

out:
        fnd_put(fnd);
        return err;
}































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Key-agreement Protocol Primitives (KPP)
 *
 * Copyright (c) 2016, Intel Corporation
 * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
 */

#ifndef _CRYPTO_KPP_
#define _CRYPTO_KPP_

#include <linux/atomic.h>
#include <linux/container_of.h>
#include <linux/crypto.h>
#include <linux/slab.h>

/**
 * struct kpp_request
 *
 * @base:        Common attributes for async crypto requests
 * @src:        Source data
 * @dst:        Destination data
 * @src_len:        Size of the input buffer
 * @dst_len:        Size of the output buffer. It needs to be at least
 *                as big as the expected result depending        on the operation
 *                After operation it will be updated with the actual size of the
 *                result. In case of error where the dst sgl size was insufficient,
 *                it will be updated to the size required for the operation.
 * @__ctx:        Start of private context data
 */
struct kpp_request {
        struct crypto_async_request base;
        struct scatterlist *src;
        struct scatterlist *dst;
        unsigned int src_len;
        unsigned int dst_len;
        void *__ctx[] CRYPTO_MINALIGN_ATTR;
};

/**
 * struct crypto_kpp - user-instantiated object which encapsulate
 * algorithms and core processing logic
 *
 * @reqsize:                Request context size required by algorithm
 *                        implementation
 * @base:        Common crypto API algorithm data structure
 */
struct crypto_kpp {
        unsigned int reqsize;

        struct crypto_tfm base;
};

/**
 * struct kpp_alg - generic key-agreement protocol primitives
 *
 * @set_secret:                Function invokes the protocol specific function to
 *                        store the secret private key along with parameters.
 *                        The implementation knows how to decode the buffer
 * @generate_public_key: Function generate the public key to be sent to the
 *                        counterpart. In case of error, where output is not big
 *                        enough req->dst_len will be updated to the size
 *                        required
 * @compute_shared_secret: Function compute the shared secret as defined by
 *                        the algorithm. The result is given back to the user.
 *                        In case of error, where output is not big enough,
 *                        req->dst_len will be updated to the size required
 * @max_size:                Function returns the size of the output buffer
 * @init:                Initialize the object. This is called only once at
 *                        instantiation time. In case the cryptographic hardware
 *                        needs to be initialized. Software fallback should be
 *                        put in place here.
 * @exit:                Undo everything @init did.
 *
 * @base:                Common crypto API algorithm data structure
 */
struct kpp_alg {
        int (*set_secret)(struct crypto_kpp *tfm, const void *buffer,
                          unsigned int len);
        int (*generate_public_key)(struct kpp_request *req);
        int (*compute_shared_secret)(struct kpp_request *req);

        unsigned int (*max_size)(struct crypto_kpp *tfm);

        int (*init)(struct crypto_kpp *tfm);
        void (*exit)(struct crypto_kpp *tfm);

        struct crypto_alg base;
};

/**
 * DOC: Generic Key-agreement Protocol Primitives API
 *
 * The KPP API is used with the algorithm type
 * CRYPTO_ALG_TYPE_KPP (listed as type "kpp" in /proc/crypto)
 */

/**
 * crypto_alloc_kpp() - allocate KPP tfm handle
 * @alg_name: is the name of the kpp algorithm (e.g. "dh", "ecdh")
 * @type: specifies the type of the algorithm
 * @mask: specifies the mask for the algorithm
 *
 * Allocate a handle for kpp algorithm. The returned struct crypto_kpp
 * is required for any following API invocation
 *
 * Return: allocated handle in case of success; IS_ERR() is true in case of
 *           an error, PTR_ERR() returns the error code.
 */
struct crypto_kpp *crypto_alloc_kpp(const char *alg_name, u32 type, u32 mask);

int crypto_has_kpp(const char *alg_name, u32 type, u32 mask);

static inline struct crypto_tfm *crypto_kpp_tfm(struct crypto_kpp *tfm)
{
        return &tfm->base;
}

static inline struct kpp_alg *__crypto_kpp_alg(struct crypto_alg *alg)
{
        return container_of(alg, struct kpp_alg, base);
}

static inline struct crypto_kpp *__crypto_kpp_tfm(struct crypto_tfm *tfm)
{
        return container_of(tfm, struct crypto_kpp, base);
}

static inline struct kpp_alg *crypto_kpp_alg(struct crypto_kpp *tfm)
{
        return __crypto_kpp_alg(crypto_kpp_tfm(tfm)->__crt_alg);
}

static inline unsigned int crypto_kpp_reqsize(struct crypto_kpp *tfm)
{
        return tfm->reqsize;
}

static inline void kpp_request_set_tfm(struct kpp_request *req,
                                       struct crypto_kpp *tfm)
{
        req->base.tfm = crypto_kpp_tfm(tfm);
}

static inline struct crypto_kpp *crypto_kpp_reqtfm(struct kpp_request *req)
{
        return __crypto_kpp_tfm(req->base.tfm);
}

static inline u32 crypto_kpp_get_flags(struct crypto_kpp *tfm)
{
        return crypto_tfm_get_flags(crypto_kpp_tfm(tfm));
}

static inline void crypto_kpp_set_flags(struct crypto_kpp *tfm, u32 flags)
{
        crypto_tfm_set_flags(crypto_kpp_tfm(tfm), flags);
}

/**
 * crypto_free_kpp() - free KPP tfm handle
 *
 * @tfm: KPP tfm handle allocated with crypto_alloc_kpp()
 *
 * If @tfm is a NULL or error pointer, this function does nothing.
 */
static inline void crypto_free_kpp(struct crypto_kpp *tfm)
{
        crypto_destroy_tfm(tfm, crypto_kpp_tfm(tfm));
}

/**
 * kpp_request_alloc() - allocates kpp request
 *
 * @tfm:        KPP tfm handle allocated with crypto_alloc_kpp()
 * @gfp:        allocation flags
 *
 * Return: allocated handle in case of success or NULL in case of an error.
 */
static inline struct kpp_request *kpp_request_alloc(struct crypto_kpp *tfm,
                                                    gfp_t gfp)
{
        struct kpp_request *req;

        req = kmalloc(sizeof(*req) + crypto_kpp_reqsize(tfm), gfp);
        if (likely(req))
                kpp_request_set_tfm(req, tfm);

        return req;
}

/**
 * kpp_request_free() - zeroize and free kpp request
 *
 * @req:        request to free
 */
static inline void kpp_request_free(struct kpp_request *req)
{
        kfree_sensitive(req);
}

/**
 * kpp_request_set_callback() - Sets an asynchronous callback.
 *
 * Callback will be called when an asynchronous operation on a given
 * request is finished.
 *
 * @req:        request that the callback will be set for
 * @flgs:        specify for instance if the operation may backlog
 * @cmpl:        callback which will be called
 * @data:        private data used by the caller
 */
static inline void kpp_request_set_callback(struct kpp_request *req,
                                            u32 flgs,
                                            crypto_completion_t cmpl,
                                            void *data)
{
        req->base.complete = cmpl;
        req->base.data = data;
        req->base.flags = flgs;
}

/**
 * kpp_request_set_input() - Sets input buffer
 *
 * Sets parameters required by generate_public_key
 *
 * @req:        kpp request
 * @input:        ptr to input scatter list
 * @input_len:        size of the input scatter list
 */
static inline void kpp_request_set_input(struct kpp_request *req,
                                         struct scatterlist *input,
                                         unsigned int input_len)
{
        req->src = input;
        req->src_len = input_len;
}

/**
 * kpp_request_set_output() - Sets output buffer
 *
 * Sets parameters required by kpp operation
 *
 * @req:        kpp request
 * @output:        ptr to output scatter list
 * @output_len:        size of the output scatter list
 */
static inline void kpp_request_set_output(struct kpp_request *req,
                                          struct scatterlist *output,
                                          unsigned int output_len)
{
        req->dst = output;
        req->dst_len = output_len;
}

enum {
        CRYPTO_KPP_SECRET_TYPE_UNKNOWN,
        CRYPTO_KPP_SECRET_TYPE_DH,
        CRYPTO_KPP_SECRET_TYPE_ECDH,
};

/**
 * struct kpp_secret - small header for packing secret buffer
 *
 * @type:        define type of secret. Each kpp type will define its own
 * @len:        specify the len of the secret, include the header, that
 *                follows the struct
 */
struct kpp_secret {
        unsigned short type;
        unsigned short len;
};

/**
 * crypto_kpp_set_secret() - Invoke kpp operation
 *
 * Function invokes the specific kpp operation for a given alg.
 *
 * @tfm:        tfm handle
 * @buffer:        Buffer holding the packet representation of the private
 *                key. The structure of the packet key depends on the particular
 *                KPP implementation. Packing and unpacking helpers are provided
 *                for ECDH and DH (see the respective header files for those
 *                implementations).
 * @len:        Length of the packet private key buffer.
 *
 * Return: zero on success; error code in case of error
 */
static inline int crypto_kpp_set_secret(struct crypto_kpp *tfm,
                                        const void *buffer, unsigned int len)
{
        return crypto_kpp_alg(tfm)->set_secret(tfm, buffer, len);
}

/**
 * crypto_kpp_generate_public_key() - Invoke kpp operation
 *
 * Function invokes the specific kpp operation for generating the public part
 * for a given kpp algorithm.
 *
 * To generate a private key, the caller should use a random number generator.
 * The output of the requested length serves as the private key.
 *
 * @req:        kpp key request
 *
 * Return: zero on success; error code in case of error
 */
static inline int crypto_kpp_generate_public_key(struct kpp_request *req)
{
        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);

        return crypto_kpp_alg(tfm)->generate_public_key(req);
}

/**
 * crypto_kpp_compute_shared_secret() - Invoke kpp operation
 *
 * Function invokes the specific kpp operation for computing the shared secret
 * for a given kpp algorithm.
 *
 * @req:        kpp key request
 *
 * Return: zero on success; error code in case of error
 */
static inline int crypto_kpp_compute_shared_secret(struct kpp_request *req)
{
        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);

        return crypto_kpp_alg(tfm)->compute_shared_secret(req);
}

/**
 * crypto_kpp_maxsize() - Get len for output buffer
 *
 * Function returns the output buffer size required for a given key.
 * Function assumes that the key is already set in the transformation. If this
 * function is called without a setkey or with a failed setkey, you will end up
 * in a NULL dereference.
 *
 * @tfm:        KPP tfm handle allocated with crypto_alloc_kpp()
 */
static inline unsigned int crypto_kpp_maxsize(struct crypto_kpp *tfm)
{
        struct kpp_alg *alg = crypto_kpp_alg(tfm);

        return alg->max_size(tfm);
}

#endif

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   19 

   19 
   19 


























































































































































































































































































































































































































































































































































































































































































































































































































































   70 








   70 












































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/nfs/inode.c
 *
 *  Copyright (C) 1992  Rick Sladkey
 *
 *  nfs inode and superblock handling functions
 *
 *  Modularised by Alan Cox <alan@lxorguk.ukuu.org.uk>, while hacking some
 *  experimental NFS changes. Modularisation taken straight from SYS5 fs.
 *
 *  Change to nfs_read_super() to permit NFS mounts to multi-homed hosts.
 *  J.S.Peatfield@damtp.cam.ac.uk
 *
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/sched/signal.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/unistd.h>
#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/metrics.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs4_mount.h>
#include <linux/lockd/bind.h>
#include <linux/seq_file.h>
#include <linux/mount.h>
#include <linux/vfs.h>
#include <linux/inet.h>
#include <linux/nfs_xdr.h>
#include <linux/slab.h>
#include <linux/compat.h>
#include <linux/freezer.h>
#include <linux/uaccess.h>
#include <linux/iversion.h>

#include "nfs4_fs.h"
#include "callback.h"
#include "delegation.h"
#include "iostat.h"
#include "internal.h"
#include "fscache.h"
#include "pnfs.h"
#include "nfs.h"
#include "netns.h"
#include "sysfs.h"

#include "nfstrace.h"

#define NFSDBG_FACILITY                NFSDBG_VFS

#define NFS_64_BIT_INODE_NUMBERS_ENABLED        1

/* Default is to see 64-bit inode numbers */
static bool enable_ino64 = NFS_64_BIT_INODE_NUMBERS_ENABLED;

static int nfs_update_inode(struct inode *, struct nfs_fattr *);

static struct kmem_cache * nfs_inode_cachep;

static inline unsigned long
nfs_fattr_to_ino_t(struct nfs_fattr *fattr)
{
        return nfs_fileid_to_ino_t(fattr->fileid);
}

int nfs_wait_bit_killable(struct wait_bit_key *key, int mode)
{
        schedule();
        if (signal_pending_state(mode, current))
                return -ERESTARTSYS;
        return 0;
}
EXPORT_SYMBOL_GPL(nfs_wait_bit_killable);

/**
 * nfs_compat_user_ino64 - returns the user-visible inode number
 * @fileid: 64-bit fileid
 *
 * This function returns a 32-bit inode number if the boot parameter
 * nfs.enable_ino64 is zero.
 */
u64 nfs_compat_user_ino64(u64 fileid)
{
#ifdef CONFIG_COMPAT
        compat_ulong_t ino;
#else        
        unsigned long ino;
#endif

        if (enable_ino64)
                return fileid;
        ino = fileid;
        if (sizeof(ino) < sizeof(fileid))
                ino ^= fileid >> (sizeof(fileid)-sizeof(ino)) * 8;
        return ino;
}

int nfs_drop_inode(struct inode *inode)
{
        return NFS_STALE(inode) || generic_drop_inode(inode);
}
EXPORT_SYMBOL_GPL(nfs_drop_inode);

void nfs_clear_inode(struct inode *inode)
{
        /*
         * The following should never happen...
         */
        WARN_ON_ONCE(nfs_have_writebacks(inode));
        WARN_ON_ONCE(!list_empty(&NFS_I(inode)->open_files));
        nfs_zap_acl_cache(inode);
        nfs_access_zap_cache(inode);
        nfs_fscache_clear_inode(inode);
}
EXPORT_SYMBOL_GPL(nfs_clear_inode);

void nfs_evict_inode(struct inode *inode)
{
        truncate_inode_pages_final(&inode->i_data);
        clear_inode(inode);
        nfs_clear_inode(inode);
}

int nfs_sync_inode(struct inode *inode)
{
        inode_dio_wait(inode);
        return nfs_wb_all(inode);
}
EXPORT_SYMBOL_GPL(nfs_sync_inode);

/**
 * nfs_sync_mapping - helper to flush all mmapped dirty data to disk
 * @mapping: pointer to struct address_space
 */
int nfs_sync_mapping(struct address_space *mapping)
{
        int ret = 0;

        if (mapping->nrpages != 0) {
                unmap_mapping_range(mapping, 0, 0, 0);
                ret = nfs_wb_all(mapping->host);
        }
        return ret;
}

static int nfs_attribute_timeout(struct inode *inode)
{
        struct nfs_inode *nfsi = NFS_I(inode);

        return !time_in_range_open(jiffies, nfsi->read_cache_jiffies, nfsi->read_cache_jiffies + nfsi->attrtimeo);
}

static bool nfs_check_cache_flags_invalid(struct inode *inode,
                                          unsigned long flags)
{
        unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity);

        return (cache_validity & flags) != 0;
}

bool nfs_check_cache_invalid(struct inode *inode, unsigned long flags)
{
        if (nfs_check_cache_flags_invalid(inode, flags))
                return true;
        return nfs_attribute_cache_expired(inode);
}
EXPORT_SYMBOL_GPL(nfs_check_cache_invalid);

#ifdef CONFIG_NFS_V4_2
static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi)
{
        return nfsi->xattr_cache != NULL;
}
#else
static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi)
{
        return false;
}
#endif

void nfs_set_cache_invalid(struct inode *inode, unsigned long flags)
{
        struct nfs_inode *nfsi = NFS_I(inode);

        if (nfs_have_delegated_attributes(inode)) {
                if (!(flags & NFS_INO_REVAL_FORCED))
                        flags &= ~(NFS_INO_INVALID_MODE |
                                   NFS_INO_INVALID_OTHER |
                                   NFS_INO_INVALID_XATTR);
                flags &= ~(NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE);
        }

        if (!nfs_has_xattr_cache(nfsi))
                flags &= ~NFS_INO_INVALID_XATTR;
        if (flags & NFS_INO_INVALID_DATA)
                nfs_fscache_invalidate(inode, 0);
        flags &= ~NFS_INO_REVAL_FORCED;

        flags |= nfsi->cache_validity;
        if (inode->i_mapping->nrpages == 0)
                flags &= ~NFS_INO_INVALID_DATA;

        /* pairs with nfs_clear_invalid_mapping()'s smp_load_acquire() */
        smp_store_release(&nfsi->cache_validity, flags);

        if (inode->i_mapping->nrpages == 0 ||
            nfsi->cache_validity & NFS_INO_INVALID_DATA) {
                nfs_ooo_clear(nfsi);
        }
        trace_nfs_set_cache_invalid(inode, 0);
}
EXPORT_SYMBOL_GPL(nfs_set_cache_invalid);

/*
 * Invalidate the local caches
 */
static void nfs_zap_caches_locked(struct inode *inode)
{
        struct nfs_inode *nfsi = NFS_I(inode);
        int mode = inode->i_mode;

        nfs_inc_stats(inode, NFSIOS_ATTRINVALIDATE);

        nfsi->attrtimeo = NFS_MINATTRTIMEO(inode);
        nfsi->attrtimeo_timestamp = jiffies;

        if (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATTR |
                                                     NFS_INO_INVALID_DATA |
                                                     NFS_INO_INVALID_ACCESS |
                                                     NFS_INO_INVALID_ACL |
                                                     NFS_INO_INVALID_XATTR);
        else
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATTR |
                                                     NFS_INO_INVALID_ACCESS |
                                                     NFS_INO_INVALID_ACL |
                                                     NFS_INO_INVALID_XATTR);
        nfs_zap_label_cache_locked(nfsi);
}

void nfs_zap_caches(struct inode *inode)
{
        spin_lock(&inode->i_lock);
        nfs_zap_caches_locked(inode);
        spin_unlock(&inode->i_lock);
}

void nfs_zap_mapping(struct inode *inode, struct address_space *mapping)
{
        if (mapping->nrpages != 0) {
                spin_lock(&inode->i_lock);
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
                spin_unlock(&inode->i_lock);
        }
}

void nfs_zap_acl_cache(struct inode *inode)
{
        void (*clear_acl_cache)(struct inode *);

        clear_acl_cache = NFS_PROTO(inode)->clear_acl_cache;
        if (clear_acl_cache != NULL)
                clear_acl_cache(inode);
        spin_lock(&inode->i_lock);
        NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_ACL;
        spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_zap_acl_cache);

void nfs_invalidate_atime(struct inode *inode)
{
        if (nfs_have_delegated_atime(inode))
                return;
        spin_lock(&inode->i_lock);
        nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
        spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_invalidate_atime);

/*
 * Invalidate, but do not unhash, the inode.
 * NB: must be called with inode->i_lock held!
 */
static void nfs_set_inode_stale_locked(struct inode *inode)
{
        set_bit(NFS_INO_STALE, &NFS_I(inode)->flags);
        nfs_zap_caches_locked(inode);
        trace_nfs_set_inode_stale(inode);
}

void nfs_set_inode_stale(struct inode *inode)
{
        spin_lock(&inode->i_lock);
        nfs_set_inode_stale_locked(inode);
        spin_unlock(&inode->i_lock);
}

struct nfs_find_desc {
        struct nfs_fh                *fh;
        struct nfs_fattr        *fattr;
};

/*
 * In NFSv3 we can have 64bit inode numbers. In order to support
 * this, and re-exported directories (also seen in NFSv2)
 * we are forced to allow 2 different inodes to have the same
 * i_ino.
 */
static int
nfs_find_actor(struct inode *inode, void *opaque)
{
        struct nfs_find_desc        *desc = opaque;
        struct nfs_fh                *fh = desc->fh;
        struct nfs_fattr        *fattr = desc->fattr;

        if (NFS_FILEID(inode) != fattr->fileid)
                return 0;
        if (inode_wrong_type(inode, fattr->mode))
                return 0;
        if (nfs_compare_fh(NFS_FH(inode), fh))
                return 0;
        if (is_bad_inode(inode) || NFS_STALE(inode))
                return 0;
        return 1;
}

static int
nfs_init_locked(struct inode *inode, void *opaque)
{
        struct nfs_find_desc        *desc = opaque;
        struct nfs_fattr        *fattr = desc->fattr;

        set_nfs_fileid(inode, fattr->fileid);
        inode->i_mode = fattr->mode;
        nfs_copy_fh(NFS_FH(inode), desc->fh);
        return 0;
}

#ifdef CONFIG_NFS_V4_SECURITY_LABEL
static void nfs_clear_label_invalid(struct inode *inode)
{
        spin_lock(&inode->i_lock);
        NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_LABEL;
        spin_unlock(&inode->i_lock);
}

void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr)
{
        int error;

        if (fattr->label == NULL)
                return;

        if ((fattr->valid & NFS_ATTR_FATTR_V4_SECURITY_LABEL) && inode->i_security) {
                error = security_inode_notifysecctx(inode, fattr->label->label,
                                fattr->label->len);
                if (error)
                        printk(KERN_ERR "%s() %s %d "
                                        "security_inode_notifysecctx() %d\n",
                                        __func__,
                                        (char *)fattr->label->label,
                                        fattr->label->len, error);
                nfs_clear_label_invalid(inode);
        }
}

struct nfs4_label *nfs4_label_alloc(struct nfs_server *server, gfp_t flags)
{
        struct nfs4_label *label;

        if (!(server->caps & NFS_CAP_SECURITY_LABEL))
                return NULL;

        label = kzalloc(sizeof(struct nfs4_label), flags);
        if (label == NULL)
                return ERR_PTR(-ENOMEM);

        label->label = kzalloc(NFS4_MAXLABELLEN, flags);
        if (label->label == NULL) {
                kfree(label);
                return ERR_PTR(-ENOMEM);
        }
        label->len = NFS4_MAXLABELLEN;

        return label;
}
EXPORT_SYMBOL_GPL(nfs4_label_alloc);
#else
void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr)
{
}
#endif
EXPORT_SYMBOL_GPL(nfs_setsecurity);

/* Search for inode identified by fh, fileid and i_mode in inode cache. */
struct inode *
nfs_ilookup(struct super_block *sb, struct nfs_fattr *fattr, struct nfs_fh *fh)
{
        struct nfs_find_desc desc = {
                .fh        = fh,
                .fattr        = fattr,
        };
        struct inode *inode;
        unsigned long hash;

        if (!(fattr->valid & NFS_ATTR_FATTR_FILEID) ||
            !(fattr->valid & NFS_ATTR_FATTR_TYPE))
                return NULL;

        hash = nfs_fattr_to_ino_t(fattr);
        inode = ilookup5(sb, hash, nfs_find_actor, &desc);

        dprintk("%s: returning %p\n", __func__, inode);
        return inode;
}

static void nfs_inode_init_regular(struct nfs_inode *nfsi)
{
        atomic_long_set(&nfsi->nrequests, 0);
        atomic_long_set(&nfsi->redirtied_pages, 0);
        INIT_LIST_HEAD(&nfsi->commit_info.list);
        atomic_long_set(&nfsi->commit_info.ncommit, 0);
        atomic_set(&nfsi->commit_info.rpcs_out, 0);
        mutex_init(&nfsi->commit_mutex);
}

static void nfs_inode_init_dir(struct nfs_inode *nfsi)
{
        nfsi->cache_change_attribute = 0;
        memset(nfsi->cookieverf, 0, sizeof(nfsi->cookieverf));
        init_rwsem(&nfsi->rmdir_sem);
}

/*
 * This is our front-end to iget that looks up inodes by file handle
 * instead of inode number.
 */
struct inode *
nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr)
{
        struct nfs_find_desc desc = {
                .fh        = fh,
                .fattr        = fattr
        };
        struct inode *inode = ERR_PTR(-ENOENT);
        u64 fattr_supported = NFS_SB(sb)->fattr_valid;
        unsigned long hash;

        nfs_attr_check_mountpoint(sb, fattr);

        if (nfs_attr_use_mounted_on_fileid(fattr))
                fattr->fileid = fattr->mounted_on_fileid;
        else if ((fattr->valid & NFS_ATTR_FATTR_FILEID) == 0)
                goto out_no_inode;
        if ((fattr->valid & NFS_ATTR_FATTR_TYPE) == 0)
                goto out_no_inode;

        hash = nfs_fattr_to_ino_t(fattr);

        inode = iget5_locked(sb, hash, nfs_find_actor, nfs_init_locked, &desc);
        if (inode == NULL) {
                inode = ERR_PTR(-ENOMEM);
                goto out_no_inode;
        }

        if (inode->i_state & I_NEW) {
                struct nfs_inode *nfsi = NFS_I(inode);
                unsigned long now = jiffies;

                /* We set i_ino for the few things that still rely on it,
                 * such as stat(2) */
                inode->i_ino = hash;

                /* We can't support update_atime(), since the server will reset it */
                inode->i_flags |= S_NOATIME|S_NOCMTIME;
                inode->i_mode = fattr->mode;
                nfsi->cache_validity = 0;
                if ((fattr->valid & NFS_ATTR_FATTR_MODE) == 0
                                && (fattr_supported & NFS_ATTR_FATTR_MODE))
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE);
                /* Why so? Because we want revalidate for devices/FIFOs, and
                 * that's precisely what we have in nfs_file_inode_operations.
                 */
                inode->i_op = NFS_SB(sb)->nfs_client->rpc_ops->file_inode_ops;
                if (S_ISREG(inode->i_mode)) {
                        inode->i_fop = NFS_SB(sb)->nfs_client->rpc_ops->file_ops;
                        inode->i_data.a_ops = &nfs_file_aops;
                        nfs_inode_init_regular(nfsi);
                        mapping_set_large_folios(inode->i_mapping);
                } else if (S_ISDIR(inode->i_mode)) {
                        inode->i_op = NFS_SB(sb)->nfs_client->rpc_ops->dir_inode_ops;
                        inode->i_fop = &nfs_dir_operations;
                        inode->i_data.a_ops = &nfs_dir_aops;
                        nfs_inode_init_dir(nfsi);
                        /* Deal with crossing mountpoints */
                        if (fattr->valid & NFS_ATTR_FATTR_MOUNTPOINT ||
                                        fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL) {
                                if (fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL)
                                        inode->i_op = &nfs_referral_inode_operations;
                                else
                                        inode->i_op = &nfs_mountpoint_inode_operations;
                                inode->i_fop = NULL;
                                inode->i_flags |= S_AUTOMOUNT;
                        }
                } else if (S_ISLNK(inode->i_mode)) {
                        inode->i_op = &nfs_symlink_inode_operations;
                        inode_nohighmem(inode);
                } else
                        init_special_inode(inode, inode->i_mode, fattr->rdev);

                inode_set_atime(inode, 0, 0);
                inode_set_mtime(inode, 0, 0);
                inode_set_ctime(inode, 0, 0);
                inode_set_iversion_raw(inode, 0);
                inode->i_size = 0;
                clear_nlink(inode);
                inode->i_uid = make_kuid(&init_user_ns, -2);
                inode->i_gid = make_kgid(&init_user_ns, -2);
                inode->i_blocks = 0;
                nfsi->write_io = 0;
                nfsi->read_io = 0;

                nfsi->read_cache_jiffies = fattr->time_start;
                nfsi->attr_gencount = fattr->gencount;
                if (fattr->valid & NFS_ATTR_FATTR_ATIME)
                        inode_set_atime_to_ts(inode, fattr->atime);
                else if (fattr_supported & NFS_ATTR_FATTR_ATIME)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);
                if (fattr->valid & NFS_ATTR_FATTR_MTIME)
                        inode_set_mtime_to_ts(inode, fattr->mtime);
                else if (fattr_supported & NFS_ATTR_FATTR_MTIME)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
                if (fattr->valid & NFS_ATTR_FATTR_CTIME)
                        inode_set_ctime_to_ts(inode, fattr->ctime);
                else if (fattr_supported & NFS_ATTR_FATTR_CTIME)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME);
                if (fattr->valid & NFS_ATTR_FATTR_CHANGE)
                        inode_set_iversion_raw(inode, fattr->change_attr);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE);
                if (fattr->valid & NFS_ATTR_FATTR_SIZE)
                        inode->i_size = nfs_size_to_loff_t(fattr->size);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_SIZE);
                if (fattr->valid & NFS_ATTR_FATTR_NLINK)
                        set_nlink(inode, fattr->nlink);
                else if (fattr_supported & NFS_ATTR_FATTR_NLINK)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_NLINK);
                if (fattr->valid & NFS_ATTR_FATTR_OWNER)
                        inode->i_uid = fattr->uid;
                else if (fattr_supported & NFS_ATTR_FATTR_OWNER)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER);
                if (fattr->valid & NFS_ATTR_FATTR_GROUP)
                        inode->i_gid = fattr->gid;
                else if (fattr_supported & NFS_ATTR_FATTR_GROUP)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER);
                if (fattr->valid & NFS_ATTR_FATTR_BLOCKS_USED)
                        inode->i_blocks = fattr->du.nfs2.blocks;
                else if (fattr_supported & NFS_ATTR_FATTR_BLOCKS_USED &&
                         fattr->size != 0)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);
                if (fattr->valid & NFS_ATTR_FATTR_SPACE_USED) {
                        /*
                         * report the blocks in 512byte units
                         */
                        inode->i_blocks = nfs_calc_block_size(fattr->du.nfs3.used);
                } else if (fattr_supported & NFS_ATTR_FATTR_SPACE_USED &&
                           fattr->size != 0)
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);

                nfs_setsecurity(inode, fattr);

                nfsi->attrtimeo = NFS_MINATTRTIMEO(inode);
                nfsi->attrtimeo_timestamp = now;
                nfsi->access_cache = RB_ROOT;

                nfs_fscache_init_inode(inode);

                unlock_new_inode(inode);
        } else {
                int err = nfs_refresh_inode(inode, fattr);
                if (err < 0) {
                        iput(inode);
                        inode = ERR_PTR(err);
                        goto out_no_inode;
                }
        }
        dprintk("NFS: nfs_fhget(%s/%Lu fh_crc=0x%08x ct=%d)\n",
                inode->i_sb->s_id,
                (unsigned long long)NFS_FILEID(inode),
                nfs_display_fhandle_hash(fh),
                atomic_read(&inode->i_count));

out:
        return inode;

out_no_inode:
        dprintk("nfs_fhget: iget failed with error %ld\n", PTR_ERR(inode));
        goto out;
}
EXPORT_SYMBOL_GPL(nfs_fhget);

static void
nfs_fattr_fixup_delegated(struct inode *inode, struct nfs_fattr *fattr)
{
        unsigned long cache_validity = NFS_I(inode)->cache_validity;

        if (nfs_have_delegated_mtime(inode)) {
                if (!(cache_validity & NFS_INO_INVALID_CTIME))
                        fattr->valid &= ~(NFS_ATTR_FATTR_PRECTIME |
                                          NFS_ATTR_FATTR_CTIME);

                if (!(cache_validity & NFS_INO_INVALID_MTIME))
                        fattr->valid &= ~(NFS_ATTR_FATTR_PREMTIME |
                                          NFS_ATTR_FATTR_MTIME);

                if (!(cache_validity & NFS_INO_INVALID_ATIME))
                        fattr->valid &= ~NFS_ATTR_FATTR_ATIME;
        } else if (nfs_have_delegated_atime(inode)) {
                if (!(cache_validity & NFS_INO_INVALID_ATIME))
                        fattr->valid &= ~NFS_ATTR_FATTR_ATIME;
        }
}

static void nfs_update_timestamps(struct inode *inode, unsigned int ia_valid)
{
        enum file_time_flags time_flags = 0;
        unsigned int cache_flags = 0;

        if (ia_valid & ATTR_MTIME) {
                time_flags |= S_MTIME | S_CTIME;
                cache_flags |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME;
        }
        if (ia_valid & ATTR_ATIME) {
                time_flags |= S_ATIME;
                cache_flags |= NFS_INO_INVALID_ATIME;
        }
        inode_update_timestamps(inode, time_flags);
        NFS_I(inode)->cache_validity &= ~cache_flags;
}

void nfs_update_delegated_atime(struct inode *inode)
{
        spin_lock(&inode->i_lock);
        if (nfs_have_delegated_atime(inode))
                nfs_update_timestamps(inode, ATTR_ATIME);
        spin_unlock(&inode->i_lock);
}

void nfs_update_delegated_mtime_locked(struct inode *inode)
{
        if (nfs_have_delegated_mtime(inode))
                nfs_update_timestamps(inode, ATTR_MTIME);
}

void nfs_update_delegated_mtime(struct inode *inode)
{
        spin_lock(&inode->i_lock);
        nfs_update_delegated_mtime_locked(inode);
        spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime);

#define NFS_VALID_ATTRS (ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_SIZE|ATTR_ATIME|ATTR_ATIME_SET|ATTR_MTIME|ATTR_MTIME_SET|ATTR_FILE|ATTR_OPEN)

int
nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
            struct iattr *attr)
{
        struct inode *inode = d_inode(dentry);
        struct nfs_fattr *fattr;
        int error = 0;

        nfs_inc_stats(inode, NFSIOS_VFSSETATTR);

        /* skip mode change if it's just for clearing setuid/setgid */
        if (attr->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID))
                attr->ia_valid &= ~ATTR_MODE;

        if (attr->ia_valid & ATTR_SIZE) {
                BUG_ON(!S_ISREG(inode->i_mode));

                error = inode_newsize_ok(inode, attr->ia_size);
                if (error)
                        return error;

                if (attr->ia_size == i_size_read(inode))
                        attr->ia_valid &= ~ATTR_SIZE;
        }

        if (nfs_have_delegated_mtime(inode) && attr->ia_valid & ATTR_MTIME) {
                spin_lock(&inode->i_lock);
                nfs_update_timestamps(inode, attr->ia_valid);
                spin_unlock(&inode->i_lock);
                attr->ia_valid &= ~(ATTR_MTIME | ATTR_ATIME);
        } else if (nfs_have_delegated_atime(inode) &&
                   attr->ia_valid & ATTR_ATIME &&
                   !(attr->ia_valid & ATTR_MTIME)) {
                nfs_update_delegated_atime(inode);
                attr->ia_valid &= ~ATTR_ATIME;
        }

        /* Optimization: if the end result is no change, don't RPC */
        if (((attr->ia_valid & NFS_VALID_ATTRS) & ~(ATTR_FILE|ATTR_OPEN)) == 0)
                return 0;

        trace_nfs_setattr_enter(inode);

        /* Write all dirty data */
        if (S_ISREG(inode->i_mode))
                nfs_sync_inode(inode);

        fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode));
        if (fattr == NULL) {
                error = -ENOMEM;
                goto out;
        }

        error = NFS_PROTO(inode)->setattr(dentry, fattr, attr);
        if (error == 0)
                error = nfs_refresh_inode(inode, fattr);
        nfs_free_fattr(fattr);
out:
        trace_nfs_setattr_exit(inode, error);
        return error;
}
EXPORT_SYMBOL_GPL(nfs_setattr);

/**
 * nfs_vmtruncate - unmap mappings "freed" by truncate() syscall
 * @inode: inode of the file used
 * @offset: file offset to start truncating
 *
 * This is a copy of the common vmtruncate, but with the locking
 * corrected to take into account the fact that NFS requires
 * inode->i_size to be updated under the inode->i_lock.
 * Note: must be called with inode->i_lock held!
 */
static int nfs_vmtruncate(struct inode * inode, loff_t offset)
{
        int err;

        err = inode_newsize_ok(inode, offset);
        if (err)
                goto out;

        trace_nfs_size_truncate(inode, offset);
        i_size_write(inode, offset);
        /* Optimisation */
        if (offset == 0) {
                NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_DATA;
                nfs_ooo_clear(NFS_I(inode));
        }
        NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;

        spin_unlock(&inode->i_lock);
        truncate_pagecache(inode, offset);
        nfs_update_delegated_mtime_locked(inode);
        spin_lock(&inode->i_lock);
out:
        return err;
}

/**
 * nfs_setattr_update_inode - Update inode metadata after a setattr call.
 * @inode: pointer to struct inode
 * @attr: pointer to struct iattr
 * @fattr: pointer to struct nfs_fattr
 *
 * Note: we do this in the *proc.c in order to ensure that
 *       it works for things like exclusive creates too.
 */
void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr,
                struct nfs_fattr *fattr)
{
        /* Barrier: bump the attribute generation count. */
        nfs_fattr_set_barrier(fattr);

        spin_lock(&inode->i_lock);
        NFS_I(inode)->attr_gencount = fattr->gencount;
        if ((attr->ia_valid & ATTR_SIZE) != 0) {
                if (!nfs_have_delegated_mtime(inode))
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);
                nfs_inc_stats(inode, NFSIOS_SETATTRTRUNC);
                nfs_vmtruncate(inode, attr->ia_size);
        }
        if ((attr->ia_valid & (ATTR_MODE|ATTR_UID|ATTR_GID)) != 0) {
                NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_CTIME;
                if ((attr->ia_valid & ATTR_KILL_SUID) != 0 &&
                    inode->i_mode & S_ISUID)
                        inode->i_mode &= ~S_ISUID;
                if (setattr_should_drop_sgid(&nop_mnt_idmap, inode))
                        inode->i_mode &= ~S_ISGID;
                if ((attr->ia_valid & ATTR_MODE) != 0) {
                        int mode = attr->ia_mode & S_IALLUGO;
                        mode |= inode->i_mode & ~S_IALLUGO;
                        inode->i_mode = mode;
                }
                if ((attr->ia_valid & ATTR_UID) != 0)
                        inode->i_uid = attr->ia_uid;
                if ((attr->ia_valid & ATTR_GID) != 0)
                        inode->i_gid = attr->ia_gid;
                if (fattr->valid & NFS_ATTR_FATTR_CTIME)
                        inode_set_ctime_to_ts(inode, fattr->ctime);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
                                        | NFS_INO_INVALID_CTIME);
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_ACCESS
                                | NFS_INO_INVALID_ACL);
        }
        if (attr->ia_valid & (ATTR_ATIME_SET|ATTR_ATIME)) {
                NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_ATIME
                                | NFS_INO_INVALID_CTIME);
                if (fattr->valid & NFS_ATTR_FATTR_ATIME)
                        inode_set_atime_to_ts(inode, fattr->atime);
                else if (attr->ia_valid & ATTR_ATIME_SET)
                        inode_set_atime_to_ts(inode, attr->ia_atime);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME);

                if (fattr->valid & NFS_ATTR_FATTR_CTIME)
                        inode_set_ctime_to_ts(inode, fattr->ctime);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
                                        | NFS_INO_INVALID_CTIME);
        }
        if (attr->ia_valid & (ATTR_MTIME_SET|ATTR_MTIME)) {
                NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_MTIME
                                | NFS_INO_INVALID_CTIME);
                if (fattr->valid & NFS_ATTR_FATTR_MTIME)
                        inode_set_mtime_to_ts(inode, fattr->mtime);
                else if (attr->ia_valid & ATTR_MTIME_SET)
                        inode_set_mtime_to_ts(inode, attr->ia_mtime);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);

                if (fattr->valid & NFS_ATTR_FATTR_CTIME)
                        inode_set_ctime_to_ts(inode, fattr->ctime);
                else
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE
                                        | NFS_INO_INVALID_CTIME);
        }
        if (fattr->valid)
                nfs_update_inode(inode, fattr);
        spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_setattr_update_inode);

/*
 * Don't request help from readdirplus if the file is being written to,
 * or if attribute caching is turned off
 */
static bool nfs_getattr_readdirplus_enable(const struct inode *inode)
{
        return nfs_server_capable(inode, NFS_CAP_READDIRPLUS) &&
               !nfs_have_writebacks(inode) && NFS_MAXATTRTIMEO(inode) > 5 * HZ;
}

static void nfs_readdirplus_parent_cache_miss(struct dentry *dentry)
{
        if (!IS_ROOT(dentry)) {
                struct dentry *parent = dget_parent(dentry);
                nfs_readdir_record_entry_cache_miss(d_inode(parent));
                dput(parent);
        }
}

static void nfs_readdirplus_parent_cache_hit(struct dentry *dentry)
{
        if (!IS_ROOT(dentry)) {
                struct dentry *parent = dget_parent(dentry);
                nfs_readdir_record_entry_cache_hit(d_inode(parent));
                dput(parent);
        }
}

static u32 nfs_get_valid_attrmask(struct inode *inode)
{
        unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity);
        u32 reply_mask = STATX_INO | STATX_TYPE;

        if (!(cache_validity & NFS_INO_INVALID_ATIME))
                reply_mask |= STATX_ATIME;
        if (!(cache_validity & NFS_INO_INVALID_CTIME))
                reply_mask |= STATX_CTIME;
        if (!(cache_validity & NFS_INO_INVALID_MTIME))
                reply_mask |= STATX_MTIME;
        if (!(cache_validity & NFS_INO_INVALID_SIZE))
                reply_mask |= STATX_SIZE;
        if (!(cache_validity & NFS_INO_INVALID_NLINK))
                reply_mask |= STATX_NLINK;
        if (!(cache_validity & NFS_INO_INVALID_MODE))
                reply_mask |= STATX_MODE;
        if (!(cache_validity & NFS_INO_INVALID_OTHER))
                reply_mask |= STATX_UID | STATX_GID;
        if (!(cache_validity & NFS_INO_INVALID_BLOCKS))
                reply_mask |= STATX_BLOCKS;
        if (!(cache_validity & NFS_INO_INVALID_CHANGE))
                reply_mask |= STATX_CHANGE_COOKIE;
        return reply_mask;
}

int nfs_getattr(struct mnt_idmap *idmap, const struct path *path,
                struct kstat *stat, u32 request_mask, unsigned int query_flags)
{
        struct inode *inode = d_inode(path->dentry);
        struct nfs_server *server = NFS_SERVER(inode);
        unsigned long cache_validity;
        int err = 0;
        bool force_sync = query_flags & AT_STATX_FORCE_SYNC;
        bool do_update = false;
        bool readdirplus_enabled = nfs_getattr_readdirplus_enable(inode);

        trace_nfs_getattr_enter(inode);

        request_mask &= STATX_TYPE | STATX_MODE | STATX_NLINK | STATX_UID |
                        STATX_GID | STATX_ATIME | STATX_MTIME | STATX_CTIME |
                        STATX_INO | STATX_SIZE | STATX_BLOCKS |
                        STATX_CHANGE_COOKIE;

        if ((query_flags & AT_STATX_DONT_SYNC) && !force_sync) {
                if (readdirplus_enabled)
                        nfs_readdirplus_parent_cache_hit(path->dentry);
                goto out_no_revalidate;
        }

        /* Flush out writes to the server in order to update c/mtime/version.  */
        if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_CHANGE_COOKIE)) &&
            S_ISREG(inode->i_mode)) {
                if (nfs_have_delegated_mtime(inode))
                        filemap_fdatawrite(inode->i_mapping);
                else
                        filemap_write_and_wait(inode->i_mapping);
        }

        /*
         * We may force a getattr if the user cares about atime.
         *
         * Note that we only have to check the vfsmount flags here:
         *  - NFS always sets S_NOATIME by so checking it would give a
         *    bogus result
         *  - NFS never sets SB_NOATIME or SB_NODIRATIME so there is
         *    no point in checking those.
         */
        if ((path->mnt->mnt_flags & MNT_NOATIME) ||
            ((path->mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)))
                request_mask &= ~STATX_ATIME;

        /* Is the user requesting attributes that might need revalidation? */
        if (!(request_mask & (STATX_MODE|STATX_NLINK|STATX_ATIME|STATX_CTIME|
                                        STATX_MTIME|STATX_UID|STATX_GID|
                                        STATX_SIZE|STATX_BLOCKS|
                                        STATX_CHANGE_COOKIE)))
                goto out_no_revalidate;

        /* Check whether the cached attributes are stale */
        do_update |= force_sync || nfs_attribute_cache_expired(inode);
        cache_validity = READ_ONCE(NFS_I(inode)->cache_validity);
        do_update |= cache_validity & NFS_INO_INVALID_CHANGE;
        if (request_mask & STATX_ATIME)
                do_update |= cache_validity & NFS_INO_INVALID_ATIME;
        if (request_mask & STATX_CTIME)
                do_update |= cache_validity & NFS_INO_INVALID_CTIME;
        if (request_mask & STATX_MTIME)
                do_update |= cache_validity & NFS_INO_INVALID_MTIME;
        if (request_mask & STATX_SIZE)
                do_update |= cache_validity & NFS_INO_INVALID_SIZE;
        if (request_mask & STATX_NLINK)
                do_update |= cache_validity & NFS_INO_INVALID_NLINK;
        if (request_mask & STATX_MODE)
                do_update |= cache_validity & NFS_INO_INVALID_MODE;
        if (request_mask & (STATX_UID | STATX_GID))
                do_update |= cache_validity & NFS_INO_INVALID_OTHER;
        if (request_mask & STATX_BLOCKS)
                do_update |= cache_validity & NFS_INO_INVALID_BLOCKS;

        if (do_update) {
                if (readdirplus_enabled)
                        nfs_readdirplus_parent_cache_miss(path->dentry);
                err = __nfs_revalidate_inode(server, inode);
                if (err)
                        goto out;
        } else if (readdirplus_enabled)
                nfs_readdirplus_parent_cache_hit(path->dentry);
out_no_revalidate:
        /* Only return attributes that were revalidated. */
        stat->result_mask = nfs_get_valid_attrmask(inode) | request_mask;

        generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat);
        stat->ino = nfs_compat_user_ino64(NFS_FILEID(inode));
        stat->change_cookie = inode_peek_iversion_raw(inode);
        stat->attributes_mask |= STATX_ATTR_CHANGE_MONOTONIC;
        if (server->change_attr_type != NFS4_CHANGE_TYPE_IS_UNDEFINED)
                stat->attributes |= STATX_ATTR_CHANGE_MONOTONIC;
        if (S_ISDIR(inode->i_mode))
                stat->blksize = NFS_SERVER(inode)->dtsize;
out:
        trace_nfs_getattr_exit(inode, err);
        return err;
}
EXPORT_SYMBOL_GPL(nfs_getattr);

static void nfs_init_lock_context(struct nfs_lock_context *l_ctx)
{
        refcount_set(&l_ctx->count, 1);
        l_ctx->lockowner = current->files;
        INIT_LIST_HEAD(&l_ctx->list);
        atomic_set(&l_ctx->io_count, 0);
}

static struct nfs_lock_context *__nfs_find_lock_context(struct nfs_open_context *ctx)
{
        struct nfs_lock_context *pos;

        list_for_each_entry_rcu(pos, &ctx->lock_context.list, list) {
                if (pos->lockowner != current->files)
                        continue;
                if (refcount_inc_not_zero(&pos->count))
                        return pos;
        }
        return NULL;
}

struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx)
{
        struct nfs_lock_context *res, *new = NULL;
        struct inode *inode = d_inode(ctx->dentry);

        rcu_read_lock();
        res = __nfs_find_lock_context(ctx);
        rcu_read_unlock();
        if (res == NULL) {
                new = kmalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
                if (new == NULL)
                        return ERR_PTR(-ENOMEM);
                nfs_init_lock_context(new);
                spin_lock(&inode->i_lock);
                res = __nfs_find_lock_context(ctx);
                if (res == NULL) {
                        new->open_context = get_nfs_open_context(ctx);
                        if (new->open_context) {
                                list_add_tail_rcu(&new->list,
                                                &ctx->lock_context.list);
                                res = new;
                                new = NULL;
                        } else
                                res = ERR_PTR(-EBADF);
                }
                spin_unlock(&inode->i_lock);
                kfree(new);
        }
        return res;
}
EXPORT_SYMBOL_GPL(nfs_get_lock_context);

void nfs_put_lock_context(struct nfs_lock_context *l_ctx)
{
        struct nfs_open_context *ctx = l_ctx->open_context;
        struct inode *inode = d_inode(ctx->dentry);

        if (!refcount_dec_and_lock(&l_ctx->count, &inode->i_lock))
                return;
        list_del_rcu(&l_ctx->list);
        spin_unlock(&inode->i_lock);
        put_nfs_open_context(ctx);
        kfree_rcu(l_ctx, rcu_head);
}
EXPORT_SYMBOL_GPL(nfs_put_lock_context);

/**
 * nfs_close_context - Common close_context() routine NFSv2/v3
 * @ctx: pointer to context
 * @is_sync: is this a synchronous close
 *
 * Ensure that the attributes are up to date if we're mounted
 * with close-to-open semantics and we have cached data that will
 * need to be revalidated on open.
 */
void nfs_close_context(struct nfs_open_context *ctx, int is_sync)
{
        struct nfs_inode *nfsi;
        struct inode *inode;

        if (!(ctx->mode & FMODE_WRITE))
                return;
        if (!is_sync)
                return;
        inode = d_inode(ctx->dentry);
        if (nfs_have_read_or_write_delegation(inode))
                return;
        nfsi = NFS_I(inode);
        if (inode->i_mapping->nrpages == 0)
                return;
        if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
                return;
        if (!list_empty(&nfsi->open_files))
                return;
        if (NFS_SERVER(inode)->flags & NFS_MOUNT_NOCTO)
                return;
        nfs_revalidate_inode(inode,
                             NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE);
}
EXPORT_SYMBOL_GPL(nfs_close_context);

struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry,
                                                fmode_t f_mode,
                                                struct file *filp)
{
        struct nfs_open_context *ctx;

        ctx = kmalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT);
        if (!ctx)
                return ERR_PTR(-ENOMEM);
        nfs_sb_active(dentry->d_sb);
        ctx->dentry = dget(dentry);
        if (filp)
                ctx->cred = get_cred(filp->f_cred);
        else
                ctx->cred = get_current_cred();
        rcu_assign_pointer(ctx->ll_cred, NULL);
        ctx->state = NULL;
        ctx->mode = f_mode;
        ctx->flags = 0;
        ctx->error = 0;
        ctx->flock_owner = (fl_owner_t)filp;
        nfs_init_lock_context(&ctx->lock_context);
        ctx->lock_context.open_context = ctx;
        INIT_LIST_HEAD(&ctx->list);
        ctx->mdsthreshold = NULL;
        return ctx;
}
EXPORT_SYMBOL_GPL(alloc_nfs_open_context);

struct nfs_open_context *get_nfs_open_context(struct nfs_open_context *ctx)
{
        if (ctx != NULL && refcount_inc_not_zero(&ctx->lock_context.count))
                return ctx;
        return NULL;
}
EXPORT_SYMBOL_GPL(get_nfs_open_context);

static void __put_nfs_open_context(struct nfs_open_context *ctx, int is_sync)
{
        struct inode *inode = d_inode(ctx->dentry);
        struct super_block *sb = ctx->dentry->d_sb;

        if (!refcount_dec_and_test(&ctx->lock_context.count))
                return;
        if (!list_empty(&ctx->list)) {
                spin_lock(&inode->i_lock);
                list_del_rcu(&ctx->list);
                spin_unlock(&inode->i_lock);
        }
        if (inode != NULL)
                NFS_PROTO(inode)->close_context(ctx, is_sync);
        put_cred(ctx->cred);
        dput(ctx->dentry);
        nfs_sb_deactive(sb);
        put_rpccred(rcu_dereference_protected(ctx->ll_cred, 1));
        kfree(ctx->mdsthreshold);
        kfree_rcu(ctx, rcu_head);
}

void put_nfs_open_context(struct nfs_open_context *ctx)
{
        __put_nfs_open_context(ctx, 0);
}
EXPORT_SYMBOL_GPL(put_nfs_open_context);

static void put_nfs_open_context_sync(struct nfs_open_context *ctx)
{
        __put_nfs_open_context(ctx, 1);
}

/*
 * Ensure that mmap has a recent RPC credential for use when writing out
 * shared pages
 */
void nfs_inode_attach_open_context(struct nfs_open_context *ctx)
{
        struct inode *inode = d_inode(ctx->dentry);
        struct nfs_inode *nfsi = NFS_I(inode);

        spin_lock(&inode->i_lock);
        if (list_empty(&nfsi->open_files) &&
            nfs_ooo_test(nfsi))
                nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA |
                                                     NFS_INO_REVAL_FORCED);
        list_add_tail_rcu(&ctx->list, &nfsi->open_files);
        spin_unlock(&inode->i_lock);
}
EXPORT_SYMBOL_GPL(nfs_inode_attach_open_context);

void nfs_file_set_open_context(struct file *filp, struct nfs_open_context *ctx)
{
        filp->private_data = get_nfs_open_context(ctx);
        set_bit(NFS_CONTEXT_FILE_OPEN, &ctx->flags);
        if (list_empty(&ctx->list))
                nfs_inode_attach_open_context(ctx);
}
EXPORT_SYMBOL_GPL(nfs_file_set_open_context);

/*
 * Given an inode, search for an open context with the desired characteristics
 */
struct nfs_open_context *nfs_find_open_context(struct inode *inode, const struct cred *cred, fmode_t mode)
{
        struct nfs_inode *nfsi = NFS_I(inode);
        struct nfs_open_context *pos, *ctx = NULL;

        rcu_read_lock();
        list_for_each_entry_rcu(pos, &nfsi->open_files, list) {
                if (cred != NULL && cred_fscmp(pos->cred, cred) != 0)
                        continue;
                if ((pos->mode & (FMODE_READ|FMODE_WRITE)) != mode)
                        continue;
                if (!test_bit(NFS_CONTEXT_FILE_OPEN, &pos->flags))
                        continue;
                ctx = get_nfs_open_context(pos);
                if (ctx)
                        break;
        }
        rcu_read_unlock();
        return ctx;
}

void nfs_file_clear_open_context(struct file *filp)
{
        struct nfs_open_context *ctx = nfs_file_open_context(filp);

        if (ctx) {
                struct inode *inode = d_inode(ctx->dentry);

                clear_bit(NFS_CONTEXT_FILE_OPEN, &ctx->flags);
                /*
                 * We fatal error on write before. Try to writeback
                 * every page again.
                 */
                if (ctx->error < 0)
                        invalidate_inode_pages2(inode->i_mapping);
                filp->private_data = NULL;
                put_nfs_open_context_sync(ctx);
        }
}

/*
 * These allocate and release file read/write context information.
 */
int nfs_open(struct inode *inode, struct file *filp)
{
        struct nfs_open_context *ctx;

        ctx = alloc_nfs_open_context(file_dentry(filp),
                                     flags_to_mode(filp->f_flags), filp);
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);
        nfs_file_set_open_context(filp, ctx);
        put_nfs_open_context(ctx);
        nfs_fscache_open_file(inode, filp);
        return 0;
}

/*
 * This function is called whenever some part of NFS notices that
 * the cached attributes have to be refreshed.
 */
int
__nfs_revalidate_inode(struct nfs_server *server, struct inode *inode)
{
        int                 status = -ESTALE;
        struct nfs_fattr *fattr = NULL;
        struct nfs_inode *nfsi = NFS_I(inode);

        dfprintk(PAGECACHE, "NFS: revalidating (%s/%Lu)\n",
                inode->i_sb->s_id, (unsigned long long)NFS_FILEID(inode));

        trace_nfs_revalidate_inode_enter(inode);

        if (is_bad_inode(inode))
                goto out;
        if (NFS_STALE(inode))
                goto out;

        /* pNFS: Attributes aren't updated until we layoutcommit */
        if (S_ISREG(inode->i_mode)) {
                status = pnfs_sync_inode(inode, false);
                if (status)
                        goto out;
        }

        status = -ENOMEM;
        fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode));
        if (fattr == NULL)
                goto out;

        nfs_inc_stats(inode, NFSIOS_INODEREVALIDATE);

        status = NFS_PROTO(inode)->getattr(server, NFS_FH(inode), fattr, inode);
        if (status != 0) {
                dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) getattr failed, error=%d\n",
                         inode->i_sb->s_id,
                         (unsigned long long)NFS_FILEID(inode), status);
                switch (status) {
                case -ETIMEDOUT:
                        /* A soft timeout occurred. Use cached information? */
                        if (server->flags & NFS_MOUNT_SOFTREVAL)
                                status = 0;
                        break;
                case -ESTALE:
                        if (!S_ISDIR(inode->i_mode))
                                nfs_set_inode_stale(inode);
                        else
                                nfs_zap_caches(inode);
                }
                goto out;
        }

        status = nfs_refresh_inode(inode, fattr);
        if (status) {
                dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) refresh failed, error=%d\n",
                         inode->i_sb->s_id,
                         (unsigned long long)NFS_FILEID(inode), status);
                goto out;
        }

        if (nfsi->cache_validity & NFS_INO_INVALID_ACL)
                nfs_zap_acl_cache(inode);

        nfs_setsecurity(inode, fattr);

        dfprintk(PAGECACHE, "NFS: (%s/%Lu) revalidation complete\n",
                inode->i_sb->s_id,
                (unsigned long long)NFS_FILEID(inode));

out:
        nfs_free_fattr(fattr);
        trace_nfs_revalidate_inode_exit(inode, status);
        return status;
}

int nfs_attribute_cache_expired(struct inode *inode)
{
        if (nfs_have_delegated_attributes(inode))
                return 0;
        return nfs_attribute_timeout(inode);
}

/**
 * nfs_revalidate_inode - Revalidate the inode attributes
 * @inode: pointer to inode struct
 * @flags: cache flags to check
 *
 * Updates inode attribute information by retrieving the data from the server.
 */
int nfs_revalidate_inode(struct inode *inode, unsigned long flags)
{
        if (!nfs_check_cache_invalid(inode, flags))
                return NFS_STALE(inode) ? -ESTALE : 0;
        return __nfs_revalidate_inode(NFS_SERVER(inode), inode);
}
EXPORT_SYMBOL_GPL(nfs_revalidate_inode);

static int nfs_invalidate_mapping(struct inode *inode, struct address_space *mapping)
{
        int ret;

        nfs_fscache_invalidate(inode, 0);
        if (mapping->nrpages != 0) {
                if (S_ISREG(inode->i_mode)) {
                        ret = nfs_sync_mapping(mapping);
                        if (ret < 0)
                                return ret;
                }
                ret = invalidate_inode_pages2(mapping);
                if (ret < 0)
                        return ret;
        }
        nfs_inc_stats(inode, NFSIOS_DATAINVALIDATE);

        dfprintk(PAGECACHE, "NFS: (%s/%Lu) data cache invalidated\n",
                        inode->i_sb->s_id,
                        (unsigned long long)NFS_FILEID(inode));
        return 0;
}

/**
 * nfs_clear_invalid_mapping - Conditionally clear a mapping
 * @mapping: pointer to mapping
 *
 * If the NFS_INO_INVALID_DATA inode flag is set, clear the mapping.
 */
int nfs_clear_invalid_mapping(struct address_space *mapping)
{
        struct inode *inode = mapping->host;
        struct nfs_inode *nfsi = NFS_I(inode);
        unsigned long *bitlock = &nfsi->flags;
        int ret = 0;

        /*
         * We must clear NFS_INO_INVALID_DATA first to ensure that
         * invalidations that come in while we're shooting down the mappings
         * are respected. But, that leaves a race window where one revalidator
         * can clear the flag, and then another checks it before the mapping
         * gets invalidated. Fix that by serializing access to this part of
         * the function.
         *
         * At the same time, we need to allow other tasks to see whether we
         * might be in the middle of invalidating the pages, so we only set
         * the bit lock here if it looks like we're going to be doing that.
         */
        for (;;) {
                ret = wait_on_bit_action(bitlock, NFS_INO_INVALIDATING,
                                         nfs_wait_bit_killable,
                                         TASK_KILLABLE|TASK_FREEZABLE_UNSAFE);
                if (ret)
                        goto out;
                smp_rmb(); /* pairs with smp_wmb() below */
                if (test_bit(NFS_INO_INVALIDATING, bitlock))
                        continue;
                /* pairs with nfs_set_cache_invalid()'s smp_store_release() */
                if (!(smp_load_acquire(&nfsi->cache_validity) & NFS_INO_INVALID_DATA))
                        goto out;
                /* Slow-path that double-checks with spinlock held */
                spin_lock(&inode->i_lock);
                if (test_bit(NFS_INO_INVALIDATING, bitlock)) {
                        spin_unlock(&inode->i_lock);
                        continue;
                }
                if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
                        break;
                spin_unlock(&inode->i_lock);
                goto out;
        }

        set_bit(NFS_INO_INVALIDATING, bitlock);
        smp_wmb();
        nfsi->cache_validity &= ~NFS_INO_INVALID_DATA;
        nfs_ooo_clear(nfsi);
        spin_unlock(&inode->i_lock);
        trace_nfs_invalidate_mapping_enter(inode);
        ret = nfs_invalidate_mapping(inode, mapping);
        trace_nfs_invalidate_mapping_exit(inode, ret);

        clear_bit_unlock(NFS_INO_INVALIDATING, bitlock);
        smp_mb__after_atomic();
        wake_up_bit(bitlock, NFS_INO_INVALIDATING);
out:
        return ret;
}

bool nfs_mapping_need_revalidate_inode(struct inode *inode)
{
        return nfs_check_cache_invalid(inode, NFS_INO_INVALID_CHANGE) ||
                NFS_STALE(inode);
}

int nfs_revalidate_mapping_rcu(struct inode *inode)
{
        struct nfs_inode *nfsi = NFS_I(inode);
        unsigned long *bitlock = &nfsi->flags;
        int ret = 0;

        if (IS_SWAPFILE(inode))
                goto out;
        if (nfs_mapping_need_revalidate_inode(inode)) {
                ret = -ECHILD;
                goto out;
        }
        spin_lock(&inode->i_lock);
        if (test_bit(NFS_INO_INVALIDATING, bitlock) ||
            (nfsi->cache_validity & NFS_INO_INVALID_DATA))
                ret = -ECHILD;
        spin_unlock(&inode->i_lock);
out:
        return ret;
}

/**
 * nfs_revalidate_mapping - Revalidate the pagecache
 * @inode: pointer to host inode
 * @mapping: pointer to mapping
 */
int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping)
{
        /* swapfiles are not supposed to be shared. */
        if (IS_SWAPFILE(inode))
                return 0;

        if (nfs_mapping_need_revalidate_inode(inode)) {
                int ret = __nfs_revalidate_inode(NFS_SERVER(inode), inode);
                if (ret < 0)
                        return ret;
        }

        return nfs_clear_invalid_mapping(mapping);
}

static bool nfs_file_has_writers(struct nfs_inode *nfsi)
{
        struct inode *inode = &nfsi->vfs_inode;

        if (!S_ISREG(inode->i_mode))
                return false;
        if (list_empty(&nfsi->open_files))
                return false;
        return inode_is_open_for_write(inode);
}

static bool nfs_file_has_buffered_writers(struct nfs_inode *nfsi)
{
        return nfs_file_has_writers(nfsi) && nfs_file_io_is_buffered(nfsi);
}

static void nfs_wcc_update_inode(struct inode *inode, struct nfs_fattr *fattr)
{
        struct timespec64 ts;

        if ((fattr->valid & NFS_ATTR_FATTR_PRECHANGE)
                        && (fattr->valid & NFS_ATTR_FATTR_CHANGE)
                        && inode_eq_iversion_raw(inode, fattr->pre_change_attr)) {
                inode_set_iversion_raw(inode, fattr->change_attr);
                if (S_ISDIR(inode->i_mode))
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA);
                else if (nfs_server_capable(inode, NFS_CAP_XATTR))
                        nfs_set_cache_invalid(inode, NFS_INO_INVALID_XATTR);
        }
        /* If we have atomic WCC data, we may update some attributes */
        ts = inode_get_ctime(inode);
        if ((fattr->valid & NFS_ATTR_FATTR_PRECTIME)
                        && (fattr->valid & NFS_ATTR_FATTR_CTIME)
                        && timespec64_equal(&ts, &fattr->pre_ctime)) {
                inode_set_ctime_to_ts(inode, fattr->ctime);
        }

        ts = inode_get_mtime(inode);
        if ((fattr->valid & NFS_ATTR_FATTR_PREMTIME)
                        && (fattr->valid & NFS_ATTR_FATTR_MTIME)
                        && timespec64_equal(&ts, &fattr->pre_mtime)) {
                inode_set_mtime_to_ts(inode, fattr->mtime);
        }
        if ((fattr->valid & NFS_ATTR_FATTR_PRESIZE)
                        && (fattr->valid & NFS_ATTR_FATTR_SIZE)
                        && i_size_read(inode) == nfs_size_to_loff_t(fattr->pre_size)
                        && !nfs_have_writebacks(inode)) {
                trace_nfs_size_wcc(inode, fattr->size);
                i_size_write(inode, nfs_size_to_loff_t(fattr->size));
        }
}

/**
 * nfs_check_inode_attributes - verify consistency of the inode attribute cache
 * @inode: pointer to inode
 * @fattr: updated attributes
 *
 * Verifies the attribute cache. If we have just changed the attributes,
 * so that fattr carries weak cache consistency data, then it may
 * also update the ctime/mtime/change_attribute.
 */
static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fattr)
{
        struct nfs_inode *nfsi = NFS_I(inode);
        loff_t cur_size, new_isize;
        unsigned long invalid = 0;
        struct timespec64 ts;

        if (nfs_have_delegated_attributes(inode))
                return 0;

        if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) {
                /* Only a mounted-on-fileid? Just exit */
                if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID)
                        return 0;
        /* Has the inode gone and changed behind our back? */
        } else if (nfsi->fileid != fattr->fileid) {
                /* Is this perhaps the mounted-on fileid? */
                if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) &&
                    nfsi->fileid == fattr->mounted_on_fileid)
                        return 0;
                return -ESTALE;
        }
        if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode))
                return -ESTALE;


        if (!nfs_file_has_buffered_writers(nfsi)) {
                /* Verify a few of the more important attributes */
                if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 && !inode_eq_iversion_raw(inode, fattr->change_attr))
                        invalid |= NFS_INO_INVALID_CHANGE;

                ts = inode_get_mtime(inode);
                if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec64_equal(&ts, &fattr->mtime))
                        invalid |= NFS_INO_INVALID_MTIME;

                ts = inode_get_ctime(inode);
                if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec64_equal(&ts, &fattr->ctime))
                        invalid |= NFS_INO_INVALID_CTIME;

                if (fattr->valid & NFS_ATTR_FATTR_SIZE) {
                        cur_size = i_size_read(inode);
                        new_isize = nfs_size_to_loff_t(fattr->size);
                        if (cur_size != new_isize)
                                invalid |= NFS_INO_INVALID_SIZE;
                }
        }

        /* Have any file permissions changed? */
        if ((fattr->valid & NFS_ATTR_FATTR_MODE) && (inode->i_mode & S_IALLUGO) != (fattr->mode & S_IALLUGO))
                invalid |= NFS_INO_INVALID_MODE;
        if ((fattr->valid & NFS_ATTR_FATTR_OWNER) && !uid_eq(inode->i_uid, fattr->uid))
                invalid |= NFS_INO_INVALID_OTHER;
        if ((fattr->valid & NFS_ATTR_FATTR_GROUP) && !gid_eq(inode->i_gid, fattr->gid))
                invalid |= NFS_INO_INVALID_OTHER;

        /* Has the link count changed? */
        if ((fattr->valid & NFS_ATTR_FATTR_NLINK) && inode->i_nlink != fattr->nlink)
                invalid |= NFS_INO_INVALID_NLINK;

        ts = inode_get_atime(inode);
        if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec64_equal(&ts, &fattr->atime))
                invalid |= NFS_INO_INVALID_ATIME;

        if (invalid != 0)
                nfs_set_cache_invalid(inode, invalid);

        nfsi->read_cache_jiffies = fattr->time_start;
        return 0;
}

static atomic_long_t nfs_attr_generation_counter;

static unsigned long nfs_read_attr_generation_counter(void)
{
        return atomic_long_read(&nfs_attr_generation_counter);
}

unsigned long nfs_inc_attr_generation_counter(void)
{
        return atomic_long_inc_return(&nfs_attr_generation_counter);
}
EXPORT_SYMBOL_GPL(nfs_inc_attr_generation_counter);

void nfs_fattr_init(struct nfs_fattr *fattr)
{
        fattr->valid = 0;
        fattr->time_start = jiffies;
        fattr->gencount = nfs_inc_attr_generation_counter();
        fattr->owner_name = NULL;
        fattr->group_name = NULL;
        fattr->mdsthreshold = NULL;
}
EXPORT_SYMBOL_GPL(nfs_fattr_init);

/**
 * nfs_fattr_set_barrier
 * @fattr: attributes
 *
 * Used to set a barrier after an attribute was updated. This
 * barrier ensures that older attributes from RPC calls that may
 * have raced with our update cannot clobber these new values.
 * Note that you are still responsible for ensuring that other
 * operations which change the attribute on the server do not
 * collide.
 */
void nfs_fattr_set_barrier(struct nfs_fattr *fattr)
{
        fattr->gencount = nfs_inc_attr_generation_counter();
}

struct nfs_fattr *nfs_alloc_fattr(void)
{
        struct nfs_fattr *fattr;

        fattr = kmalloc(sizeof(*fattr), GFP_KERNEL);
        if (fattr != NULL) {
                nfs_fattr_init(fattr);
                fattr->label = NULL;
        }
        return fattr;
}
EXPORT_SYMBOL_GPL(nfs_alloc_fattr);

struct nfs_fattr *nfs_alloc_fattr_with_label(struct nfs_server *server)
{
        struct nfs_fattr *fattr = nfs_alloc_fattr();

        if (!fattr)
                return NULL;

        fattr->label = nfs4_label_alloc(server, GFP_KERNEL);
        if (IS_ERR(fattr->label)) {
                kfree(fattr);
                return NULL;
        }

        return fattr;
}
EXPORT_SYMBOL_GPL(nfs_alloc_fattr_with_label);

struct nfs_fh *nfs_alloc_fhandle(void)
{
        struct nfs_fh *fh;

        fh = kmalloc(sizeof(struct nfs_fh), GFP_KERNEL);
        if (fh != NULL)
                fh->size = 0;
        return fh;
}
EXPORT_SYMBOL_GPL(nfs_alloc_fhandle);

#ifdef NFS_DEBUG
/*
 * _nfs_display_fhandle_hash - calculate the crc32 hash for the filehandle
 *                             in the same way that wireshark does
 *
 * @fh: file handle
 *
 * For debugging only.
 */
u32 _nfs_display_fhandle_hash(const struct nfs_fh *fh)
{
        /* wireshark uses 32-bit AUTODIN crc and does a bitwise
         * not on the result */
        return nfs_fhandle_hash(fh);
}
EXPORT_SYMBOL_GPL(_nfs_display_fhandle_hash);

/*
 * _nfs_display_fhandle - display an NFS file handle on the console
 *
 * @fh: file handle to display
 * @caption: display caption
 *
 * For debugging only.
 */
void _nfs_display_fhandle(const struct nfs_fh *fh, const char *caption)
{
        unsigned short i;

        if (fh == NULL || fh->size == 0) {
                printk(KERN_DEFAULT "%s at %p is empty\n", caption, fh);
                return;
        }

        printk(KERN_DEFAULT "%s at %p is %u bytes, crc: 0x%08x:\n",
               caption, fh, fh->size, _nfs_display_fhandle_hash(fh));
        for (i = 0; i < fh->size; i += 16) {
                __be32 *pos = (__be32 *)&fh->data[i];

                switch ((fh->size - i - 1) >> 2) {
                case 0:
                        printk(KERN_DEFAULT " %08x\n",
                                be32_to_cpup(pos));
                        break;
                case 1:
                        printk(KERN_DEFAULT " %08x %08x\n",
                                be32_to_cpup(pos), be32_to_cpup(pos + 1));
                        break;
                case 2:
                        printk(KERN_DEFAULT " %08x %08x %08x\n",
                                be32_to_cpup(pos), be32_to_cpup(pos + 1),
                                be32_to_cpup(pos + 2));
                        break;
                default:
                        printk(KERN_DEFAULT " %08x %08x %08x %08x\n",
                                be32_to_cpup(pos), be32_to_cpup(pos + 1),
                                be32_to_cpup(pos + 2), be32_to_cpup(pos + 3));
                }
        }
}
EXPORT_SYMBOL_GPL(_nfs_display_fhandle);
#endif

/**
 * nfs_inode_attrs_cmp_generic - compare attributes
 * @fattr: attributes
 * @inode: pointer to inode
 *
 * Attempt to divine whether or not an RPC call reply carrying stale
 * attributes got scheduled after another call carrying updated ones.
 * Note also the check for wraparound of 'attr_gencount'
 *
 * The function returns '1' if it thinks the attributes in @fattr are
 * more recent than the ones cached in @inode. Otherwise it returns
 * the value '0'.
 */
static int nfs_inode_attrs_cmp_generic(const struct nfs_fattr *fattr,
                                       const struct inode *inode)
{
        unsigned long attr_gencount = NFS_I(inode)->attr_gencount;

        return (long)(fattr->gencount - attr_gencount) > 0 ||
               (long)(attr_gencount - nfs_read_attr_generation_counter()) > 0;
}

/**
 * nfs_inode_attrs_cmp_monotonic - compare attributes
 * @fattr: attributes
 * @inode: pointer to inode
 *
 * Attempt to divine whether or not an RPC call reply carrying stale
 * attributes got scheduled after another call carrying updated ones.
 *
 * We assume that the server observes monotonic semantics for
 * the change attribute, so a larger value means that the attributes in
 * @fattr are more recent, in which case the function returns the
 * value '1'.
 * A return value of '0' indicates no measurable change
 * A return value of '-1' means that the attributes in @inode are
 * more recent.
 */
static int nfs_inode_attrs_cmp_monotonic(const struct nfs_fattr *fattr,
                                         const struct inode *inode)
{
        s64 diff = fattr->change_attr - inode_peek_iversion_raw(inode);
        if (diff > 0)
                return 1;
        return diff == 0 ? 0 : -1;
}

/**
 * nfs_inode_attrs_cmp_strict_monotonic - compare attributes
 * @fattr: attributes
 * @inode: pointer to inode
 *
 * Attempt to divine whether or not an RPC call reply carrying stale
 * attributes got scheduled after another call carrying updated ones.
 *
 * We assume that the server observes strictly monotonic semantics for
 * the change attribute, so a larger value means that the attributes in
 * @fattr are more recent, in which case the function returns the
 * value '1'.
 * A return value of '-1' means that the attributes in @inode are
 * more recent or unchanged.
 */
static int nfs_inode_attrs_cmp_strict_monotonic(const struct nfs_fattr *fattr,
                                                const struct inode *inode)
{
        return  nfs_inode_attrs_cmp_monotonic(fattr, inode) > 0 ? 1 : -1;
}

/**
 * nfs_inode_attrs_cmp - compare attributes
 * @fattr: attributes
 * @inode: pointer to inode
 *
 * This function returns '1' if it thinks the attributes in @fattr are
 * more recent than the ones cached in @inode. It returns '-1' if
 * the attributes in @inode are more recent than the ones in @fattr,
 * and it returns 0 if not sure.
 */
static int nfs_inode_attrs_cmp(const struct nfs_fattr *fattr,
                               const struct inode *inode)
{
        if (nfs_inode_attrs_cmp_generic(fattr, inode) > 0)
                return 1;
        switch (NFS_SERVER(inode)->change_attr_type) {
        case NFS4_CHANGE_TYPE_IS_UNDEFINED:
                break;
        case NFS4_CHANGE_TYPE_IS_TIME_METADATA:
                if (!(fattr->valid & NFS_ATTR_FATTR_CHANGE))
                        break;
                return nfs_inode_attrs_cmp_monotonic(fattr, inode);
        default:
                if (!(fattr->valid & NFS_ATTR_FATTR_CHANGE))
                        break;
                return nfs_inode_attrs_cmp_strict_monotonic(fattr, inode);
        }
        return 0;
}

/**
 * nfs_inode_finish_partial_attr_update - complete a previous inode update
 * @fattr: attributes
 * @inode: pointer to inode
 *
 * Returns '1' if the last attribute update left the inode cached
 * attributes in a partially unrevalidated state, and @fattr
 * matches the change attribute of that partial update.
 * Otherwise returns '0'.
 */
static int nfs_inode_finish_partial_attr_update(const struct nfs_fattr *fattr,
                                                const struct inode *inode)
{
        const unsigned long check_valid =
                NFS_INO_INVALID_ATIME | NFS_INO_INVALID_CTIME |
                NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE |
                NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_OTHER |
                NFS_INO_INVALID_NLINK;
        unsigned long cache_validity = NFS_I(inode)->cache_validity;
        enum nfs4_change_attr_type ctype = NFS_SERVER(inode)->change_attr_type;

        if (ctype != NFS4_CHANGE_TYPE_IS_UNDEFINED &&
            !(cache_validity & NFS_INO_INVALID_CHANGE) &&
            (cache_validity & check_valid) != 0 &&
            (fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 &&
            nfs_inode_attrs_cmp_monotonic(fattr, inode) == 0)
                return 1;
        return 0;
}

static void nfs_ooo_merge(struct nfs_inode *nfsi,
                          u64 start, u64 end)
{
        int i, cnt;

        if (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)
                /* No point merging anything */
                return;

        if (!nfsi->ooo) {
                nfsi->ooo = kmalloc(sizeof(*nfsi->ooo), GFP_ATOMIC);
                if (!nfsi->ooo) {
                        nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER;
                        return;
                }
                nfsi->ooo->cnt = 0;
        }

        /* add this range, merging if possible */
        cnt = nfsi->ooo->cnt;
        for (i = 0; i < cnt; i++) {
                if (end == nfsi->ooo->gap[i].start)
                        end = nfsi->ooo->gap[i].end;
                else if (start == nfsi->ooo->gap[i].end)
                        start = nfsi->ooo->gap[i].start;
                else
                        continue;
                /* Remove 'i' from table and loop to insert the new range */
                cnt -= 1;
                nfsi->ooo->gap[i] = nfsi->ooo->gap[cnt];
                i = -1;
        }
        if (start != end) {
                if (cnt >= ARRAY_SIZE(nfsi->ooo->gap)) {
                        nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER;
                        kfree(nfsi->ooo);
                        nfsi->ooo = NULL;
                        return;
                }
                nfsi->ooo->gap[cnt].start = start;
                nfsi->ooo->gap[cnt].end = end;
                cnt += 1;
        }
        nfsi->ooo->cnt = cnt;
}

static void nfs_ooo_record(struct nfs_inode *nfsi,
                           struct nfs_fattr *fattr)
{
        /* This reply was out-of-order, so record in the
         * pre/post change id, possibly cancelling
         * gaps created when iversion was jumpped forward.
         */
        if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) &&
            (fattr->valid & NFS_ATTR_FATTR_PRECHANGE))
                nfs_ooo_merge(nfsi,
                              fattr->change_attr,
                              fattr->pre_change_attr);
}

static int nfs_refresh_inode_locked(struct inode *inode,
                                    struct nfs_fattr *fattr)
{
        int attr_cmp = nfs_inode_attrs_cmp(fattr, inode);
        int ret = 0;

        trace_nfs_refresh_inode_enter(inode);

        if (attr_cmp > 0 || nfs_inode_finish_partial_attr_update(fattr, inode))
                ret = nfs_update_inode(inode, fattr);
        else {
                nfs_ooo_record(NFS_I(inode), fattr);

                if (attr_cmp == 0)
                        ret = nfs_check_inode_attributes(inode, fattr);
        }

        trace_nfs_refresh_inode_exit(inode, ret);
        return ret;
}

/**
 * nfs_refresh_inode - try to update the inode attribute cache
 * @inode: pointer to inode
 * @fattr: updated attributes
 *
 * Check that an RPC call that returned attributes has not overlapped with
 * other recent updates of the inode metadata, then decide whether it is
 * safe to do a full update of the inode attributes, or whether just to
 * call nfs_check_inode_attributes.
 */
int nfs_refresh_inode(struct inode *inode, struct nfs_fattr *fattr)
{
        int status;

        if ((fattr->valid & NFS_ATTR_FATTR) == 0)
                return 0;
        spin_lock(&inode->i_lock);
        status = nfs_refresh_inode_locked(inode, fattr);
        spin_unlock(&inode->i_lock);

        return status;
}
EXPORT_SYMBOL_GPL(nfs_refresh_inode);

static int nfs_post_op_update_inode_locked(struct inode *inode,
                struct nfs_fattr *fattr, unsigned int invalid)
{
        if (S_ISDIR(inode->i_mode))
                invalid |= NFS_INO_INVALID_DATA;
        nfs_set_cache_invalid(inode, invalid);
        if ((fattr->valid & NFS_ATTR_FATTR) == 0)
                return 0;
        return nfs_refresh_inode_locked(inode, fattr);
}

/**
 * nfs_post_op_update_inode - try to update the inode attribute cache
 * @inode: pointer to inode
 * @fattr: updated attributes
 *
 * After an operation that has changed the inode metadata, mark the
 * attribute cache as being invalid, then try to update it.
 *
 * NB: if the server didn't return any post op attributes, this
 * function will force the retrieval of attributes before the next
 * NFS request.  Thus it should be used only for operations that
 * are expected to change one or more attributes, to avoid
 * unnecessary NFS requests and trips through nfs_update_inode().
 */
int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr)
{
        int status;

        spin_lock(&inode->i_lock);
        nfs_fattr_set_barrier(fattr);
        status = nfs_post_op_update_inode_locked(inode, fattr,
                        NFS_INO_INVALID_CHANGE
                        | NFS_INO_INVALID_CTIME
                        | NFS_INO_REVAL_FORCED);
        spin_unlock(&inode->i_lock);

        return status;
}
EXPORT_SYMBOL_GPL(nfs_post_op_update_inode);

/**
 * nfs_post_op_update_inode_force_wcc_locked - update the inode attribute cache
 * @inode: pointer to inode
 * @fattr: updated attributes
 *
 * After an operation that has changed the inode metadata, mark the
 * attribute cache as being invalid, then try to update it. Fake up
 * weak cache consistency data, if none exist.
 *
 * This function is mainly designed to be used by the ->write_done() functions.
 */
int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr)
{
        int attr_cmp = nfs_inode_attrs_cmp(fattr, inode);
        int status;

        /* Don't do a WCC update if these attributes are already stale */
        if (attr_cmp < 0)
                return 0;
        if ((fattr->valid & NFS_ATTR_FATTR) == 0 || !attr_cmp) {
                /* Record the pre/post change info before clearing PRECHANGE */
                nfs_ooo_record(NFS_I(inode), fattr);
                fattr->valid &= ~(NFS_ATTR_FATTR_PRECHANGE
                                | NFS_ATTR_FATTR_PRESIZE
                                | NFS_ATTR_FATTR_PREMTIME
                                | NFS_ATTR_FATTR_PRECTIME);
                goto out_noforce;
        }
        if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 &&
                        (fattr->valid & NFS_ATTR_FATTR_PRECHANGE) == 0) {
                fattr->pre_change_attr = inode_peek_iversion_raw(inode);
                fattr->valid |= NFS_ATTR_FATTR_PRECHANGE;
        }
        if ((fattr->valid & NFS_ATTR_FATTR_CTIME) != 0 &&
                        (fattr->valid & NFS_ATTR_FATTR_PRECTIME) == 0) {
                fattr->pre_ctime = inode_get_ctime(inode);
                fattr->valid |= NFS_ATTR_FATTR_PRECTIME;
        }
        if ((fattr->valid & NFS_ATTR_FATTR_MTIME) != 0 &&
                        (fattr->valid & NFS_ATTR_FATTR_PREMTIME) == 0) {
                fattr->pre_mtime = inode_get_mtime(inode);
                fattr->valid |= NFS_ATTR_FATTR_PREMTIME;
        }
        if ((fattr->valid & NFS_ATTR_FATTR_SIZE) != 0 &&
                        (fattr->valid & NFS_ATTR_FATTR_PRESIZE) == 0) {
                fattr->pre_size = i_size_read(inode);
                fattr->valid |= NFS_ATTR_FATTR_PRESIZE;
        }
out_noforce:
        status = nfs_post_op_update_inode_locked(inode, fattr,
                        NFS_INO_INVALID_CHANGE
                        | NFS_INO_INVALID_CTIME
                        | NFS_INO_INVALID_MTIME
                        | NFS_INO_INVALID_BLOCKS);
        return status;
}

/**
 * nfs_post_op_update_inode_force_wcc - try to update the inode attribute cache
 * @inode: pointer to inode
 * @fattr: updated attributes
 *
 * After an operation that has changed the inode metadata, mark the
 * attribute cache as being invalid, then try to update it. Fake up
 * weak cache consistency data, if none exist.
 *
 * This function is mainly designed to be used by the ->write_done() functions.
 */
int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr)
{
        int status;

        spin_lock(&inode->i_lock);
        nfs_fattr_set_barrier(fattr);
        status = nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
        spin_unlock(&inode->i_lock);
        return status;
}
EXPORT_SYMBOL_GPL(nfs_post_op_update_inode_force_wcc);


/*
 * Many nfs protocol calls return the new file attributes after
 * an operation.  Here we update the inode to reflect the state
 * of the server's inode.
 *
 * This is a bit tricky because we have to make sure all dirty pages
 * have been sent off to the server before calling invalidate_inode_pages.
 * To make sure no other process adds more write requests while we try
 * our best to flush them, we make them sleep during the attribute refresh.
 *
 * A very similar scenario holds for the dir cache.
 */
static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
{
        struct nfs_server *server = NFS_SERVER(inode);
        struct nfs_inode *nfsi = NFS_I(inode);
        loff_t cur_isize, new_isize;
        u64 fattr_supported = server->fattr_valid;
        unsigned long invalid = 0;
        unsigned long now = jiffies;
        unsigned long save_cache_validity;
        bool have_writers = nfs_file_has_buffered_writers(nfsi);
        bool cache_revalidated = true;
        bool attr_changed = false;
        bool have_delegation;

        dfprintk(VFS, "NFS: %s(%s/%lu fh_crc=0x%08x ct=%d info=0x%x)\n",
                        __func__, inode->i_sb->s_id, inode->i_ino,
                        nfs_display_fhandle_hash(NFS_FH(inode)),
                        atomic_read(&inode->i_count), fattr->valid);

        if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) {
                /* Only a mounted-on-fileid? Just exit */
                if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID)
                        return 0;
        /* Has the inode gone and changed behind our back? */
        } else if (nfsi->fileid != fattr->fileid) {
                /* Is this perhaps the mounted-on fileid? */
                if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) &&
                    nfsi->fileid == fattr->mounted_on_fileid)
                        return 0;
                printk(KERN_ERR "NFS: server %s error: fileid changed\n"
                        "fsid %s: expected fileid 0x%Lx, got 0x%Lx\n",
                        NFS_SERVER(inode)->nfs_client->cl_hostname,
                        inode->i_sb->s_id, (long long)nfsi->fileid,
                        (long long)fattr->fileid);
                goto out_err;
        }

        /*
         * Make sure the inode's type hasn't changed.
         */
        if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode)) {
                /*
                * Big trouble! The inode has become a different object.
                */
                printk(KERN_DEBUG "NFS: %s: inode %lu mode changed, %07o to %07o\n",
                                __func__, inode->i_ino, inode->i_mode, fattr->mode);
                goto out_err;
        }

        /* Update the fsid? */
        if (S_ISDIR(inode->i_mode) && (fattr->valid & NFS_ATTR_FATTR_FSID) &&
                        !nfs_fsid_equal(&server->fsid, &fattr->fsid) &&
                        !IS_AUTOMOUNT(inode))
                server->fsid = fattr->fsid;

        /* Save the delegation state before clearing cache_validity */
        have_delegation = nfs_have_delegated_attributes(inode);

        /*
         * Update the read time so we don't revalidate too often.
         */
        nfsi->read_cache_jiffies = fattr->time_start;

        /* Fix up any delegated attributes in the struct nfs_fattr */
        nfs_fattr_fixup_delegated(inode, fattr);

        save_cache_validity = nfsi->cache_validity;
        nfsi->cache_validity &= ~(NFS_INO_INVALID_ATTR
                        | NFS_INO_INVALID_ATIME
                        | NFS_INO_REVAL_FORCED
                        | NFS_INO_INVALID_BLOCKS);

        /* Do atomic weak cache consistency updates */
        nfs_wcc_update_inode(inode, fattr);

        if (pnfs_layoutcommit_outstanding(inode)) {
                nfsi->cache_validity |=
                        save_cache_validity &
                        (NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME |
                         NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE |
                         NFS_INO_INVALID_BLOCKS);
                cache_revalidated = false;
        }

        /* More cache consistency checks */
        if (fattr->valid & NFS_ATTR_FATTR_CHANGE) {
                if (!have_writers && nfsi->ooo && nfsi->ooo->cnt == 1 &&
                    nfsi->ooo->gap[0].end == inode_peek_iversion_raw(inode)) {
                        /* There is one remaining gap that hasn't been
                         * merged into iversion - do that now.
                         */
                        inode_set_iversion_raw(inode, nfsi->ooo->gap[0].start);
                        kfree(nfsi->ooo);
                        nfsi->ooo = NULL;
                }
                if (!inode_eq_iversion_raw(inode, fattr->change_attr)) {
                        /* Could it be a race with writeback? */
                        if (!(have_writers || have_delegation)) {
                                invalid |= NFS_INO_INVALID_DATA
                                        | NFS_INO_INVALID_ACCESS
                                        | NFS_INO_INVALID_ACL
                                        | NFS_INO_INVALID_XATTR;
                                /* Force revalidate of all attributes */
                                save_cache_validity |= NFS_INO_INVALID_CTIME
                                        | NFS_INO_INVALID_MTIME
                                        | NFS_INO_INVALID_SIZE
                                        | NFS_INO_INVALID_BLOCKS
                                        | NFS_INO_INVALID_NLINK
                                        | NFS_INO_INVALID_MODE
                                        | NFS_INO_INVALID_OTHER;
                                if (S_ISDIR(inode->i_mode))
                                        nfs_force_lookup_revalidate(inode);
                                attr_changed = true;
                                dprintk("NFS: change_attr change on server for file %s/%ld\n",
                                                inode->i_sb->s_id,
                                                inode->i_ino);
                        } else if (!have_delegation) {
                                nfs_ooo_record(nfsi, fattr);
                                nfs_ooo_merge(nfsi, inode_peek_iversion_raw(inode),
                                              fattr->change_attr);
                        }
                        inode_set_iversion_raw(inode, fattr->change_attr);
                }
        } else {
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_CHANGE;
                if (!have_delegation ||
                    (nfsi->cache_validity & NFS_INO_INVALID_CHANGE) != 0)
                        cache_revalidated = false;
        }

        if (fattr->valid & NFS_ATTR_FATTR_MTIME)
                inode_set_mtime_to_ts(inode, fattr->mtime);
        else if (fattr_supported & NFS_ATTR_FATTR_MTIME)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_MTIME;

        if (fattr->valid & NFS_ATTR_FATTR_CTIME)
                inode_set_ctime_to_ts(inode, fattr->ctime);
        else if (fattr_supported & NFS_ATTR_FATTR_CTIME)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_CTIME;

        /* Check if our cached file size is stale */
        if (fattr->valid & NFS_ATTR_FATTR_SIZE) {
                new_isize = nfs_size_to_loff_t(fattr->size);
                cur_isize = i_size_read(inode);
                if (new_isize != cur_isize && !have_delegation) {
                        /* Do we perhaps have any outstanding writes, or has
                         * the file grown beyond our last write? */
                        if (!nfs_have_writebacks(inode) || new_isize > cur_isize) {
                                trace_nfs_size_update(inode, new_isize);
                                i_size_write(inode, new_isize);
                                if (!have_writers)
                                        invalid |= NFS_INO_INVALID_DATA;
                        }
                }
                if (new_isize == 0 &&
                    !(fattr->valid & (NFS_ATTR_FATTR_SPACE_USED |
                                      NFS_ATTR_FATTR_BLOCKS_USED))) {
                        fattr->du.nfs3.used = 0;
                        fattr->valid |= NFS_ATTR_FATTR_SPACE_USED;
                }
        } else
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_SIZE;

        if (fattr->valid & NFS_ATTR_FATTR_ATIME)
                inode_set_atime_to_ts(inode, fattr->atime);
        else if (fattr_supported & NFS_ATTR_FATTR_ATIME)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_ATIME;

        if (fattr->valid & NFS_ATTR_FATTR_MODE) {
                if ((inode->i_mode & S_IALLUGO) != (fattr->mode & S_IALLUGO)) {
                        umode_t newmode = inode->i_mode & S_IFMT;
                        newmode |= fattr->mode & S_IALLUGO;
                        inode->i_mode = newmode;
                        invalid |= NFS_INO_INVALID_ACCESS
                                | NFS_INO_INVALID_ACL;
                }
        } else if (fattr_supported & NFS_ATTR_FATTR_MODE)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_MODE;

        if (fattr->valid & NFS_ATTR_FATTR_OWNER) {
                if (!uid_eq(inode->i_uid, fattr->uid)) {
                        invalid |= NFS_INO_INVALID_ACCESS
                                | NFS_INO_INVALID_ACL;
                        inode->i_uid = fattr->uid;
                }
        } else if (fattr_supported & NFS_ATTR_FATTR_OWNER)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_OTHER;

        if (fattr->valid & NFS_ATTR_FATTR_GROUP) {
                if (!gid_eq(inode->i_gid, fattr->gid)) {
                        invalid |= NFS_INO_INVALID_ACCESS
                                | NFS_INO_INVALID_ACL;
                        inode->i_gid = fattr->gid;
                }
        } else if (fattr_supported & NFS_ATTR_FATTR_GROUP)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_OTHER;

        if (fattr->valid & NFS_ATTR_FATTR_NLINK) {
                if (inode->i_nlink != fattr->nlink)
                        set_nlink(inode, fattr->nlink);
        } else if (fattr_supported & NFS_ATTR_FATTR_NLINK)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_NLINK;

        if (fattr->valid & NFS_ATTR_FATTR_SPACE_USED) {
                /*
                 * report the blocks in 512byte units
                 */
                inode->i_blocks = nfs_calc_block_size(fattr->du.nfs3.used);
        } else if (fattr_supported & NFS_ATTR_FATTR_SPACE_USED)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_BLOCKS;

        if (fattr->valid & NFS_ATTR_FATTR_BLOCKS_USED)
                inode->i_blocks = fattr->du.nfs2.blocks;
        else if (fattr_supported & NFS_ATTR_FATTR_BLOCKS_USED)
                nfsi->cache_validity |=
                        save_cache_validity & NFS_INO_INVALID_BLOCKS;

        /* Update attrtimeo value if we're out of the unstable period */
        if (attr_changed) {
                nfs_inc_stats(inode, NFSIOS_ATTRINVALIDATE);
                nfsi->attrtimeo = NFS_MINATTRTIMEO(inode);
                nfsi->attrtimeo_timestamp = now;
                /* Set barrier to be more recent than all outstanding updates */
                nfsi->attr_gencount = nfs_inc_attr_generation_counter();
        } else {
                if (cache_revalidated) {
                        if (!time_in_range_open(now, nfsi->attrtimeo_timestamp,
                                nfsi->attrtimeo_timestamp + nfsi->attrtimeo)) {
                                nfsi->attrtimeo <<= 1;
                                if (nfsi->attrtimeo > NFS_MAXATTRTIMEO(inode))
                                        nfsi->attrtimeo = NFS_MAXATTRTIMEO(inode);
                        }
                        nfsi->attrtimeo_timestamp = now;
                }
                /* Set the barrier to be more recent than this fattr */
                if ((long)(fattr->gencount - nfsi->attr_gencount) > 0)
                        nfsi->attr_gencount = fattr->gencount;
        }

        /* Don't invalidate the data if we were to blame */
        if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)
                                || S_ISLNK(inode->i_mode)))
                invalid &= ~NFS_INO_INVALID_DATA;
        nfs_set_cache_invalid(inode, invalid);

        return 0;
 out_err:
        /*
         * No need to worry about unhashing the dentry, as the
         * lookup validation will know that the inode is bad.
         * (But we fall through to invalidate the caches.)
         */
        nfs_set_inode_stale_locked(inode);
        return -ESTALE;
}

struct inode *nfs_alloc_inode(struct super_block *sb)
{
        struct nfs_inode *nfsi;
        nfsi = alloc_inode_sb(sb, nfs_inode_cachep, GFP_KERNEL);
        if (!nfsi)
                return NULL;
        nfsi->flags = 0UL;
        nfsi->cache_validity = 0UL;
        nfsi->ooo = NULL;
#if IS_ENABLED(CONFIG_NFS_V4)
        nfsi->nfs4_acl = NULL;
#endif /* CONFIG_NFS_V4 */
#ifdef CONFIG_NFS_V4_2
        nfsi->xattr_cache = NULL;
#endif
        nfs_netfs_inode_init(nfsi);

        return &nfsi->vfs_inode;
}
EXPORT_SYMBOL_GPL(nfs_alloc_inode);

void nfs_free_inode(struct inode *inode)
{
        kfree(NFS_I(inode)->ooo);
        kmem_cache_free(nfs_inode_cachep, NFS_I(inode));
}
EXPORT_SYMBOL_GPL(nfs_free_inode);

static inline void nfs4_init_once(struct nfs_inode *nfsi)
{
#if IS_ENABLED(CONFIG_NFS_V4)
        INIT_LIST_HEAD(&nfsi->open_states);
        nfsi->delegation = NULL;
        init_rwsem(&nfsi->rwsem);
        nfsi->layout = NULL;
#endif
}

static void init_once(void *foo)
{
        struct nfs_inode *nfsi = foo;

        inode_init_once(&nfsi->vfs_inode);
        INIT_LIST_HEAD(&nfsi->open_files);
        INIT_LIST_HEAD(&nfsi->access_cache_entry_lru);
        INIT_LIST_HEAD(&nfsi->access_cache_inode_lru);
        nfs4_init_once(nfsi);
}

static int __init nfs_init_inodecache(void)
{
        nfs_inode_cachep = kmem_cache_create("nfs_inode_cache",
                                             sizeof(struct nfs_inode),
                                             0, (SLAB_RECLAIM_ACCOUNT|
                                                SLAB_ACCOUNT),
                                             init_once);
        if (nfs_inode_cachep == NULL)
                return -ENOMEM;

        return 0;
}

static void nfs_destroy_inodecache(void)
{
        /*
         * Make sure all delayed rcu free inodes are flushed before we
         * destroy cache.
         */
        rcu_barrier();
        kmem_cache_destroy(nfs_inode_cachep);
}

struct workqueue_struct *nfslocaliod_workqueue;
struct workqueue_struct *nfsiod_workqueue;
EXPORT_SYMBOL_GPL(nfsiod_workqueue);

/*
 * Destroy the nfsiod workqueues
 */
static void nfsiod_stop(void)
{
        struct workqueue_struct *wq;

        wq = nfsiod_workqueue;
        if (wq != NULL) {
                nfsiod_workqueue = NULL;
                destroy_workqueue(wq);
        }
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        wq = nfslocaliod_workqueue;
        if (wq != NULL) {
                nfslocaliod_workqueue = NULL;
                destroy_workqueue(wq);
        }
#endif /* CONFIG_NFS_LOCALIO */
}

/*
 * Start the nfsiod workqueues
 */
static int nfsiod_start(void)
{
        dprintk("RPC:       creating workqueue nfsiod\n");
        nfsiod_workqueue = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
        if (nfsiod_workqueue == NULL)
                return -ENOMEM;
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        /*
         * localio writes need to use a normal (non-memreclaim) workqueue.
         * When we start getting low on space, XFS goes and calls flush_work() on
         * a non-memreclaim work queue, which causes a priority inversion problem.
         */
        dprintk("RPC:       creating workqueue nfslocaliod\n");
        nfslocaliod_workqueue = alloc_workqueue("nfslocaliod", WQ_UNBOUND, 0);
        if (unlikely(nfslocaliod_workqueue == NULL)) {
                nfsiod_stop();
                return -ENOMEM;
        }
#endif /* CONFIG_NFS_LOCALIO */
        return 0;
}

unsigned int nfs_net_id;
EXPORT_SYMBOL_GPL(nfs_net_id);

static int nfs_net_init(struct net *net)
{
        struct nfs_net *nn = net_generic(net, nfs_net_id);

        nfs_clients_init(net);

        if (!rpc_proc_register(net, &nn->rpcstats)) {
                nfs_clients_exit(net);
                return -ENOMEM;
        }

        return nfs_fs_proc_net_init(net);
}

static void nfs_net_exit(struct net *net)
{
        rpc_proc_unregister(net, "nfs");
        nfs_fs_proc_net_exit(net);
        nfs_clients_exit(net);
}

static struct pernet_operations nfs_net_ops = {
        .init = nfs_net_init,
        .exit = nfs_net_exit,
        .id   = &nfs_net_id,
        .size = sizeof(struct nfs_net),
};

/*
 * Initialize NFS
 */
static int __init init_nfs_fs(void)
{
        int err;

        err = nfs_sysfs_init();
        if (err < 0)
                goto out10;

        err = register_pernet_subsys(&nfs_net_ops);
        if (err < 0)
                goto out9;

        err = nfsiod_start();
        if (err)
                goto out7;

        err = nfs_fs_proc_init();
        if (err)
                goto out6;

        err = nfs_init_nfspagecache();
        if (err)
                goto out5;

        err = nfs_init_inodecache();
        if (err)
                goto out4;

        err = nfs_init_readpagecache();
        if (err)
                goto out3;

        err = nfs_init_writepagecache();
        if (err)
                goto out2;

        err = nfs_init_directcache();
        if (err)
                goto out1;

        err = register_nfs_fs();
        if (err)
                goto out0;

        return 0;
out0:
        nfs_destroy_directcache();
out1:
        nfs_destroy_writepagecache();
out2:
        nfs_destroy_readpagecache();
out3:
        nfs_destroy_inodecache();
out4:
        nfs_destroy_nfspagecache();
out5:
        nfs_fs_proc_exit();
out6:
        nfsiod_stop();
out7:
        unregister_pernet_subsys(&nfs_net_ops);
out9:
        nfs_sysfs_exit();
out10:
        return err;
}

static void __exit exit_nfs_fs(void)
{
        nfs_destroy_directcache();
        nfs_destroy_writepagecache();
        nfs_destroy_readpagecache();
        nfs_destroy_inodecache();
        nfs_destroy_nfspagecache();
        unregister_pernet_subsys(&nfs_net_ops);
        unregister_nfs_fs();
        nfs_fs_proc_exit();
        nfsiod_stop();
        nfs_sysfs_exit();
}

/* Not quite true; I just maintain it */
MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>");
MODULE_DESCRIPTION("NFS client support");
MODULE_LICENSE("GPL");
module_param(enable_ino64, bool, 0644);

module_init(init_nfs_fs)
module_exit(exit_nfs_fs)































    4 

    4 
    4 





    1 

    4 

    4 



    4 










    3 

    3 
    3 





    1 










    3 








    4 















    4 


    3 




    4 








    4 
    2 

    3 
    4 




    4 









    3 


    3 


    3 









    2 













    5 
    1 



















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright Red Hat Inc. 2017
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions manipulate sctp stream queue/scheduling.
 *
 * Please send any bug reports or fixes you make to the
 * email addresched(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
 */

#include <linux/list.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>
#include <net/sctp/stream_sched.h>

/* Priority handling
 * RFC DRAFT ndata section 3.2
 */
static void sctp_sched_rr_unsched_all(struct sctp_stream *stream);

static void sctp_sched_rr_next_stream(struct sctp_stream *stream)
{
        struct list_head *pos;

        pos = stream->rr_next->rr_list.next;
        if (pos == &stream->rr_list)
                pos = pos->next;
        stream->rr_next = list_entry(pos, struct sctp_stream_out_ext, rr_list);
}

static void sctp_sched_rr_unsched(struct sctp_stream *stream,
                                  struct sctp_stream_out_ext *soute)
{
        if (stream->rr_next == soute)
                /* Try to move to the next stream */
                sctp_sched_rr_next_stream(stream);

        list_del_init(&soute->rr_list);

        /* If we have no other stream queued, clear next */
        if (list_empty(&stream->rr_list))
                stream->rr_next = NULL;
}

static void sctp_sched_rr_sched(struct sctp_stream *stream,
                                struct sctp_stream_out_ext *soute)
{
        if (!list_empty(&soute->rr_list))
                /* Already scheduled. */
                return;

        /* Schedule the stream */
        list_add_tail(&soute->rr_list, &stream->rr_list);

        if (!stream->rr_next)
                stream->rr_next = soute;
}

static int sctp_sched_rr_set(struct sctp_stream *stream, __u16 sid,
                             __u16 prio, gfp_t gfp)
{
        return 0;
}

static int sctp_sched_rr_get(struct sctp_stream *stream, __u16 sid,
                             __u16 *value)
{
        return 0;
}

static int sctp_sched_rr_init(struct sctp_stream *stream)
{
        INIT_LIST_HEAD(&stream->rr_list);
        stream->rr_next = NULL;

        return 0;
}

static int sctp_sched_rr_init_sid(struct sctp_stream *stream, __u16 sid,
                                  gfp_t gfp)
{
        INIT_LIST_HEAD(&SCTP_SO(stream, sid)->ext->rr_list);

        return 0;
}

static void sctp_sched_rr_free_sid(struct sctp_stream *stream, __u16 sid)
{
}

static void sctp_sched_rr_enqueue(struct sctp_outq *q,
                                  struct sctp_datamsg *msg)
{
        struct sctp_stream *stream;
        struct sctp_chunk *ch;
        __u16 sid;

        ch = list_first_entry(&msg->chunks, struct sctp_chunk, frag_list);
        sid = sctp_chunk_stream_no(ch);
        stream = &q->asoc->stream;
        sctp_sched_rr_sched(stream, SCTP_SO(stream, sid)->ext);
}

static struct sctp_chunk *sctp_sched_rr_dequeue(struct sctp_outq *q)
{
        struct sctp_stream *stream = &q->asoc->stream;
        struct sctp_stream_out_ext *soute;
        struct sctp_chunk *ch = NULL;

        /* Bail out quickly if queue is empty */
        if (list_empty(&q->out_chunk_list))
                goto out;

        /* Find which chunk is next */
        if (stream->out_curr)
                soute = stream->out_curr->ext;
        else
                soute = stream->rr_next;
        ch = list_entry(soute->outq.next, struct sctp_chunk, stream_list);

        sctp_sched_dequeue_common(q, ch);

out:
        return ch;
}

static void sctp_sched_rr_dequeue_done(struct sctp_outq *q,
                                       struct sctp_chunk *ch)
{
        struct sctp_stream_out_ext *soute;
        __u16 sid;

        /* Last chunk on that msg, move to the next stream */
        sid = sctp_chunk_stream_no(ch);
        soute = SCTP_SO(&q->asoc->stream, sid)->ext;

        sctp_sched_rr_next_stream(&q->asoc->stream);

        if (list_empty(&soute->outq))
                sctp_sched_rr_unsched(&q->asoc->stream, soute);
}

static void sctp_sched_rr_sched_all(struct sctp_stream *stream)
{
        struct sctp_association *asoc;
        struct sctp_stream_out_ext *soute;
        struct sctp_chunk *ch;

        asoc = container_of(stream, struct sctp_association, stream);
        list_for_each_entry(ch, &asoc->outqueue.out_chunk_list, list) {
                __u16 sid;

                sid = sctp_chunk_stream_no(ch);
                soute = SCTP_SO(stream, sid)->ext;
                if (soute)
                        sctp_sched_rr_sched(stream, soute);
        }
}

static void sctp_sched_rr_unsched_all(struct sctp_stream *stream)
{
        struct sctp_stream_out_ext *soute, *tmp;

        list_for_each_entry_safe(soute, tmp, &stream->rr_list, rr_list)
                sctp_sched_rr_unsched(stream, soute);
}

static struct sctp_sched_ops sctp_sched_rr = {
        .set = sctp_sched_rr_set,
        .get = sctp_sched_rr_get,
        .init = sctp_sched_rr_init,
        .init_sid = sctp_sched_rr_init_sid,
        .free_sid = sctp_sched_rr_free_sid,
        .enqueue = sctp_sched_rr_enqueue,
        .dequeue = sctp_sched_rr_dequeue,
        .dequeue_done = sctp_sched_rr_dequeue_done,
        .sched_all = sctp_sched_rr_sched_all,
        .unsched_all = sctp_sched_rr_unsched_all,
};

void sctp_sched_ops_rr_init(void)
{
        sctp_sched_ops_register(SCTP_SS_RR, &sctp_sched_rr);
}


























































































































    8 
    5 
    6 

    1 
    6 
    5 

    5 
    6 

    6 
    6 
    5 
    6 












    5 







    4 













    6 




    6 























































































































    6 





    6 




















    6 
    6 





























































    5 

























    1 

    1 









    5 















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * include/linux/buffer_head.h
 *
 * Everything to do with buffer_heads.
 */

#ifndef _LINUX_BUFFER_HEAD_H
#define _LINUX_BUFFER_HEAD_H

#include <linux/types.h>
#include <linux/blk_types.h>
#include <linux/fs.h>
#include <linux/linkage.h>
#include <linux/pagemap.h>
#include <linux/wait.h>
#include <linux/atomic.h>

enum bh_state_bits {
        BH_Uptodate,        /* Contains valid data */
        BH_Dirty,        /* Is dirty */
        BH_Lock,        /* Is locked */
        BH_Req,                /* Has been submitted for I/O */

        BH_Mapped,        /* Has a disk mapping */
        BH_New,                /* Disk mapping was newly created by get_block */
        BH_Async_Read,        /* Is under end_buffer_async_read I/O */
        BH_Async_Write,        /* Is under end_buffer_async_write I/O */
        BH_Delay,        /* Buffer is not yet allocated on disk */
        BH_Boundary,        /* Block is followed by a discontiguity */
        BH_Write_EIO,        /* I/O error on write */
        BH_Unwritten,        /* Buffer is allocated on disk but not written */
        BH_Quiet,        /* Buffer Error Prinks to be quiet */
        BH_Meta,        /* Buffer contains metadata */
        BH_Prio,        /* Buffer should be submitted with REQ_PRIO */
        BH_Defer_Completion, /* Defer AIO completion to workqueue */

        BH_PrivateStart,/* not a state bit, but the first bit available
                         * for private allocation by other entities
                         */
};

#define MAX_BUF_PER_PAGE (PAGE_SIZE / 512)

struct page;
struct buffer_head;
struct address_space;
typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate);

/*
 * Historically, a buffer_head was used to map a single block
 * within a page, and of course as the unit of I/O through the
 * filesystem and block layers.  Nowadays the basic I/O unit
 * is the bio, and buffer_heads are used for extracting block
 * mappings (via a get_block_t call), for tracking state within
 * a folio (via a folio_mapping) and for wrapping bio submission
 * for backward compatibility reasons (e.g. submit_bh).
 */
struct buffer_head {
        unsigned long b_state;                /* buffer state bitmap (see above) */
        struct buffer_head *b_this_page;/* circular list of page's buffers */
        union {
                struct page *b_page;        /* the page this bh is mapped to */
                struct folio *b_folio;        /* the folio this bh is mapped to */
        };

        sector_t b_blocknr;                /* start block number */
        size_t b_size;                        /* size of mapping */
        char *b_data;                        /* pointer to data within the page */

        struct block_device *b_bdev;
        bh_end_io_t *b_end_io;                /* I/O completion */
         void *b_private;                /* reserved for b_end_io */
        struct list_head b_assoc_buffers; /* associated with another mapping */
        struct address_space *b_assoc_map;        /* mapping this buffer is
                                                   associated with */
        atomic_t b_count;                /* users using this buffer_head */
        spinlock_t b_uptodate_lock;        /* Used by the first bh in a page, to
                                         * serialise IO completion of other
                                         * buffers in the page */
};

/*
 * macro tricks to expand the set_buffer_foo(), clear_buffer_foo()
 * and buffer_foo() functions.
 * To avoid reset buffer flags that are already set, because that causes
 * a costly cache line transition, check the flag first.
 */
#define BUFFER_FNS(bit, name)                                                \
static __always_inline void set_buffer_##name(struct buffer_head *bh)        \
{                                                                        \
        if (!test_bit(BH_##bit, &(bh)->b_state))                        \
                set_bit(BH_##bit, &(bh)->b_state);                        \
}                                                                        \
static __always_inline void clear_buffer_##name(struct buffer_head *bh)        \
{                                                                        \
        clear_bit(BH_##bit, &(bh)->b_state);                                \
}                                                                        \
static __always_inline int buffer_##name(const struct buffer_head *bh)        \
{                                                                        \
        return test_bit(BH_##bit, &(bh)->b_state);                        \
}

/*
 * test_set_buffer_foo() and test_clear_buffer_foo()
 */
#define TAS_BUFFER_FNS(bit, name)                                        \
static __always_inline int test_set_buffer_##name(struct buffer_head *bh) \
{                                                                        \
        return test_and_set_bit(BH_##bit, &(bh)->b_state);                \
}                                                                        \
static __always_inline int test_clear_buffer_##name(struct buffer_head *bh) \
{                                                                        \
        return test_and_clear_bit(BH_##bit, &(bh)->b_state);                \
}                                                                        \

/*
 * Emit the buffer bitops functions.   Note that there are also functions
 * of the form "mark_buffer_foo()".  These are higher-level functions which
 * do something in addition to setting a b_state bit.
 */
BUFFER_FNS(Dirty, dirty)
TAS_BUFFER_FNS(Dirty, dirty)
BUFFER_FNS(Lock, locked)
BUFFER_FNS(Req, req)
TAS_BUFFER_FNS(Req, req)
BUFFER_FNS(Mapped, mapped)
BUFFER_FNS(New, new)
BUFFER_FNS(Async_Read, async_read)
BUFFER_FNS(Async_Write, async_write)
BUFFER_FNS(Delay, delay)
BUFFER_FNS(Boundary, boundary)
BUFFER_FNS(Write_EIO, write_io_error)
BUFFER_FNS(Unwritten, unwritten)
BUFFER_FNS(Meta, meta)
BUFFER_FNS(Prio, prio)
BUFFER_FNS(Defer_Completion, defer_completion)

static __always_inline void set_buffer_uptodate(struct buffer_head *bh)
{
        /*
         * If somebody else already set this uptodate, they will
         * have done the memory barrier, and a reader will thus
         * see *some* valid buffer state.
         *
         * Any other serialization (with IO errors or whatever that
         * might clear the bit) has to come from other state (eg BH_Lock).
         */
        if (test_bit(BH_Uptodate, &bh->b_state))
                return;

        /*
         * make it consistent with folio_mark_uptodate
         * pairs with smp_load_acquire in buffer_uptodate
         */
        smp_mb__before_atomic();
        set_bit(BH_Uptodate, &bh->b_state);
}

static __always_inline void clear_buffer_uptodate(struct buffer_head *bh)
{
        clear_bit(BH_Uptodate, &bh->b_state);
}

static __always_inline int buffer_uptodate(const struct buffer_head *bh)
{
        /*
         * make it consistent with folio_test_uptodate
         * pairs with smp_mb__before_atomic in set_buffer_uptodate
         */
        return test_bit_acquire(BH_Uptodate, &bh->b_state);
}

static inline unsigned long bh_offset(const struct buffer_head *bh)
{
        return (unsigned long)(bh)->b_data & (page_size(bh->b_page) - 1);
}

/* If we *know* page->private refers to buffer_heads */
#define page_buffers(page)                                        \
        ({                                                        \
                BUG_ON(!PagePrivate(page));                        \
                ((struct buffer_head *)page_private(page));        \
        })
#define page_has_buffers(page)        PagePrivate(page)
#define folio_buffers(folio)                folio_get_private(folio)

void buffer_check_dirty_writeback(struct folio *folio,
                                     bool *dirty, bool *writeback);

/*
 * Declarations
 */

void mark_buffer_dirty(struct buffer_head *bh);
void mark_buffer_write_io_error(struct buffer_head *bh);
void touch_buffer(struct buffer_head *bh);
void folio_set_bh(struct buffer_head *bh, struct folio *folio,
                  unsigned long offset);
struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
                                        gfp_t gfp);
struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size);
struct buffer_head *create_empty_buffers(struct folio *folio,
                unsigned long blocksize, unsigned long b_state);
void end_buffer_read_sync(struct buffer_head *bh, int uptodate);
void end_buffer_write_sync(struct buffer_head *bh, int uptodate);

/* Things to do with buffers at mapping->private_list */
void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode);
int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end,
                                  bool datasync);
int generic_buffers_fsync(struct file *file, loff_t start, loff_t end,
                          bool datasync);
void clean_bdev_aliases(struct block_device *bdev, sector_t block,
                        sector_t len);
static inline void clean_bdev_bh_alias(struct buffer_head *bh)
{
        clean_bdev_aliases(bh->b_bdev, bh->b_blocknr, 1);
}

void mark_buffer_async_write(struct buffer_head *bh);
void __wait_on_buffer(struct buffer_head *);
wait_queue_head_t *bh_waitq_head(struct buffer_head *bh);
struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block,
                        unsigned size);
struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block,
                unsigned size, gfp_t gfp);
void __brelse(struct buffer_head *);
void __bforget(struct buffer_head *);
void __breadahead(struct block_device *, sector_t block, unsigned int size);
struct buffer_head *__bread_gfp(struct block_device *,
                                sector_t block, unsigned size, gfp_t gfp);
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
void free_buffer_head(struct buffer_head * bh);
void unlock_buffer(struct buffer_head *bh);
void __lock_buffer(struct buffer_head *bh);
int sync_dirty_buffer(struct buffer_head *bh);
int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
void submit_bh(blk_opf_t, struct buffer_head *);
void write_boundary_block(struct block_device *bdev,
                        sector_t bblock, unsigned blocksize);
int bh_uptodate_or_lock(struct buffer_head *bh);
int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait);
void __bh_read_batch(int nr, struct buffer_head *bhs[],
                     blk_opf_t op_flags, bool force_lock);

/*
 * Generic address_space_operations implementations for buffer_head-backed
 * address_spaces.
 */
void block_invalidate_folio(struct folio *folio, size_t offset, size_t length);
int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,
                void *get_block);
int __block_write_full_folio(struct inode *inode, struct folio *folio,
                get_block_t *get_block, struct writeback_control *wbc);
int block_read_full_folio(struct folio *, get_block_t *);
bool block_is_partially_uptodate(struct folio *, size_t from, size_t count);
int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
                struct folio **foliop, get_block_t *get_block);
int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,
                get_block_t *get_block);
int block_write_end(struct file *, struct address_space *,
                                loff_t, unsigned len, unsigned copied,
                                struct folio *, void *);
int generic_write_end(struct file *, struct address_space *,
                                loff_t, unsigned len, unsigned copied,
                                struct folio *, void *);
void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to);
int cont_write_begin(struct file *, struct address_space *, loff_t,
                        unsigned, struct folio **, void **,
                        get_block_t *, loff_t *);
int generic_cont_expand_simple(struct inode *inode, loff_t size);
void block_commit_write(struct page *page, unsigned int from, unsigned int to);
int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
                                get_block_t get_block);
sector_t generic_block_bmap(struct address_space *, sector_t, get_block_t *);
int block_truncate_page(struct address_space *, loff_t, get_block_t *);

#ifdef CONFIG_MIGRATION
extern int buffer_migrate_folio(struct address_space *,
                struct folio *dst, struct folio *src, enum migrate_mode);
extern int buffer_migrate_folio_norefs(struct address_space *,
                struct folio *dst, struct folio *src, enum migrate_mode);
#else
#define buffer_migrate_folio NULL
#define buffer_migrate_folio_norefs NULL
#endif

/*
 * inline definitions
 */

static inline void get_bh(struct buffer_head *bh)
{
        atomic_inc(&bh->b_count);
}

static inline void put_bh(struct buffer_head *bh)
{
        smp_mb__before_atomic();
        atomic_dec(&bh->b_count);
}

/**
 * brelse - Release a buffer.
 * @bh: The buffer to release.
 *
 * Decrement a buffer_head's reference count.  If @bh is NULL, this
 * function is a no-op.
 *
 * If all buffers on a folio have zero reference count, are clean
 * and unlocked, and if the folio is unlocked and not under writeback
 * then try_to_free_buffers() may strip the buffers from the folio in
 * preparation for freeing it (sometimes, rarely, buffers are removed
 * from a folio but it ends up not being freed, and buffers may later
 * be reattached).
 *
 * Context: Any context.
 */
static inline void brelse(struct buffer_head *bh)
{
        if (bh)
                __brelse(bh);
}

/**
 * bforget - Discard any dirty data in a buffer.
 * @bh: The buffer to forget.
 *
 * Call this function instead of brelse() if the data written to a buffer
 * no longer needs to be written back.  It will clear the buffer's dirty
 * flag so writeback of this buffer will be skipped.
 *
 * Context: Any context.
 */
static inline void bforget(struct buffer_head *bh)
{
        if (bh)
                __bforget(bh);
}

static inline struct buffer_head *
sb_bread(struct super_block *sb, sector_t block)
{
        return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE);
}

static inline struct buffer_head *
sb_bread_unmovable(struct super_block *sb, sector_t block)
{
        return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
}

static inline void
sb_breadahead(struct super_block *sb, sector_t block)
{
        __breadahead(sb->s_bdev, block, sb->s_blocksize);
}

static inline struct buffer_head *getblk_unmovable(struct block_device *bdev,
                sector_t block, unsigned size)
{
        gfp_t gfp;

        gfp = mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
        gfp |= __GFP_NOFAIL;

        return bdev_getblk(bdev, block, size, gfp);
}

static inline struct buffer_head *__getblk(struct block_device *bdev,
                sector_t block, unsigned size)
{
        gfp_t gfp;

        gfp = mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS);
        gfp |= __GFP_MOVABLE | __GFP_NOFAIL;

        return bdev_getblk(bdev, block, size, gfp);
}

static inline struct buffer_head *sb_getblk(struct super_block *sb,
                sector_t block)
{
        return __getblk(sb->s_bdev, block, sb->s_blocksize);
}

static inline struct buffer_head *sb_getblk_gfp(struct super_block *sb,
                sector_t block, gfp_t gfp)
{
        return bdev_getblk(sb->s_bdev, block, sb->s_blocksize, gfp);
}

static inline struct buffer_head *
sb_find_get_block(struct super_block *sb, sector_t block)
{
        return __find_get_block(sb->s_bdev, block, sb->s_blocksize);
}

static inline void
map_bh(struct buffer_head *bh, struct super_block *sb, sector_t block)
{
        set_buffer_mapped(bh);
        bh->b_bdev = sb->s_bdev;
        bh->b_blocknr = block;
        bh->b_size = sb->s_blocksize;
}

static inline void wait_on_buffer(struct buffer_head *bh)
{
        might_sleep();
        if (buffer_locked(bh))
                __wait_on_buffer(bh);
}

static inline int trylock_buffer(struct buffer_head *bh)
{
        return likely(!test_and_set_bit_lock(BH_Lock, &bh->b_state));
}

static inline void lock_buffer(struct buffer_head *bh)
{
        might_sleep();
        if (!trylock_buffer(bh))
                __lock_buffer(bh);
}

static inline void bh_readahead(struct buffer_head *bh, blk_opf_t op_flags)
{
        if (!buffer_uptodate(bh) && trylock_buffer(bh)) {
                if (!buffer_uptodate(bh))
                        __bh_read(bh, op_flags, false);
                else
                        unlock_buffer(bh);
        }
}

static inline void bh_read_nowait(struct buffer_head *bh, blk_opf_t op_flags)
{
        if (!bh_uptodate_or_lock(bh))
                __bh_read(bh, op_flags, false);
}

/* Returns 1 if buffer uptodated, 0 on success, and -EIO on error. */
static inline int bh_read(struct buffer_head *bh, blk_opf_t op_flags)
{
        if (bh_uptodate_or_lock(bh))
                return 1;
        return __bh_read(bh, op_flags, true);
}

static inline void bh_read_batch(int nr, struct buffer_head *bhs[])
{
        __bh_read_batch(nr, bhs, 0, true);
}

static inline void bh_readahead_batch(int nr, struct buffer_head *bhs[],
                                      blk_opf_t op_flags)
{
        __bh_read_batch(nr, bhs, op_flags, false);
}

/**
 * __bread() - Read a block.
 * @bdev: The block device to read from.
 * @block: Block number in units of block size.
 * @size: The block size of this device in bytes.
 *
 * Read a specified block, and return the buffer head that refers
 * to it.  The memory is allocated from the movable area so that it can
 * be migrated.  The returned buffer head has its refcount increased.
 * The caller should call brelse() when it has finished with the buffer.
 *
 * Context: May sleep waiting for I/O.
 * Return: NULL if the block was unreadable.
 */
static inline struct buffer_head *__bread(struct block_device *bdev,
                sector_t block, unsigned size)
{
        return __bread_gfp(bdev, block, size, __GFP_MOVABLE);
}

/**
 * get_nth_bh - Get a reference on the n'th buffer after this one.
 * @bh: The buffer to start counting from.
 * @count: How many buffers to skip.
 *
 * This is primarily useful for finding the nth buffer in a folio; in
 * that case you pass the head buffer and the byte offset in the folio
 * divided by the block size.  It can be used for other purposes, but
 * it will wrap at the end of the folio rather than returning NULL or
 * proceeding to the next folio for you.
 *
 * Return: The requested buffer with an elevated refcount.
 */
static inline __must_check
struct buffer_head *get_nth_bh(struct buffer_head *bh, unsigned int count)
{
        while (count--)
                bh = bh->b_this_page;
        get_bh(bh);
        return bh;
}

bool block_dirty_folio(struct address_space *mapping, struct folio *folio);

#ifdef CONFIG_BUFFER_HEAD

void buffer_init(void);
bool try_to_free_buffers(struct folio *folio);
int inode_has_buffers(struct inode *inode);
void invalidate_inode_buffers(struct inode *inode);
int remove_inode_buffers(struct inode *inode);
int sync_mapping_buffers(struct address_space *mapping);
void invalidate_bh_lrus(void);
void invalidate_bh_lrus_cpu(void);
bool has_bh_in_lru(int cpu, void *dummy);
extern int buffer_heads_over_limit;

#else /* CONFIG_BUFFER_HEAD */

static inline void buffer_init(void) {}
static inline bool try_to_free_buffers(struct folio *folio) { return true; }
static inline int inode_has_buffers(struct inode *inode) { return 0; }
static inline void invalidate_inode_buffers(struct inode *inode) {}
static inline int remove_inode_buffers(struct inode *inode) { return 1; }
static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
static inline void invalidate_bh_lrus(void) {}
static inline void invalidate_bh_lrus_cpu(void) {}
static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; }
#define buffer_heads_over_limit 0

#endif /* CONFIG_BUFFER_HEAD */
#endif /* _LINUX_BUFFER_HEAD_H */















































































  122 




































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  pm_wakeup.h - Power management wakeup interface
 *
 *  Copyright (C) 2008 Alan Stern
 *  Copyright (C) 2010 Rafael J. Wysocki, Novell Inc.
 */

#ifndef _LINUX_PM_WAKEUP_H
#define _LINUX_PM_WAKEUP_H

#ifndef _DEVICE_H_
# error "please don't include this file directly"
#endif

#include <linux/types.h>

struct wake_irq;

/**
 * struct wakeup_source - Representation of wakeup sources
 *
 * @name: Name of the wakeup source
 * @id: Wakeup source id
 * @entry: Wakeup source list entry
 * @lock: Wakeup source lock
 * @wakeirq: Optional device specific wakeirq
 * @timer: Wakeup timer list
 * @timer_expires: Wakeup timer expiration
 * @total_time: Total time this wakeup source has been active.
 * @max_time: Maximum time this wakeup source has been continuously active.
 * @last_time: Monotonic clock when the wakeup source's was touched last time.
 * @prevent_sleep_time: Total time this source has been preventing autosleep.
 * @event_count: Number of signaled wakeup events.
 * @active_count: Number of times the wakeup source was activated.
 * @relax_count: Number of times the wakeup source was deactivated.
 * @expire_count: Number of times the wakeup source's timeout has expired.
 * @wakeup_count: Number of times the wakeup source might abort suspend.
 * @dev: Struct device for sysfs statistics about the wakeup source.
 * @active: Status of the wakeup source.
 * @autosleep_enabled: Autosleep is active, so update @prevent_sleep_time.
 */
struct wakeup_source {
        const char                 *name;
        int                        id;
        struct list_head        entry;
        spinlock_t                lock;
        struct wake_irq                *wakeirq;
        struct timer_list        timer;
        unsigned long                timer_expires;
        ktime_t total_time;
        ktime_t max_time;
        ktime_t last_time;
        ktime_t start_prevent_time;
        ktime_t prevent_sleep_time;
        unsigned long                event_count;
        unsigned long                active_count;
        unsigned long                relax_count;
        unsigned long                expire_count;
        unsigned long                wakeup_count;
        struct device                *dev;
        bool                        active:1;
        bool                        autosleep_enabled:1;
};

#define for_each_wakeup_source(ws) \
        for ((ws) = wakeup_sources_walk_start();        \
             (ws);                                        \
             (ws) = wakeup_sources_walk_next((ws)))

#ifdef CONFIG_PM_SLEEP

/*
 * Changes to device_may_wakeup take effect on the next pm state change.
 */

static inline bool device_can_wakeup(struct device *dev)
{
        return dev->power.can_wakeup;
}

static inline bool device_may_wakeup(struct device *dev)
{
        return dev->power.can_wakeup && !!dev->power.wakeup;
}

static inline bool device_wakeup_path(struct device *dev)
{
        return dev->power.wakeup_path;
}

static inline void device_set_wakeup_path(struct device *dev)
{
        dev->power.wakeup_path = true;
}

/* drivers/base/power/wakeup.c */
extern struct wakeup_source *wakeup_source_create(const char *name);
extern void wakeup_source_destroy(struct wakeup_source *ws);
extern void wakeup_source_add(struct wakeup_source *ws);
extern void wakeup_source_remove(struct wakeup_source *ws);
extern struct wakeup_source *wakeup_source_register(struct device *dev,
                                                    const char *name);
extern void wakeup_source_unregister(struct wakeup_source *ws);
extern int wakeup_sources_read_lock(void);
extern void wakeup_sources_read_unlock(int idx);
extern struct wakeup_source *wakeup_sources_walk_start(void);
extern struct wakeup_source *wakeup_sources_walk_next(struct wakeup_source *ws);
extern int device_wakeup_enable(struct device *dev);
extern void device_wakeup_disable(struct device *dev);
extern void device_set_wakeup_capable(struct device *dev, bool capable);
extern int device_set_wakeup_enable(struct device *dev, bool enable);
extern void __pm_stay_awake(struct wakeup_source *ws);
extern void pm_stay_awake(struct device *dev);
extern void __pm_relax(struct wakeup_source *ws);
extern void pm_relax(struct device *dev);
extern void pm_wakeup_ws_event(struct wakeup_source *ws, unsigned int msec, bool hard);
extern void pm_wakeup_dev_event(struct device *dev, unsigned int msec, bool hard);

#else /* !CONFIG_PM_SLEEP */

static inline void device_set_wakeup_capable(struct device *dev, bool capable)
{
        dev->power.can_wakeup = capable;
}

static inline bool device_can_wakeup(struct device *dev)
{
        return dev->power.can_wakeup;
}

static inline struct wakeup_source *wakeup_source_create(const char *name)
{
        return NULL;
}

static inline void wakeup_source_destroy(struct wakeup_source *ws) {}

static inline void wakeup_source_add(struct wakeup_source *ws) {}

static inline void wakeup_source_remove(struct wakeup_source *ws) {}

static inline struct wakeup_source *wakeup_source_register(struct device *dev,
                                                           const char *name)
{
        return NULL;
}

static inline void wakeup_source_unregister(struct wakeup_source *ws) {}

static inline int device_wakeup_enable(struct device *dev)
{
        dev->power.should_wakeup = true;
        return 0;
}

static inline void device_wakeup_disable(struct device *dev)
{
        dev->power.should_wakeup = false;
}

static inline int device_set_wakeup_enable(struct device *dev, bool enable)
{
        dev->power.should_wakeup = enable;
        return 0;
}

static inline bool device_may_wakeup(struct device *dev)
{
        return dev->power.can_wakeup && dev->power.should_wakeup;
}

static inline bool device_wakeup_path(struct device *dev)
{
        return false;
}

static inline void device_set_wakeup_path(struct device *dev) {}

static inline void __pm_stay_awake(struct wakeup_source *ws) {}

static inline void pm_stay_awake(struct device *dev) {}

static inline void __pm_relax(struct wakeup_source *ws) {}

static inline void pm_relax(struct device *dev) {}

static inline void pm_wakeup_ws_event(struct wakeup_source *ws,
                                      unsigned int msec, bool hard) {}

static inline void pm_wakeup_dev_event(struct device *dev, unsigned int msec,
                                       bool hard) {}

#endif /* !CONFIG_PM_SLEEP */

static inline bool device_awake_path(struct device *dev)
{
        return device_wakeup_path(dev);
}

static inline void device_set_awake_path(struct device *dev)
{
        device_set_wakeup_path(dev);
}

static inline void __pm_wakeup_event(struct wakeup_source *ws, unsigned int msec)
{
        return pm_wakeup_ws_event(ws, msec, false);
}

static inline void pm_wakeup_event(struct device *dev, unsigned int msec)
{
        return pm_wakeup_dev_event(dev, msec, false);
}

static inline void pm_wakeup_hard_event(struct device *dev)
{
        return pm_wakeup_dev_event(dev, 0, true);
}

/**
 * device_init_wakeup - Device wakeup initialization.
 * @dev: Device to handle.
 * @enable: Whether or not to enable @dev as a wakeup device.
 *
 * By default, most devices should leave wakeup disabled.  The exceptions are
 * devices that everyone expects to be wakeup sources: keyboards, power buttons,
 * possibly network interfaces, etc.  Also, devices that don't generate their
 * own wakeup requests but merely forward requests from one bus to another
 * (like PCI bridges) should have wakeup enabled by default.
 */
static inline int device_init_wakeup(struct device *dev, bool enable)
{
        if (enable) {
                device_set_wakeup_capable(dev, true);
                return device_wakeup_enable(dev);
        }
        device_wakeup_disable(dev);
        device_set_wakeup_capable(dev, false);
        return 0;
}

#endif /* _LINUX_PM_WAKEUP_H */















































































































































































   11 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Network device features.
 */
#ifndef _LINUX_NETDEV_FEATURES_H
#define _LINUX_NETDEV_FEATURES_H

#include <linux/types.h>
#include <linux/bitops.h>
#include <asm/byteorder.h>

typedef u64 netdev_features_t;

enum {
        NETIF_F_SG_BIT,                        /* Scatter/gather IO. */
        NETIF_F_IP_CSUM_BIT,                /* Can checksum TCP/UDP over IPv4. */
        __UNUSED_NETIF_F_1,
        NETIF_F_HW_CSUM_BIT,                /* Can checksum all the packets. */
        NETIF_F_IPV6_CSUM_BIT,                /* Can checksum TCP/UDP over IPV6 */
        NETIF_F_HIGHDMA_BIT,                /* Can DMA to high memory. */
        NETIF_F_FRAGLIST_BIT,                /* Scatter/gather IO. */
        NETIF_F_HW_VLAN_CTAG_TX_BIT,        /* Transmit VLAN CTAG HW acceleration */
        NETIF_F_HW_VLAN_CTAG_RX_BIT,        /* Receive VLAN CTAG HW acceleration */
        NETIF_F_HW_VLAN_CTAG_FILTER_BIT,/* Receive filtering on VLAN CTAGs */
        NETIF_F_VLAN_CHALLENGED_BIT,        /* Device cannot handle VLAN packets */
        NETIF_F_GSO_BIT,                /* Enable software GSO. */
        __UNUSED_NETIF_F_12,
        __UNUSED_NETIF_F_13,
        NETIF_F_GRO_BIT,                /* Generic receive offload */
        NETIF_F_LRO_BIT,                /* large receive offload */

        /**/NETIF_F_GSO_SHIFT,                /* keep the order of SKB_GSO_* bits */
        NETIF_F_TSO_BIT                        /* ... TCPv4 segmentation */
                = NETIF_F_GSO_SHIFT,
        NETIF_F_GSO_ROBUST_BIT,                /* ... ->SKB_GSO_DODGY */
        NETIF_F_TSO_ECN_BIT,                /* ... TCP ECN support */
        NETIF_F_TSO_MANGLEID_BIT,        /* ... IPV4 ID mangling allowed */
        NETIF_F_TSO6_BIT,                /* ... TCPv6 segmentation */
        NETIF_F_FSO_BIT,                /* ... FCoE segmentation */
        NETIF_F_GSO_GRE_BIT,                /* ... GRE with TSO */
        NETIF_F_GSO_GRE_CSUM_BIT,        /* ... GRE with csum with TSO */
        NETIF_F_GSO_IPXIP4_BIT,                /* ... IP4 or IP6 over IP4 with TSO */
        NETIF_F_GSO_IPXIP6_BIT,                /* ... IP4 or IP6 over IP6 with TSO */
        NETIF_F_GSO_UDP_TUNNEL_BIT,        /* ... UDP TUNNEL with TSO */
        NETIF_F_GSO_UDP_TUNNEL_CSUM_BIT,/* ... UDP TUNNEL with TSO & CSUM */
        NETIF_F_GSO_PARTIAL_BIT,        /* ... Only segment inner-most L4
                                         *     in hardware and all other
                                         *     headers in software.
                                         */
        NETIF_F_GSO_TUNNEL_REMCSUM_BIT, /* ... TUNNEL with TSO & REMCSUM */
        NETIF_F_GSO_SCTP_BIT,                /* ... SCTP fragmentation */
        NETIF_F_GSO_ESP_BIT,                /* ... ESP with TSO */
        NETIF_F_GSO_UDP_BIT,                /* ... UFO, deprecated except tuntap */
        NETIF_F_GSO_UDP_L4_BIT,                /* ... UDP payload GSO (not UFO) */
        NETIF_F_GSO_FRAGLIST_BIT,                /* ... Fraglist GSO */
        /**/NETIF_F_GSO_LAST =                /* last bit, see GSO_MASK */
                NETIF_F_GSO_FRAGLIST_BIT,

        NETIF_F_FCOE_CRC_BIT,                /* FCoE CRC32 */
        NETIF_F_SCTP_CRC_BIT,                /* SCTP checksum offload */
        __UNUSED_NETIF_F_37,
        NETIF_F_NTUPLE_BIT,                /* N-tuple filters supported */
        NETIF_F_RXHASH_BIT,                /* Receive hashing offload */
        NETIF_F_RXCSUM_BIT,                /* Receive checksumming offload */
        NETIF_F_NOCACHE_COPY_BIT,        /* Use no-cache copyfromuser */
        NETIF_F_LOOPBACK_BIT,                /* Enable loopback */
        NETIF_F_RXFCS_BIT,                /* Append FCS to skb pkt data */
        NETIF_F_RXALL_BIT,                /* Receive errored frames too */
        NETIF_F_HW_VLAN_STAG_TX_BIT,        /* Transmit VLAN STAG HW acceleration */
        NETIF_F_HW_VLAN_STAG_RX_BIT,        /* Receive VLAN STAG HW acceleration */
        NETIF_F_HW_VLAN_STAG_FILTER_BIT,/* Receive filtering on VLAN STAGs */
        NETIF_F_HW_L2FW_DOFFLOAD_BIT,        /* Allow L2 Forwarding in Hardware */

        NETIF_F_HW_TC_BIT,                /* Offload TC infrastructure */
        NETIF_F_HW_ESP_BIT,                /* Hardware ESP transformation offload */
        NETIF_F_HW_ESP_TX_CSUM_BIT,        /* ESP with TX checksum offload */
        NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
        NETIF_F_HW_TLS_TX_BIT,                /* Hardware TLS TX offload */
        NETIF_F_HW_TLS_RX_BIT,                /* Hardware TLS RX offload */

        NETIF_F_GRO_HW_BIT,                /* Hardware Generic receive offload */
        NETIF_F_HW_TLS_RECORD_BIT,        /* Offload TLS record */
        NETIF_F_GRO_FRAGLIST_BIT,        /* Fraglist GRO */

        NETIF_F_HW_MACSEC_BIT,                /* Offload MACsec operations */
        NETIF_F_GRO_UDP_FWD_BIT,        /* Allow UDP GRO for forwarding */

        NETIF_F_HW_HSR_TAG_INS_BIT,        /* Offload HSR tag insertion */
        NETIF_F_HW_HSR_TAG_RM_BIT,        /* Offload HSR tag removal */
        NETIF_F_HW_HSR_FWD_BIT,                /* Offload HSR forwarding */
        NETIF_F_HW_HSR_DUP_BIT,                /* Offload HSR duplication */

        /*
         * Add your fresh new feature above and remember to update
         * netdev_features_strings[] in net/ethtool/common.c and maybe
         * some feature mask #defines below. Please also describe it
         * in Documentation/networking/netdev-features.rst.
         */

        /**/NETDEV_FEATURE_COUNT
};

/* copy'n'paste compression ;) */
#define __NETIF_F_BIT(bit)        ((netdev_features_t)1 << (bit))
#define __NETIF_F(name)                __NETIF_F_BIT(NETIF_F_##name##_BIT)

#define NETIF_F_FCOE_CRC        __NETIF_F(FCOE_CRC)
#define NETIF_F_FRAGLIST        __NETIF_F(FRAGLIST)
#define NETIF_F_FSO                __NETIF_F(FSO)
#define NETIF_F_GRO                __NETIF_F(GRO)
#define NETIF_F_GRO_HW                __NETIF_F(GRO_HW)
#define NETIF_F_GSO                __NETIF_F(GSO)
#define NETIF_F_GSO_ROBUST        __NETIF_F(GSO_ROBUST)
#define NETIF_F_HIGHDMA                __NETIF_F(HIGHDMA)
#define NETIF_F_HW_CSUM                __NETIF_F(HW_CSUM)
#define NETIF_F_HW_VLAN_CTAG_FILTER __NETIF_F(HW_VLAN_CTAG_FILTER)
#define NETIF_F_HW_VLAN_CTAG_RX        __NETIF_F(HW_VLAN_CTAG_RX)
#define NETIF_F_HW_VLAN_CTAG_TX        __NETIF_F(HW_VLAN_CTAG_TX)
#define NETIF_F_IP_CSUM                __NETIF_F(IP_CSUM)
#define NETIF_F_IPV6_CSUM        __NETIF_F(IPV6_CSUM)
#define NETIF_F_LOOPBACK        __NETIF_F(LOOPBACK)
#define NETIF_F_LRO                __NETIF_F(LRO)
#define NETIF_F_NOCACHE_COPY        __NETIF_F(NOCACHE_COPY)
#define NETIF_F_NTUPLE                __NETIF_F(NTUPLE)
#define NETIF_F_RXCSUM                __NETIF_F(RXCSUM)
#define NETIF_F_RXHASH                __NETIF_F(RXHASH)
#define NETIF_F_SCTP_CRC        __NETIF_F(SCTP_CRC)
#define NETIF_F_SG                __NETIF_F(SG)
#define NETIF_F_TSO6                __NETIF_F(TSO6)
#define NETIF_F_TSO_ECN                __NETIF_F(TSO_ECN)
#define NETIF_F_TSO                __NETIF_F(TSO)
#define NETIF_F_VLAN_CHALLENGED        __NETIF_F(VLAN_CHALLENGED)
#define NETIF_F_RXFCS                __NETIF_F(RXFCS)
#define NETIF_F_RXALL                __NETIF_F(RXALL)
#define NETIF_F_GSO_GRE                __NETIF_F(GSO_GRE)
#define NETIF_F_GSO_GRE_CSUM        __NETIF_F(GSO_GRE_CSUM)
#define NETIF_F_GSO_IPXIP4        __NETIF_F(GSO_IPXIP4)
#define NETIF_F_GSO_IPXIP6        __NETIF_F(GSO_IPXIP6)
#define NETIF_F_GSO_UDP_TUNNEL        __NETIF_F(GSO_UDP_TUNNEL)
#define NETIF_F_GSO_UDP_TUNNEL_CSUM __NETIF_F(GSO_UDP_TUNNEL_CSUM)
#define NETIF_F_TSO_MANGLEID        __NETIF_F(TSO_MANGLEID)
#define NETIF_F_GSO_PARTIAL         __NETIF_F(GSO_PARTIAL)
#define NETIF_F_GSO_TUNNEL_REMCSUM __NETIF_F(GSO_TUNNEL_REMCSUM)
#define NETIF_F_GSO_SCTP        __NETIF_F(GSO_SCTP)
#define NETIF_F_GSO_ESP                __NETIF_F(GSO_ESP)
#define NETIF_F_GSO_UDP                __NETIF_F(GSO_UDP)
#define NETIF_F_HW_VLAN_STAG_FILTER __NETIF_F(HW_VLAN_STAG_FILTER)
#define NETIF_F_HW_VLAN_STAG_RX        __NETIF_F(HW_VLAN_STAG_RX)
#define NETIF_F_HW_VLAN_STAG_TX        __NETIF_F(HW_VLAN_STAG_TX)
#define NETIF_F_HW_L2FW_DOFFLOAD        __NETIF_F(HW_L2FW_DOFFLOAD)
#define NETIF_F_HW_TC                __NETIF_F(HW_TC)
#define NETIF_F_HW_ESP                __NETIF_F(HW_ESP)
#define NETIF_F_HW_ESP_TX_CSUM        __NETIF_F(HW_ESP_TX_CSUM)
#define        NETIF_F_RX_UDP_TUNNEL_PORT  __NETIF_F(RX_UDP_TUNNEL_PORT)
#define NETIF_F_HW_TLS_RECORD        __NETIF_F(HW_TLS_RECORD)
#define NETIF_F_GSO_UDP_L4        __NETIF_F(GSO_UDP_L4)
#define NETIF_F_HW_TLS_TX        __NETIF_F(HW_TLS_TX)
#define NETIF_F_HW_TLS_RX        __NETIF_F(HW_TLS_RX)
#define NETIF_F_GRO_FRAGLIST        __NETIF_F(GRO_FRAGLIST)
#define NETIF_F_GSO_FRAGLIST        __NETIF_F(GSO_FRAGLIST)
#define NETIF_F_HW_MACSEC        __NETIF_F(HW_MACSEC)
#define NETIF_F_GRO_UDP_FWD        __NETIF_F(GRO_UDP_FWD)
#define NETIF_F_HW_HSR_TAG_INS        __NETIF_F(HW_HSR_TAG_INS)
#define NETIF_F_HW_HSR_TAG_RM        __NETIF_F(HW_HSR_TAG_RM)
#define NETIF_F_HW_HSR_FWD        __NETIF_F(HW_HSR_FWD)
#define NETIF_F_HW_HSR_DUP        __NETIF_F(HW_HSR_DUP)

/* Finds the next feature with the highest number of the range of start-1 till 0.
 */
static inline int find_next_netdev_feature(u64 feature, unsigned long start)
{
        /* like BITMAP_LAST_WORD_MASK() for u64
         * this sets the most significant 64 - start to 0.
         */
        feature &= ~0ULL >> (-start & ((sizeof(feature) * 8) - 1));

        return fls64(feature) - 1;
}

/* This goes for the MSB to the LSB through the set feature bits,
 * mask_addr should be a u64 and bit an int
 */
#define for_each_netdev_feature(mask_addr, bit)                                \
        for ((bit) = find_next_netdev_feature((mask_addr),                \
                                              NETDEV_FEATURE_COUNT);        \
             (bit) >= 0;                                                \
             (bit) = find_next_netdev_feature((mask_addr), (bit)))

/* Features valid for ethtool to change */
/* = all defined minus driver/device-class-related */
#define NETIF_F_NEVER_CHANGE        NETIF_F_VLAN_CHALLENGED

/* remember that ((t)1 << t_BITS) is undefined in C99 */
#define NETIF_F_ETHTOOL_BITS        ((__NETIF_F_BIT(NETDEV_FEATURE_COUNT - 1) | \
                (__NETIF_F_BIT(NETDEV_FEATURE_COUNT - 1) - 1)) & \
                ~NETIF_F_NEVER_CHANGE)

/* Segmentation offload feature mask */
#define NETIF_F_GSO_MASK        (__NETIF_F_BIT(NETIF_F_GSO_LAST + 1) - \
                __NETIF_F_BIT(NETIF_F_GSO_SHIFT))

/* List of IP checksum features. Note that NETIF_F_HW_CSUM should not be
 * set in features when NETIF_F_IP_CSUM or NETIF_F_IPV6_CSUM are set--
 * this would be contradictory
 */
#define NETIF_F_CSUM_MASK        (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | \
                                 NETIF_F_HW_CSUM)

#define NETIF_F_ALL_TSO         (NETIF_F_TSO | NETIF_F_TSO6 | \
                                 NETIF_F_TSO_ECN | NETIF_F_TSO_MANGLEID)

/* List of features with software fallbacks. */
#define NETIF_F_GSO_SOFTWARE        (NETIF_F_ALL_TSO | NETIF_F_GSO_SCTP |             \
                                 NETIF_F_GSO_UDP_L4 | NETIF_F_GSO_FRAGLIST)

/*
 * If one device supports one of these features, then enable them
 * for all in netdev_increment_features.
 */
#define NETIF_F_ONE_FOR_ALL        (NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ROBUST | \
                                 NETIF_F_SG | NETIF_F_HIGHDMA |                \
                                 NETIF_F_FRAGLIST | NETIF_F_VLAN_CHALLENGED)

/*
 * If one device doesn't support one of these features, then disable it
 * for all in netdev_increment_features.
 */
#define NETIF_F_ALL_FOR_ALL        (NETIF_F_NOCACHE_COPY | NETIF_F_FSO)

/*
 * If upper/master device has these features disabled, they must be disabled
 * on all lower/slave devices as well.
 */
#define NETIF_F_UPPER_DISABLES        NETIF_F_LRO

/* changeable features with no special hardware requirements */
#define NETIF_F_SOFT_FEATURES        (NETIF_F_GSO | NETIF_F_GRO)

/* Changeable features with no special hardware requirements that defaults to off. */
#define NETIF_F_SOFT_FEATURES_OFF        (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD)

#define NETIF_F_VLAN_FEATURES        (NETIF_F_HW_VLAN_CTAG_FILTER | \
                                 NETIF_F_HW_VLAN_CTAG_RX | \
                                 NETIF_F_HW_VLAN_CTAG_TX | \
                                 NETIF_F_HW_VLAN_STAG_FILTER | \
                                 NETIF_F_HW_VLAN_STAG_RX | \
                                 NETIF_F_HW_VLAN_STAG_TX)

#define NETIF_F_GSO_ENCAP_ALL        (NETIF_F_GSO_GRE |                        \
                                 NETIF_F_GSO_GRE_CSUM |                        \
                                 NETIF_F_GSO_IPXIP4 |                        \
                                 NETIF_F_GSO_IPXIP6 |                        \
                                 NETIF_F_GSO_UDP_TUNNEL |                \
                                 NETIF_F_GSO_UDP_TUNNEL_CSUM)

#endif        /* _LINUX_NETDEV_FEATURES_H */































































































  256 























   48 


    2 



   48 







    8 


    9 























  133 





























  134 
   47 

















  134 



  133 






  133 






  134 





  101 

  134 










    7 






  127 
    9 

  127 
   11 


























    8 





    8 
    8 




    1 






    1 






    3 









    1 

    1 
    1 



    1 











































    1 




    1 



























































   31 














  384 




    4 




  381 

  362 






   31 











  384 





































































































































    6 


    7 


    7 


    6 


    7 


    7 




    7 

    7 



    7 




    6 
    7 
    3 

    4 

    7 
    2 

    5 


    7 







    7 
    7 
    7 





    7 


















































































































































































































































































































































































   13 

    4 

    1 





   10 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/prctl.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sched/idle.h>
#include <linux/sched/debug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/init.h>
#include <linux/export.h>
#include <linux/pm.h>
#include <linux/tick.h>
#include <linux/random.h>
#include <linux/user-return-notifier.h>
#include <linux/dmi.h>
#include <linux/utsname.h>
#include <linux/stackprotector.h>
#include <linux/cpuidle.h>
#include <linux/acpi.h>
#include <linux/elf-randomize.h>
#include <linux/static_call.h>
#include <trace/events/power.h>
#include <linux/hw_breakpoint.h>
#include <linux/entry-common.h>
#include <asm/cpu.h>
#include <asm/apic.h>
#include <linux/uaccess.h>
#include <asm/mwait.h>
#include <asm/fpu/api.h>
#include <asm/fpu/sched.h>
#include <asm/fpu/xstate.h>
#include <asm/debugreg.h>
#include <asm/nmi.h>
#include <asm/tlbflush.h>
#include <asm/mce.h>
#include <asm/vm86.h>
#include <asm/switch_to.h>
#include <asm/desc.h>
#include <asm/prctl.h>
#include <asm/spec-ctrl.h>
#include <asm/io_bitmap.h>
#include <asm/proto.h>
#include <asm/frame.h>
#include <asm/unwind.h>
#include <asm/tdx.h>
#include <asm/mmu_context.h>
#include <asm/shstk.h>

#include "process.h"

/*
 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
 * no more per-task TSS's. The TSS size is kept cacheline-aligned
 * so they are allowed to end up in the .data..cacheline_aligned
 * section. Since TSS's are completely CPU-local, we want them
 * on exact cacheline boundaries, to eliminate cacheline ping-pong.
 */
__visible DEFINE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw) = {
        .x86_tss = {
                /*
                 * .sp0 is only used when entering ring 0 from a lower
                 * privilege level.  Since the init task never runs anything
                 * but ring 0 code, there is no need for a valid value here.
                 * Poison it.
                 */
                .sp0 = (1UL << (BITS_PER_LONG-1)) + 1,

#ifdef CONFIG_X86_32
                .sp1 = TOP_OF_INIT_STACK,

                .ss0 = __KERNEL_DS,
                .ss1 = __KERNEL_CS,
#endif
                .io_bitmap_base        = IO_BITMAP_OFFSET_INVALID,
         },
};
EXPORT_PER_CPU_SYMBOL(cpu_tss_rw);

DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);

/*
 * this gets called so that we can store lazy state into memory and copy the
 * current task into the new thread.
 */
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
        memcpy(dst, src, arch_task_struct_size);
#ifdef CONFIG_VM86
        dst->thread.vm86 = NULL;
#endif
        /* Drop the copied pointer to current's fpstate */
        dst->thread.fpu.fpstate = NULL;

        return 0;
}

#ifdef CONFIG_X86_64
void arch_release_task_struct(struct task_struct *tsk)
{
        if (fpu_state_size_dynamic())
                fpstate_free(&tsk->thread.fpu);
}
#endif

/*
 * Free thread data structures etc..
 */
void exit_thread(struct task_struct *tsk)
{
        struct thread_struct *t = &tsk->thread;
        struct fpu *fpu = &t->fpu;

        if (test_thread_flag(TIF_IO_BITMAP))
                io_bitmap_exit(tsk);

        free_vm86(t);

        shstk_free(tsk);
        fpu__drop(fpu);
}

static int set_new_tls(struct task_struct *p, unsigned long tls)
{
        struct user_desc __user *utls = (struct user_desc __user *)tls;

        if (in_ia32_syscall())
                return do_set_thread_area(p, -1, utls, 0);
        else
                return do_set_thread_area_64(p, ARCH_SET_FS, tls);
}

__visible void ret_from_fork(struct task_struct *prev, struct pt_regs *regs,
                                     int (*fn)(void *), void *fn_arg)
{
        schedule_tail(prev);

        /* Is this a kernel thread? */
        if (unlikely(fn)) {
                fn(fn_arg);
                /*
                 * A kernel thread is allowed to return here after successfully
                 * calling kernel_execve().  Exit to userspace to complete the
                 * execve() syscall.
                 */
                regs->ax = 0;
        }

        syscall_exit_to_user_mode(regs);
}

int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
{
        unsigned long clone_flags = args->flags;
        unsigned long sp = args->stack;
        unsigned long tls = args->tls;
        struct inactive_task_frame *frame;
        struct fork_frame *fork_frame;
        struct pt_regs *childregs;
        unsigned long new_ssp;
        int ret = 0;

        childregs = task_pt_regs(p);
        fork_frame = container_of(childregs, struct fork_frame, regs);
        frame = &fork_frame->frame;

        frame->bp = encode_frame_pointer(childregs);
        frame->ret_addr = (unsigned long) ret_from_fork_asm;
        p->thread.sp = (unsigned long) fork_frame;
        p->thread.io_bitmap = NULL;
        p->thread.iopl_warn = 0;
        memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));

#ifdef CONFIG_X86_64
        current_save_fsgs();
        p->thread.fsindex = current->thread.fsindex;
        p->thread.fsbase = current->thread.fsbase;
        p->thread.gsindex = current->thread.gsindex;
        p->thread.gsbase = current->thread.gsbase;

        savesegment(es, p->thread.es);
        savesegment(ds, p->thread.ds);

        if (p->mm && (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM)
                set_bit(MM_CONTEXT_LOCK_LAM, &p->mm->context.flags);
#else
        p->thread.sp0 = (unsigned long) (childregs + 1);
        savesegment(gs, p->thread.gs);
        /*
         * Clear all status flags including IF and set fixed bit. 64bit
         * does not have this initialization as the frame does not contain
         * flags. The flags consistency (especially vs. AC) is there
         * ensured via objtool, which lacks 32bit support.
         */
        frame->flags = X86_EFLAGS_FIXED;
#endif

        /*
         * Allocate a new shadow stack for thread if needed. If shadow stack,
         * is disabled, new_ssp will remain 0, and fpu_clone() will know not to
         * update it.
         */
        new_ssp = shstk_alloc_thread_stack(p, clone_flags, args->stack_size);
        if (IS_ERR_VALUE(new_ssp))
                return PTR_ERR((void *)new_ssp);

        fpu_clone(p, clone_flags, args->fn, new_ssp);

        /* Kernel thread ? */
        if (unlikely(p->flags & PF_KTHREAD)) {
                p->thread.pkru = pkru_get_init_value();
                memset(childregs, 0, sizeof(struct pt_regs));
                kthread_frame_init(frame, args->fn, args->fn_arg);
                return 0;
        }

        /*
         * Clone current's PKRU value from hardware. tsk->thread.pkru
         * is only valid when scheduled out.
         */
        p->thread.pkru = read_pkru();

        frame->bx = 0;
        *childregs = *current_pt_regs();
        childregs->ax = 0;
        if (sp)
                childregs->sp = sp;

        if (unlikely(args->fn)) {
                /*
                 * A user space thread, but it doesn't return to
                 * ret_after_fork().
                 *
                 * In order to indicate that to tools like gdb,
                 * we reset the stack and instruction pointers.
                 *
                 * It does the same kernel frame setup to return to a kernel
                 * function that a kernel thread does.
                 */
                childregs->sp = 0;
                childregs->ip = 0;
                kthread_frame_init(frame, args->fn, args->fn_arg);
                return 0;
        }

        /* Set a new TLS for the child thread? */
        if (clone_flags & CLONE_SETTLS)
                ret = set_new_tls(p, tls);

        if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))
                io_bitmap_share(p);

        return ret;
}

static void pkru_flush_thread(void)
{
        /*
         * If PKRU is enabled the default PKRU value has to be loaded into
         * the hardware right here (similar to context switch).
         */
        pkru_write_default();
}

void flush_thread(void)
{
        struct task_struct *tsk = current;

        flush_ptrace_hw_breakpoint(tsk);
        memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));

        fpu_flush_thread();
        pkru_flush_thread();
}

void disable_TSC(void)
{
        preempt_disable();
        if (!test_and_set_thread_flag(TIF_NOTSC))
                /*
                 * Must flip the CPU state synchronously with
                 * TIF_NOTSC in the current running context.
                 */
                cr4_set_bits(X86_CR4_TSD);
        preempt_enable();
}

static void enable_TSC(void)
{
        preempt_disable();
        if (test_and_clear_thread_flag(TIF_NOTSC))
                /*
                 * Must flip the CPU state synchronously with
                 * TIF_NOTSC in the current running context.
                 */
                cr4_clear_bits(X86_CR4_TSD);
        preempt_enable();
}

int get_tsc_mode(unsigned long adr)
{
        unsigned int val;

        if (test_thread_flag(TIF_NOTSC))
                val = PR_TSC_SIGSEGV;
        else
                val = PR_TSC_ENABLE;

        return put_user(val, (unsigned int __user *)adr);
}

int set_tsc_mode(unsigned int val)
{
        if (val == PR_TSC_SIGSEGV)
                disable_TSC();
        else if (val == PR_TSC_ENABLE)
                enable_TSC();
        else
                return -EINVAL;

        return 0;
}

DEFINE_PER_CPU(u64, msr_misc_features_shadow);

static void set_cpuid_faulting(bool on)
{
        u64 msrval;

        msrval = this_cpu_read(msr_misc_features_shadow);
        msrval &= ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT;
        msrval |= (on << MSR_MISC_FEATURES_ENABLES_CPUID_FAULT_BIT);
        this_cpu_write(msr_misc_features_shadow, msrval);
        wrmsrl(MSR_MISC_FEATURES_ENABLES, msrval);
}

static void disable_cpuid(void)
{
        preempt_disable();
        if (!test_and_set_thread_flag(TIF_NOCPUID)) {
                /*
                 * Must flip the CPU state synchronously with
                 * TIF_NOCPUID in the current running context.
                 */
                set_cpuid_faulting(true);
        }
        preempt_enable();
}

static void enable_cpuid(void)
{
        preempt_disable();
        if (test_and_clear_thread_flag(TIF_NOCPUID)) {
                /*
                 * Must flip the CPU state synchronously with
                 * TIF_NOCPUID in the current running context.
                 */
                set_cpuid_faulting(false);
        }
        preempt_enable();
}

static int get_cpuid_mode(void)
{
        return !test_thread_flag(TIF_NOCPUID);
}

static int set_cpuid_mode(unsigned long cpuid_enabled)
{
        if (!boot_cpu_has(X86_FEATURE_CPUID_FAULT))
                return -ENODEV;

        if (cpuid_enabled)
                enable_cpuid();
        else
                disable_cpuid();

        return 0;
}

/*
 * Called immediately after a successful exec.
 */
void arch_setup_new_exec(void)
{
        /* If cpuid was previously disabled for this task, re-enable it. */
        if (test_thread_flag(TIF_NOCPUID))
                enable_cpuid();

        /*
         * Don't inherit TIF_SSBD across exec boundary when
         * PR_SPEC_DISABLE_NOEXEC is used.
         */
        if (test_thread_flag(TIF_SSBD) &&
            task_spec_ssb_noexec(current)) {
                clear_thread_flag(TIF_SSBD);
                task_clear_spec_ssb_disable(current);
                task_clear_spec_ssb_noexec(current);
                speculation_ctrl_update(read_thread_flags());
        }

        mm_reset_untag_mask(current->mm);
}

#ifdef CONFIG_X86_IOPL_IOPERM
static inline void switch_to_bitmap(unsigned long tifp)
{
        /*
         * Invalidate I/O bitmap if the previous task used it. This prevents
         * any possible leakage of an active I/O bitmap.
         *
         * If the next task has an I/O bitmap it will handle it on exit to
         * user mode.
         */
        if (tifp & _TIF_IO_BITMAP)
                tss_invalidate_io_bitmap();
}

static void tss_copy_io_bitmap(struct tss_struct *tss, struct io_bitmap *iobm)
{
        /*
         * Copy at least the byte range of the incoming tasks bitmap which
         * covers the permitted I/O ports.
         *
         * If the previous task which used an I/O bitmap had more bits
         * permitted, then the copy needs to cover those as well so they
         * get turned off.
         */
        memcpy(tss->io_bitmap.bitmap, iobm->bitmap,
               max(tss->io_bitmap.prev_max, iobm->max));

        /*
         * Store the new max and the sequence number of this bitmap
         * and a pointer to the bitmap itself.
         */
        tss->io_bitmap.prev_max = iobm->max;
        tss->io_bitmap.prev_sequence = iobm->sequence;
}

/**
 * native_tss_update_io_bitmap - Update I/O bitmap before exiting to user mode
 */
void native_tss_update_io_bitmap(void)
{
        struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
        struct thread_struct *t = &current->thread;
        u16 *base = &tss->x86_tss.io_bitmap_base;

        if (!test_thread_flag(TIF_IO_BITMAP)) {
                native_tss_invalidate_io_bitmap();
                return;
        }

        if (IS_ENABLED(CONFIG_X86_IOPL_IOPERM) && t->iopl_emul == 3) {
                *base = IO_BITMAP_OFFSET_VALID_ALL;
        } else {
                struct io_bitmap *iobm = t->io_bitmap;

                /*
                 * Only copy bitmap data when the sequence number differs. The
                 * update time is accounted to the incoming task.
                 */
                if (tss->io_bitmap.prev_sequence != iobm->sequence)
                        tss_copy_io_bitmap(tss, iobm);

                /* Enable the bitmap */
                *base = IO_BITMAP_OFFSET_VALID_MAP;
        }

        /*
         * Make sure that the TSS limit is covering the IO bitmap. It might have
         * been cut down by a VMEXIT to 0x67 which would cause a subsequent I/O
         * access from user space to trigger a #GP because the bitmap is outside
         * the TSS limit.
         */
        refresh_tss_limit();
}
#else /* CONFIG_X86_IOPL_IOPERM */
static inline void switch_to_bitmap(unsigned long tifp) { }
#endif

#ifdef CONFIG_SMP

struct ssb_state {
        struct ssb_state        *shared_state;
        raw_spinlock_t                lock;
        unsigned int                disable_state;
        unsigned long                local_state;
};

#define LSTATE_SSB        0

static DEFINE_PER_CPU(struct ssb_state, ssb_state);

void speculative_store_bypass_ht_init(void)
{
        struct ssb_state *st = this_cpu_ptr(&ssb_state);
        unsigned int this_cpu = smp_processor_id();
        unsigned int cpu;

        st->local_state = 0;

        /*
         * Shared state setup happens once on the first bringup
         * of the CPU. It's not destroyed on CPU hotunplug.
         */
        if (st->shared_state)
                return;

        raw_spin_lock_init(&st->lock);

        /*
         * Go over HT siblings and check whether one of them has set up the
         * shared state pointer already.
         */
        for_each_cpu(cpu, topology_sibling_cpumask(this_cpu)) {
                if (cpu == this_cpu)
                        continue;

                if (!per_cpu(ssb_state, cpu).shared_state)
                        continue;

                /* Link it to the state of the sibling: */
                st->shared_state = per_cpu(ssb_state, cpu).shared_state;
                return;
        }

        /*
         * First HT sibling to come up on the core.  Link shared state of
         * the first HT sibling to itself. The siblings on the same core
         * which come up later will see the shared state pointer and link
         * themselves to the state of this CPU.
         */
        st->shared_state = st;
}

/*
 * Logic is: First HT sibling enables SSBD for both siblings in the core
 * and last sibling to disable it, disables it for the whole core. This how
 * MSR_SPEC_CTRL works in "hardware":
 *
 *  CORE_SPEC_CTRL = THREAD0_SPEC_CTRL | THREAD1_SPEC_CTRL
 */
static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
{
        struct ssb_state *st = this_cpu_ptr(&ssb_state);
        u64 msr = x86_amd_ls_cfg_base;

        if (!static_cpu_has(X86_FEATURE_ZEN)) {
                msr |= ssbd_tif_to_amd_ls_cfg(tifn);
                wrmsrl(MSR_AMD64_LS_CFG, msr);
                return;
        }

        if (tifn & _TIF_SSBD) {
                /*
                 * Since this can race with prctl(), block reentry on the
                 * same CPU.
                 */
                if (__test_and_set_bit(LSTATE_SSB, &st->local_state))
                        return;

                msr |= x86_amd_ls_cfg_ssbd_mask;

                raw_spin_lock(&st->shared_state->lock);
                /* First sibling enables SSBD: */
                if (!st->shared_state->disable_state)
                        wrmsrl(MSR_AMD64_LS_CFG, msr);
                st->shared_state->disable_state++;
                raw_spin_unlock(&st->shared_state->lock);
        } else {
                if (!__test_and_clear_bit(LSTATE_SSB, &st->local_state))
                        return;

                raw_spin_lock(&st->shared_state->lock);
                st->shared_state->disable_state--;
                if (!st->shared_state->disable_state)
                        wrmsrl(MSR_AMD64_LS_CFG, msr);
                raw_spin_unlock(&st->shared_state->lock);
        }
}
#else
static __always_inline void amd_set_core_ssb_state(unsigned long tifn)
{
        u64 msr = x86_amd_ls_cfg_base | ssbd_tif_to_amd_ls_cfg(tifn);

        wrmsrl(MSR_AMD64_LS_CFG, msr);
}
#endif

static __always_inline void amd_set_ssb_virt_state(unsigned long tifn)
{
        /*
         * SSBD has the same definition in SPEC_CTRL and VIRT_SPEC_CTRL,
         * so ssbd_tif_to_spec_ctrl() just works.
         */
        wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn));
}

/*
 * Update the MSRs managing speculation control, during context switch.
 *
 * tifp: Previous task's thread flags
 * tifn: Next task's thread flags
 */
static __always_inline void __speculation_ctrl_update(unsigned long tifp,
                                                      unsigned long tifn)
{
        unsigned long tif_diff = tifp ^ tifn;
        u64 msr = x86_spec_ctrl_base;
        bool updmsr = false;

        lockdep_assert_irqs_disabled();

        /* Handle change of TIF_SSBD depending on the mitigation method. */
        if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) {
                if (tif_diff & _TIF_SSBD)
                        amd_set_ssb_virt_state(tifn);
        } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) {
                if (tif_diff & _TIF_SSBD)
                        amd_set_core_ssb_state(tifn);
        } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
                   static_cpu_has(X86_FEATURE_AMD_SSBD)) {
                updmsr |= !!(tif_diff & _TIF_SSBD);
                msr |= ssbd_tif_to_spec_ctrl(tifn);
        }

        /* Only evaluate TIF_SPEC_IB if conditional STIBP is enabled. */
        if (IS_ENABLED(CONFIG_SMP) &&
            static_branch_unlikely(&switch_to_cond_stibp)) {
                updmsr |= !!(tif_diff & _TIF_SPEC_IB);
                msr |= stibp_tif_to_spec_ctrl(tifn);
        }

        if (updmsr)
                update_spec_ctrl_cond(msr);
}

static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk)
{
        if (test_and_clear_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE)) {
                if (task_spec_ssb_disable(tsk))
                        set_tsk_thread_flag(tsk, TIF_SSBD);
                else
                        clear_tsk_thread_flag(tsk, TIF_SSBD);

                if (task_spec_ib_disable(tsk))
                        set_tsk_thread_flag(tsk, TIF_SPEC_IB);
                else
                        clear_tsk_thread_flag(tsk, TIF_SPEC_IB);
        }
        /* Return the updated threadinfo flags*/
        return read_task_thread_flags(tsk);
}

void speculation_ctrl_update(unsigned long tif)
{
        unsigned long flags;

        /* Forced update. Make sure all relevant TIF flags are different */
        local_irq_save(flags);
        __speculation_ctrl_update(~tif, tif);
        local_irq_restore(flags);
}

/* Called from seccomp/prctl update */
void speculation_ctrl_update_current(void)
{
        preempt_disable();
        speculation_ctrl_update(speculation_ctrl_update_tif(current));
        preempt_enable();
}

static inline void cr4_toggle_bits_irqsoff(unsigned long mask)
{
        unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);

        newval = cr4 ^ mask;
        if (newval != cr4) {
                this_cpu_write(cpu_tlbstate.cr4, newval);
                __write_cr4(newval);
        }
}

void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p)
{
        unsigned long tifp, tifn;

        tifn = read_task_thread_flags(next_p);
        tifp = read_task_thread_flags(prev_p);

        switch_to_bitmap(tifp);

        propagate_user_return_notify(prev_p, next_p);

        if ((tifp & _TIF_BLOCKSTEP || tifn & _TIF_BLOCKSTEP) &&
            arch_has_block_step()) {
                unsigned long debugctl, msk;

                rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
                debugctl &= ~DEBUGCTLMSR_BTF;
                msk = tifn & _TIF_BLOCKSTEP;
                debugctl |= (msk >> TIF_BLOCKSTEP) << DEBUGCTLMSR_BTF_SHIFT;
                wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
        }

        if ((tifp ^ tifn) & _TIF_NOTSC)
                cr4_toggle_bits_irqsoff(X86_CR4_TSD);

        if ((tifp ^ tifn) & _TIF_NOCPUID)
                set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));

        if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) {
                __speculation_ctrl_update(tifp, tifn);
        } else {
                speculation_ctrl_update_tif(prev_p);
                tifn = speculation_ctrl_update_tif(next_p);

                /* Enforce MSR update to ensure consistent state */
                __speculation_ctrl_update(~tifn, tifn);
        }
}

/*
 * Idle related variables and functions
 */
unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
EXPORT_SYMBOL(boot_option_idle_override);

/*
 * We use this if we don't have any better idle routine..
 */
void __cpuidle default_idle(void)
{
        raw_safe_halt();
        raw_local_irq_disable();
}
#if defined(CONFIG_APM_MODULE) || defined(CONFIG_HALTPOLL_CPUIDLE_MODULE)
EXPORT_SYMBOL(default_idle);
#endif

DEFINE_STATIC_CALL_NULL(x86_idle, default_idle);

static bool x86_idle_set(void)
{
        return !!static_call_query(x86_idle);
}

#ifndef CONFIG_SMP
static inline void __noreturn play_dead(void)
{
        BUG();
}
#endif

void arch_cpu_idle_enter(void)
{
        tsc_verify_tsc_adjust(false);
        local_touch_nmi();
}

void __noreturn arch_cpu_idle_dead(void)
{
        play_dead();
}

/*
 * Called from the generic idle code.
 */
void __cpuidle arch_cpu_idle(void)
{
        static_call(x86_idle)();
}
EXPORT_SYMBOL_GPL(arch_cpu_idle);

#ifdef CONFIG_XEN
bool xen_set_default_idle(void)
{
        bool ret = x86_idle_set();

        static_call_update(x86_idle, default_idle);

        return ret;
}
#endif

struct cpumask cpus_stop_mask;

void __noreturn stop_this_cpu(void *dummy)
{
        struct cpuinfo_x86 *c = this_cpu_ptr(&cpu_info);
        unsigned int cpu = smp_processor_id();

        local_irq_disable();

        /*
         * Remove this CPU from the online mask and disable it
         * unconditionally. This might be redundant in case that the reboot
         * vector was handled late and stop_other_cpus() sent an NMI.
         *
         * According to SDM and APM NMIs can be accepted even after soft
         * disabling the local APIC.
         */
        set_cpu_online(cpu, false);
        disable_local_APIC();
        mcheck_cpu_clear(c);

        /*
         * Use wbinvd on processors that support SME. This provides support
         * for performing a successful kexec when going from SME inactive
         * to SME active (or vice-versa). The cache must be cleared so that
         * if there are entries with the same physical address, both with and
         * without the encryption bit, they don't race each other when flushed
         * and potentially end up with the wrong entry being committed to
         * memory.
         *
         * Test the CPUID bit directly because the machine might've cleared
         * X86_FEATURE_SME due to cmdline options.
         */
        if (c->extended_cpuid_level >= 0x8000001f && (cpuid_eax(0x8000001f) & BIT(0)))
                native_wbinvd();

        /*
         * This brings a cache line back and dirties it, but
         * native_stop_other_cpus() will overwrite cpus_stop_mask after it
         * observed that all CPUs reported stop. This write will invalidate
         * the related cache line on this CPU.
         */
        cpumask_clear_cpu(cpu, &cpus_stop_mask);

#ifdef CONFIG_SMP
        if (smp_ops.stop_this_cpu) {
                smp_ops.stop_this_cpu();
                unreachable();
        }
#endif

        for (;;) {
                /*
                 * Use native_halt() so that memory contents don't change
                 * (stack usage and variables) after possibly issuing the
                 * native_wbinvd() above.
                 */
                native_halt();
        }
}

/*
 * Prefer MWAIT over HALT if MWAIT is supported, MWAIT_CPUID leaf
 * exists and whenever MONITOR/MWAIT extensions are present there is at
 * least one C1 substate.
 *
 * Do not prefer MWAIT if MONITOR instruction has a bug or idle=nomwait
 * is passed to kernel commandline parameter.
 */
static __init bool prefer_mwait_c1_over_halt(void)
{
        const struct cpuinfo_x86 *c = &boot_cpu_data;
        u32 eax, ebx, ecx, edx;

        /* If override is enforced on the command line, fall back to HALT. */
        if (boot_option_idle_override != IDLE_NO_OVERRIDE)
                return false;

        /* MWAIT is not supported on this platform. Fallback to HALT */
        if (!cpu_has(c, X86_FEATURE_MWAIT))
                return false;

        /* Monitor has a bug or APIC stops in C1E. Fallback to HALT */
        if (boot_cpu_has_bug(X86_BUG_MONITOR) || boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E))
                return false;

        cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);

        /*
         * If MWAIT extensions are not available, it is safe to use MWAIT
         * with EAX=0, ECX=0.
         */
        if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED))
                return true;

        /*
         * If MWAIT extensions are available, there should be at least one
         * MWAIT C1 substate present.
         */
        return !!(edx & MWAIT_C1_SUBSTATE_MASK);
}

/*
 * MONITOR/MWAIT with no hints, used for default C1 state. This invokes MWAIT
 * with interrupts enabled and no flags, which is backwards compatible with the
 * original MWAIT implementation.
 */
static __cpuidle void mwait_idle(void)
{
        if (!current_set_polling_and_test()) {
                if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
                        mb(); /* quirk */
                        clflush((void *)&current_thread_info()->flags);
                        mb(); /* quirk */
                }

                __monitor((void *)&current_thread_info()->flags, 0, 0);
                if (!need_resched()) {
                        __sti_mwait(0, 0);
                        raw_local_irq_disable();
                }
        }
        __current_clr_polling();
}

void __init select_idle_routine(void)
{
        if (boot_option_idle_override == IDLE_POLL) {
                if (IS_ENABLED(CONFIG_SMP) && __max_threads_per_core > 1)
                        pr_warn_once("WARNING: polling idle and HT enabled, performance may degrade\n");
                return;
        }

        /* Required to guard against xen_set_default_idle() */
        if (x86_idle_set())
                return;

        if (prefer_mwait_c1_over_halt()) {
                pr_info("using mwait in idle threads\n");
                static_call_update(x86_idle, mwait_idle);
        } else if (cpu_feature_enabled(X86_FEATURE_TDX_GUEST)) {
                pr_info("using TDX aware idle routine\n");
                static_call_update(x86_idle, tdx_safe_halt);
        } else {
                static_call_update(x86_idle, default_idle);
        }
}

void amd_e400_c1e_apic_setup(void)
{
        if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E)) {
                pr_info("Switch to broadcast mode on CPU%d\n", smp_processor_id());
                local_irq_disable();
                tick_broadcast_force();
                local_irq_enable();
        }
}

void __init arch_post_acpi_subsys_init(void)
{
        u32 lo, hi;

        if (!boot_cpu_has_bug(X86_BUG_AMD_E400))
                return;

        /*
         * AMD E400 detection needs to happen after ACPI has been enabled. If
         * the machine is affected K8_INTP_C1E_ACTIVE_MASK bits are set in
         * MSR_K8_INT_PENDING_MSG.
         */
        rdmsr(MSR_K8_INT_PENDING_MSG, lo, hi);
        if (!(lo & K8_INTP_C1E_ACTIVE_MASK))
                return;

        boot_cpu_set_bug(X86_BUG_AMD_APIC_C1E);

        if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
                mark_tsc_unstable("TSC halt in AMD C1E");

        if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST_IDLE))
                static_branch_enable(&arch_needs_tick_broadcast);
        pr_info("System has AMD C1E erratum E400. Workaround enabled.\n");
}

static int __init idle_setup(char *str)
{
        if (!str)
                return -EINVAL;

        if (!strcmp(str, "poll")) {
                pr_info("using polling idle threads\n");
                boot_option_idle_override = IDLE_POLL;
                cpu_idle_poll_ctrl(true);
        } else if (!strcmp(str, "halt")) {
                /* 'idle=halt' HALT for idle. C-states are disabled. */
                boot_option_idle_override = IDLE_HALT;
        } else if (!strcmp(str, "nomwait")) {
                /* 'idle=nomwait' disables MWAIT for idle */
                boot_option_idle_override = IDLE_NOMWAIT;
        } else {
                return -EINVAL;
        }

        return 0;
}
early_param("idle", idle_setup);

unsigned long arch_align_stack(unsigned long sp)
{
        if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
                sp -= get_random_u32_below(8192);
        return sp & ~0xf;
}

unsigned long arch_randomize_brk(struct mm_struct *mm)
{
        if (mmap_is_ia32())
                return randomize_page(mm->brk, SZ_32M);

        return randomize_page(mm->brk, SZ_1G);
}

/*
 * Called from fs/proc with a reference on @p to find the function
 * which called into schedule(). This needs to be done carefully
 * because the task might wake up and we might look at a stack
 * changing under us.
 */
unsigned long __get_wchan(struct task_struct *p)
{
        struct unwind_state state;
        unsigned long addr = 0;

        if (!try_get_task_stack(p))
                return 0;

        for (unwind_start(&state, p, NULL, NULL); !unwind_done(&state);
             unwind_next_frame(&state)) {
                addr = unwind_get_return_address(&state);
                if (!addr)
                        break;
                if (in_sched_functions(addr))
                        continue;
                break;
        }

        put_task_stack(p);

        return addr;
}

long do_arch_prctl_common(int option, unsigned long arg2)
{
        switch (option) {
        case ARCH_GET_CPUID:
                return get_cpuid_mode();
        case ARCH_SET_CPUID:
                return set_cpuid_mode(arg2);
        case ARCH_GET_XCOMP_SUPP:
        case ARCH_GET_XCOMP_PERM:
        case ARCH_REQ_XCOMP_PERM:
        case ARCH_GET_XCOMP_GUEST_PERM:
        case ARCH_REQ_XCOMP_GUEST_PERM:
                return fpu_xstate_prctl(option, arg2);
        }

        return -EINVAL;
}











































































































































































































   54 






























































































































































































































































































































   20 
    5 










    5 


    5 

    5 

   20 





   54 


   54 














   53 



   54 

   54 
   54 


















































   54 


































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
// SPDX-License-Identifier: GPL-2.0
/*
 * HugeTLB Vmemmap Optimization (HVO)
 *
 * Copyright (c) 2020, ByteDance. All rights reserved.
 *
 *     Author: Muchun Song <songmuchun@bytedance.com>
 *
 * See Documentation/mm/vmemmap_dedup.rst
 */
#define pr_fmt(fmt)        "HugeTLB: " fmt

#include <linux/pgtable.h>
#include <linux/moduleparam.h>
#include <linux/bootmem_info.h>
#include <linux/mmdebug.h>
#include <linux/pagewalk.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"

/**
 * struct vmemmap_remap_walk - walk vmemmap page table
 *
 * @remap_pte:                called for each lowest-level entry (PTE).
 * @nr_walked:                the number of walked pte.
 * @reuse_page:                the page which is reused for the tail vmemmap pages.
 * @reuse_addr:                the virtual address of the @reuse_page page.
 * @vmemmap_pages:        the list head of the vmemmap pages that can be freed
 *                        or is mapped from.
 * @flags:                used to modify behavior in vmemmap page table walking
 *                        operations.
 */
struct vmemmap_remap_walk {
        void                        (*remap_pte)(pte_t *pte, unsigned long addr,
                                             struct vmemmap_remap_walk *walk);
        unsigned long                nr_walked;
        struct page                *reuse_page;
        unsigned long                reuse_addr;
        struct list_head        *vmemmap_pages;

/* Skip the TLB flush when we split the PMD */
#define VMEMMAP_SPLIT_NO_TLB_FLUSH        BIT(0)
/* Skip the TLB flush when we remap the PTE */
#define VMEMMAP_REMAP_NO_TLB_FLUSH        BIT(1)
/* synchronize_rcu() to avoid writes from page_ref_add_unless() */
#define VMEMMAP_SYNCHRONIZE_RCU                BIT(2)
        unsigned long                flags;
};

static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
                             struct vmemmap_remap_walk *walk)
{
        pmd_t __pmd;
        int i;
        unsigned long addr = start;
        pte_t *pgtable;

        pgtable = pte_alloc_one_kernel(&init_mm);
        if (!pgtable)
                return -ENOMEM;

        pmd_populate_kernel(&init_mm, &__pmd, pgtable);

        for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) {
                pte_t entry, *pte;
                pgprot_t pgprot = PAGE_KERNEL;

                entry = mk_pte(head + i, pgprot);
                pte = pte_offset_kernel(&__pmd, addr);
                set_pte_at(&init_mm, addr, pte, entry);
        }

        spin_lock(&init_mm.page_table_lock);
        if (likely(pmd_leaf(*pmd))) {
                /*
                 * Higher order allocations from buddy allocator must be able to
                 * be treated as indepdenent small pages (as they can be freed
                 * individually).
                 */
                if (!PageReserved(head))
                        split_page(head, get_order(PMD_SIZE));

                /* Make pte visible before pmd. See comment in pmd_install(). */
                smp_wmb();
                pmd_populate_kernel(&init_mm, pmd, pgtable);
                if (!(walk->flags & VMEMMAP_SPLIT_NO_TLB_FLUSH))
                        flush_tlb_kernel_range(start, start + PMD_SIZE);
        } else {
                pte_free_kernel(&init_mm, pgtable);
        }
        spin_unlock(&init_mm.page_table_lock);

        return 0;
}

static int vmemmap_pmd_entry(pmd_t *pmd, unsigned long addr,
                             unsigned long next, struct mm_walk *walk)
{
        int ret = 0;
        struct page *head;
        struct vmemmap_remap_walk *vmemmap_walk = walk->private;

        /* Only splitting, not remapping the vmemmap pages. */
        if (!vmemmap_walk->remap_pte)
                walk->action = ACTION_CONTINUE;

        spin_lock(&init_mm.page_table_lock);
        head = pmd_leaf(*pmd) ? pmd_page(*pmd) : NULL;
        /*
         * Due to HugeTLB alignment requirements and the vmemmap
         * pages being at the start of the hotplugged memory
         * region in memory_hotplug.memmap_on_memory case. Checking
         * the vmemmap page associated with the first vmemmap page
         * if it is self-hosted is sufficient.
         *
         * [                  hotplugged memory                  ]
         * [        section        ][...][        section        ]
         * [ vmemmap ][              usable memory               ]
         *   ^  | ^                        |
         *   +--+ |                        |
         *        +------------------------+
         */
        if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG) && unlikely(!vmemmap_walk->nr_walked)) {
                struct page *page = head ? head + pte_index(addr) :
                                    pte_page(ptep_get(pte_offset_kernel(pmd, addr)));

                if (PageVmemmapSelfHosted(page))
                        ret = -ENOTSUPP;
        }
        spin_unlock(&init_mm.page_table_lock);
        if (!head || ret)
                return ret;

        return vmemmap_split_pmd(pmd, head, addr & PMD_MASK, vmemmap_walk);
}

static int vmemmap_pte_entry(pte_t *pte, unsigned long addr,
                             unsigned long next, struct mm_walk *walk)
{
        struct vmemmap_remap_walk *vmemmap_walk = walk->private;

        /*
         * The reuse_page is found 'first' in page table walking before
         * starting remapping.
         */
        if (!vmemmap_walk->reuse_page)
                vmemmap_walk->reuse_page = pte_page(ptep_get(pte));
        else
                vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
        vmemmap_walk->nr_walked++;

        return 0;
}

static const struct mm_walk_ops vmemmap_remap_ops = {
        .pmd_entry        = vmemmap_pmd_entry,
        .pte_entry        = vmemmap_pte_entry,
};

static int vmemmap_remap_range(unsigned long start, unsigned long end,
                               struct vmemmap_remap_walk *walk)
{
        int ret;

        VM_BUG_ON(!PAGE_ALIGNED(start | end));

        mmap_read_lock(&init_mm);
        ret = walk_page_range_novma(&init_mm, start, end, &vmemmap_remap_ops,
                                    NULL, walk);
        mmap_read_unlock(&init_mm);
        if (ret)
                return ret;

        if (walk->remap_pte && !(walk->flags & VMEMMAP_REMAP_NO_TLB_FLUSH))
                flush_tlb_kernel_range(start, end);

        return 0;
}

/*
 * Free a vmemmap page. A vmemmap page can be allocated from the memblock
 * allocator or buddy allocator. If the PG_reserved flag is set, it means
 * that it allocated from the memblock allocator, just free it via the
 * free_bootmem_page(). Otherwise, use __free_page().
 */
static inline void free_vmemmap_page(struct page *page)
{
        if (PageReserved(page)) {
                memmap_boot_pages_add(-1);
                free_bootmem_page(page);
        } else {
                memmap_pages_add(-1);
                __free_page(page);
        }
}

/* Free a list of the vmemmap pages */
static void free_vmemmap_page_list(struct list_head *list)
{
        struct page *page, *next;

        list_for_each_entry_safe(page, next, list, lru)
                free_vmemmap_page(page);
}

static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
                              struct vmemmap_remap_walk *walk)
{
        /*
         * Remap the tail pages as read-only to catch illegal write operation
         * to the tail pages.
         */
        pgprot_t pgprot = PAGE_KERNEL_RO;
        struct page *page = pte_page(ptep_get(pte));
        pte_t entry;

        /* Remapping the head page requires r/w */
        if (unlikely(addr == walk->reuse_addr)) {
                pgprot = PAGE_KERNEL;
                list_del(&walk->reuse_page->lru);

                /*
                 * Makes sure that preceding stores to the page contents from
                 * vmemmap_remap_free() become visible before the set_pte_at()
                 * write.
                 */
                smp_wmb();
        }

        entry = mk_pte(walk->reuse_page, pgprot);
        list_add(&page->lru, walk->vmemmap_pages);
        set_pte_at(&init_mm, addr, pte, entry);
}

/*
 * How many struct page structs need to be reset. When we reuse the head
 * struct page, the special metadata (e.g. page->flags or page->mapping)
 * cannot copy to the tail struct page structs. The invalid value will be
 * checked in the free_tail_page_prepare(). In order to avoid the message
 * of "corrupted mapping in tail page". We need to reset at least 3 (one
 * head struct page struct and two tail struct page structs) struct page
 * structs.
 */
#define NR_RESET_STRUCT_PAGE                3

static inline void reset_struct_pages(struct page *start)
{
        struct page *from = start + NR_RESET_STRUCT_PAGE;

        BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page));
        memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE);
}

static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
                                struct vmemmap_remap_walk *walk)
{
        pgprot_t pgprot = PAGE_KERNEL;
        struct page *page;
        void *to;

        BUG_ON(pte_page(ptep_get(pte)) != walk->reuse_page);

        page = list_first_entry(walk->vmemmap_pages, struct page, lru);
        list_del(&page->lru);
        to = page_to_virt(page);
        copy_page(to, (void *)walk->reuse_addr);
        reset_struct_pages(to);

        /*
         * Makes sure that preceding stores to the page contents become visible
         * before the set_pte_at() write.
         */
        smp_wmb();
        set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot));
}

/**
 * vmemmap_remap_split - split the vmemmap virtual address range [@start, @end)
 *                      backing PMDs of the directmap into PTEs
 * @start:     start address of the vmemmap virtual address range that we want
 *             to remap.
 * @end:       end address of the vmemmap virtual address range that we want to
 *             remap.
 * @reuse:     reuse address.
 *
 * Return: %0 on success, negative error code otherwise.
 */
static int vmemmap_remap_split(unsigned long start, unsigned long end,
                               unsigned long reuse)
{
        struct vmemmap_remap_walk walk = {
                .remap_pte        = NULL,
                .flags                = VMEMMAP_SPLIT_NO_TLB_FLUSH,
        };

        /* See the comment in the vmemmap_remap_free(). */
        BUG_ON(start - reuse != PAGE_SIZE);

        return vmemmap_remap_range(reuse, end, &walk);
}

/**
 * vmemmap_remap_free - remap the vmemmap virtual address range [@start, @end)
 *                        to the page which @reuse is mapped to, then free vmemmap
 *                        which the range are mapped to.
 * @start:        start address of the vmemmap virtual address range that we want
 *                to remap.
 * @end:        end address of the vmemmap virtual address range that we want to
 *                remap.
 * @reuse:        reuse address.
 * @vmemmap_pages: list to deposit vmemmap pages to be freed.  It is callers
 *                responsibility to free pages.
 * @flags:        modifications to vmemmap_remap_walk flags
 *
 * Return: %0 on success, negative error code otherwise.
 */
static int vmemmap_remap_free(unsigned long start, unsigned long end,
                              unsigned long reuse,
                              struct list_head *vmemmap_pages,
                              unsigned long flags)
{
        int ret;
        struct vmemmap_remap_walk walk = {
                .remap_pte        = vmemmap_remap_pte,
                .reuse_addr        = reuse,
                .vmemmap_pages        = vmemmap_pages,
                .flags                = flags,
        };
        int nid = page_to_nid((struct page *)reuse);
        gfp_t gfp_mask = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;

        /*
         * Allocate a new head vmemmap page to avoid breaking a contiguous
         * block of struct page memory when freeing it back to page allocator
         * in free_vmemmap_page_list(). This will allow the likely contiguous
         * struct page backing memory to be kept contiguous and allowing for
         * more allocations of hugepages. Fallback to the currently
         * mapped head page in case should it fail to allocate.
         */
        walk.reuse_page = alloc_pages_node(nid, gfp_mask, 0);
        if (walk.reuse_page) {
                copy_page(page_to_virt(walk.reuse_page),
                          (void *)walk.reuse_addr);
                list_add(&walk.reuse_page->lru, vmemmap_pages);
                memmap_pages_add(1);
        }

        /*
         * In order to make remapping routine most efficient for the huge pages,
         * the routine of vmemmap page table walking has the following rules
         * (see more details from the vmemmap_pte_range()):
         *
         * - The range [@start, @end) and the range [@reuse, @reuse + PAGE_SIZE)
         *   should be continuous.
         * - The @reuse address is part of the range [@reuse, @end) that we are
         *   walking which is passed to vmemmap_remap_range().
         * - The @reuse address is the first in the complete range.
         *
         * So we need to make sure that @start and @reuse meet the above rules.
         */
        BUG_ON(start - reuse != PAGE_SIZE);

        ret = vmemmap_remap_range(reuse, end, &walk);
        if (ret && walk.nr_walked) {
                end = reuse + walk.nr_walked * PAGE_SIZE;
                /*
                 * vmemmap_pages contains pages from the previous
                 * vmemmap_remap_range call which failed.  These
                 * are pages which were removed from the vmemmap.
                 * They will be restored in the following call.
                 */
                walk = (struct vmemmap_remap_walk) {
                        .remap_pte        = vmemmap_restore_pte,
                        .reuse_addr        = reuse,
                        .vmemmap_pages        = vmemmap_pages,
                        .flags                = 0,
                };

                vmemmap_remap_range(reuse, end, &walk);
        }

        return ret;
}

static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
                                   struct list_head *list)
{
        gfp_t gfp_mask = GFP_KERNEL | __GFP_RETRY_MAYFAIL;
        unsigned long nr_pages = (end - start) >> PAGE_SHIFT;
        int nid = page_to_nid((struct page *)start);
        struct page *page, *next;
        int i;

        for (i = 0; i < nr_pages; i++) {
                page = alloc_pages_node(nid, gfp_mask, 0);
                if (!page)
                        goto out;
                list_add(&page->lru, list);
        }
        memmap_pages_add(nr_pages);

        return 0;
out:
        list_for_each_entry_safe(page, next, list, lru)
                __free_page(page);
        return -ENOMEM;
}

/**
 * vmemmap_remap_alloc - remap the vmemmap virtual address range [@start, end)
 *                         to the page which is from the @vmemmap_pages
 *                         respectively.
 * @start:        start address of the vmemmap virtual address range that we want
 *                to remap.
 * @end:        end address of the vmemmap virtual address range that we want to
 *                remap.
 * @reuse:        reuse address.
 * @flags:        modifications to vmemmap_remap_walk flags
 *
 * Return: %0 on success, negative error code otherwise.
 */
static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
                               unsigned long reuse, unsigned long flags)
{
        LIST_HEAD(vmemmap_pages);
        struct vmemmap_remap_walk walk = {
                .remap_pte        = vmemmap_restore_pte,
                .reuse_addr        = reuse,
                .vmemmap_pages        = &vmemmap_pages,
                .flags                = flags,
        };

        /* See the comment in the vmemmap_remap_free(). */
        BUG_ON(start - reuse != PAGE_SIZE);

        if (alloc_vmemmap_page_list(start, end, &vmemmap_pages))
                return -ENOMEM;

        return vmemmap_remap_range(reuse, end, &walk);
}

DEFINE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key);

static bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
core_param(hugetlb_free_vmemmap, vmemmap_optimize_enabled, bool, 0);

static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
                                           struct folio *folio, unsigned long flags)
{
        int ret;
        unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
        unsigned long vmemmap_reuse;

        VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
        VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);

        if (!folio_test_hugetlb_vmemmap_optimized(folio))
                return 0;

        if (flags & VMEMMAP_SYNCHRONIZE_RCU)
                synchronize_rcu();

        vmemmap_end        = vmemmap_start + hugetlb_vmemmap_size(h);
        vmemmap_reuse        = vmemmap_start;
        vmemmap_start        += HUGETLB_VMEMMAP_RESERVE_SIZE;

        /*
         * The pages which the vmemmap virtual address range [@vmemmap_start,
         * @vmemmap_end) are mapped to are freed to the buddy allocator, and
         * the range is mapped to the page which @vmemmap_reuse is mapped to.
         * When a HugeTLB page is freed to the buddy allocator, previously
         * discarded vmemmap pages must be allocated and remapping.
         */
        ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse, flags);
        if (!ret) {
                folio_clear_hugetlb_vmemmap_optimized(folio);
                static_branch_dec(&hugetlb_optimize_vmemmap_key);
        }

        return ret;
}

/**
 * hugetlb_vmemmap_restore_folio - restore previously optimized (by
 *                                hugetlb_vmemmap_optimize_folio()) vmemmap pages which
 *                                will be reallocated and remapped.
 * @h:                struct hstate.
 * @folio:     the folio whose vmemmap pages will be restored.
 *
 * Return: %0 if @folio's vmemmap pages have been reallocated and remapped,
 * negative error code otherwise.
 */
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio)
{
        return __hugetlb_vmemmap_restore_folio(h, folio, VMEMMAP_SYNCHRONIZE_RCU);
}

/**
 * hugetlb_vmemmap_restore_folios - restore vmemmap for every folio on the list.
 * @h:                        hstate.
 * @folio_list:                list of folios.
 * @non_hvo_folios:        Output list of folios for which vmemmap exists.
 *
 * Return: number of folios for which vmemmap was restored, or an error code
 *                if an error was encountered restoring vmemmap for a folio.
 *                Folios that have vmemmap are moved to the non_hvo_folios
 *                list.  Processing of entries stops when the first error is
 *                encountered. The folio that experienced the error and all
 *                non-processed folios will remain on folio_list.
 */
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
                                        struct list_head *folio_list,
                                        struct list_head *non_hvo_folios)
{
        struct folio *folio, *t_folio;
        long restored = 0;
        long ret = 0;
        unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;

        list_for_each_entry_safe(folio, t_folio, folio_list, lru) {
                if (folio_test_hugetlb_vmemmap_optimized(folio)) {
                        ret = __hugetlb_vmemmap_restore_folio(h, folio, flags);
                        /* only need to synchronize_rcu() once for each batch */
                        flags &= ~VMEMMAP_SYNCHRONIZE_RCU;

                        if (ret)
                                break;
                        restored++;
                }

                /* Add non-optimized folios to output list */
                list_move(&folio->lru, non_hvo_folios);
        }

        if (restored)
                flush_tlb_all();
        if (!ret)
                ret = restored;
        return ret;
}

/* Return true iff a HugeTLB whose vmemmap should and can be optimized. */
static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *folio)
{
        if (folio_test_hugetlb_vmemmap_optimized(folio))
                return false;

        if (!READ_ONCE(vmemmap_optimize_enabled))
                return false;

        if (!hugetlb_vmemmap_optimizable(h))
                return false;

        return true;
}

static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
                                            struct folio *folio,
                                            struct list_head *vmemmap_pages,
                                            unsigned long flags)
{
        int ret = 0;
        unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
        unsigned long vmemmap_reuse;

        VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
        VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);

        if (!vmemmap_should_optimize_folio(h, folio))
                return ret;

        static_branch_inc(&hugetlb_optimize_vmemmap_key);

        if (flags & VMEMMAP_SYNCHRONIZE_RCU)
                synchronize_rcu();
        /*
         * Very Subtle
         * If VMEMMAP_REMAP_NO_TLB_FLUSH is set, TLB flushing is not performed
         * immediately after remapping.  As a result, subsequent accesses
         * and modifications to struct pages associated with the hugetlb
         * page could be to the OLD struct pages.  Set the vmemmap optimized
         * flag here so that it is copied to the new head page.  This keeps
         * the old and new struct pages in sync.
         * If there is an error during optimization, we will immediately FLUSH
         * the TLB and clear the flag below.
         */
        folio_set_hugetlb_vmemmap_optimized(folio);

        vmemmap_end        = vmemmap_start + hugetlb_vmemmap_size(h);
        vmemmap_reuse        = vmemmap_start;
        vmemmap_start        += HUGETLB_VMEMMAP_RESERVE_SIZE;

        /*
         * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end)
         * to the page which @vmemmap_reuse is mapped to.  Add pages previously
         * mapping the range to vmemmap_pages list so that they can be freed by
         * the caller.
         */
        ret = vmemmap_remap_free(vmemmap_start, vmemmap_end, vmemmap_reuse,
                                 vmemmap_pages, flags);
        if (ret) {
                static_branch_dec(&hugetlb_optimize_vmemmap_key);
                folio_clear_hugetlb_vmemmap_optimized(folio);
        }

        return ret;
}

/**
 * hugetlb_vmemmap_optimize_folio - optimize @folio's vmemmap pages.
 * @h:                struct hstate.
 * @folio:     the folio whose vmemmap pages will be optimized.
 *
 * This function only tries to optimize @folio's vmemmap pages and does not
 * guarantee that the optimization will succeed after it returns. The caller
 * can use folio_test_hugetlb_vmemmap_optimized(@folio) to detect if @folio's
 * vmemmap pages have been optimized.
 */
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio)
{
        LIST_HEAD(vmemmap_pages);

        __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, VMEMMAP_SYNCHRONIZE_RCU);
        free_vmemmap_page_list(&vmemmap_pages);
}

static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *folio)
{
        unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end;
        unsigned long vmemmap_reuse;

        if (!vmemmap_should_optimize_folio(h, folio))
                return 0;

        vmemmap_end        = vmemmap_start + hugetlb_vmemmap_size(h);
        vmemmap_reuse        = vmemmap_start;
        vmemmap_start        += HUGETLB_VMEMMAP_RESERVE_SIZE;

        /*
         * Split PMDs on the vmemmap virtual address range [@vmemmap_start,
         * @vmemmap_end]
         */
        return vmemmap_remap_split(vmemmap_start, vmemmap_end, vmemmap_reuse);
}

void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
{
        struct folio *folio;
        LIST_HEAD(vmemmap_pages);
        unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU;

        list_for_each_entry(folio, folio_list, lru) {
                int ret = hugetlb_vmemmap_split_folio(h, folio);

                /*
                 * Spliting the PMD requires allocating a page, thus lets fail
                 * early once we encounter the first OOM. No point in retrying
                 * as it can be dynamically done on remap with the memory
                 * we get back from the vmemmap deduplication.
                 */
                if (ret == -ENOMEM)
                        break;
        }

        flush_tlb_all();

        list_for_each_entry(folio, folio_list, lru) {
                int ret;

                ret = __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
                /* only need to synchronize_rcu() once for each batch */
                flags &= ~VMEMMAP_SYNCHRONIZE_RCU;

                /*
                 * Pages to be freed may have been accumulated.  If we
                 * encounter an ENOMEM,  free what we have and try again.
                 * This can occur in the case that both spliting fails
                 * halfway and head page allocation also failed. In this
                 * case __hugetlb_vmemmap_optimize_folio() would free memory
                 * allowing more vmemmap remaps to occur.
                 */
                if (ret == -ENOMEM && !list_empty(&vmemmap_pages)) {
                        flush_tlb_all();
                        free_vmemmap_page_list(&vmemmap_pages);
                        INIT_LIST_HEAD(&vmemmap_pages);
                        __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
                }
        }

        flush_tlb_all();
        free_vmemmap_page_list(&vmemmap_pages);
}

static struct ctl_table hugetlb_vmemmap_sysctls[] = {
        {
                .procname        = "hugetlb_optimize_vmemmap",
                .data                = &vmemmap_optimize_enabled,
                .maxlen                = sizeof(vmemmap_optimize_enabled),
                .mode                = 0644,
                .proc_handler        = proc_dobool,
        },
};

static int __init hugetlb_vmemmap_init(void)
{
        const struct hstate *h;

        /* HUGETLB_VMEMMAP_RESERVE_SIZE should cover all used struct pages */
        BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES);

        for_each_hstate(h) {
                if (hugetlb_vmemmap_optimizable(h)) {
                        register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
                        break;
                }
        }
        return 0;
}
late_initcall(hugetlb_vmemmap_init);



































































































































































   38 










   39 
   38 

   39 














   39 









  137 




   39 



  138 



















































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
// SPDX-License-Identifier: GPL-2.0

#include <linux/debugfs.h>

#include "netdevsim.h"

#define NSIM_DEV_HWSTATS_TRAFFIC_MS        100

static struct list_head *
nsim_dev_hwstats_get_list_head(struct nsim_dev_hwstats *hwstats,
                               enum netdev_offload_xstats_type type)
{
        switch (type) {
        case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
                return &hwstats->l3_list;
        }

        WARN_ON_ONCE(1);
        return NULL;
}

static void nsim_dev_hwstats_traffic_bump(struct nsim_dev_hwstats *hwstats,
                                          enum netdev_offload_xstats_type type)
{
        struct nsim_dev_hwstats_netdev *hwsdev;
        struct list_head *hwsdev_list;

        hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);
        if (WARN_ON(!hwsdev_list))
                return;

        list_for_each_entry(hwsdev, hwsdev_list, list) {
                if (hwsdev->enabled) {
                        hwsdev->stats.rx_packets += 1;
                        hwsdev->stats.tx_packets += 2;
                        hwsdev->stats.rx_bytes += 100;
                        hwsdev->stats.tx_bytes += 300;
                }
        }
}

static void nsim_dev_hwstats_traffic_work(struct work_struct *work)
{
        struct nsim_dev_hwstats *hwstats;

        hwstats = container_of(work, struct nsim_dev_hwstats, traffic_dw.work);
        mutex_lock(&hwstats->hwsdev_list_lock);
        nsim_dev_hwstats_traffic_bump(hwstats, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
        mutex_unlock(&hwstats->hwsdev_list_lock);

        schedule_delayed_work(&hwstats->traffic_dw,
                              msecs_to_jiffies(NSIM_DEV_HWSTATS_TRAFFIC_MS));
}

static struct nsim_dev_hwstats_netdev *
nsim_dev_hwslist_find_hwsdev(struct list_head *hwsdev_list,
                             int ifindex)
{
        struct nsim_dev_hwstats_netdev *hwsdev;

        list_for_each_entry(hwsdev, hwsdev_list, list) {
                if (hwsdev->netdev->ifindex == ifindex)
                        return hwsdev;
        }

        return NULL;
}

static int nsim_dev_hwsdev_enable(struct nsim_dev_hwstats_netdev *hwsdev,
                                  struct netlink_ext_ack *extack)
{
        if (hwsdev->fail_enable) {
                hwsdev->fail_enable = false;
                NL_SET_ERR_MSG_MOD(extack, "Stats enablement set to fail");
                return -ECANCELED;
        }

        hwsdev->enabled = true;
        return 0;
}

static void nsim_dev_hwsdev_disable(struct nsim_dev_hwstats_netdev *hwsdev)
{
        hwsdev->enabled = false;
        memset(&hwsdev->stats, 0, sizeof(hwsdev->stats));
}

static int
nsim_dev_hwsdev_report_delta(struct nsim_dev_hwstats_netdev *hwsdev,
                             struct netdev_notifier_offload_xstats_info *info)
{
        netdev_offload_xstats_report_delta(info->report_delta, &hwsdev->stats);
        memset(&hwsdev->stats, 0, sizeof(hwsdev->stats));
        return 0;
}

static void
nsim_dev_hwsdev_report_used(struct nsim_dev_hwstats_netdev *hwsdev,
                            struct netdev_notifier_offload_xstats_info *info)
{
        if (hwsdev->enabled)
                netdev_offload_xstats_report_used(info->report_used);
}

static int nsim_dev_hwstats_event_off_xstats(struct nsim_dev_hwstats *hwstats,
                                             struct net_device *dev,
                                             unsigned long event, void *ptr)
{
        struct netdev_notifier_offload_xstats_info *info;
        struct nsim_dev_hwstats_netdev *hwsdev;
        struct list_head *hwsdev_list;
        int err = 0;

        info = ptr;
        hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, info->type);
        if (!hwsdev_list)
                return 0;

        mutex_lock(&hwstats->hwsdev_list_lock);

        hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, dev->ifindex);
        if (!hwsdev)
                goto out;

        switch (event) {
        case NETDEV_OFFLOAD_XSTATS_ENABLE:
                err = nsim_dev_hwsdev_enable(hwsdev, info->info.extack);
                break;
        case NETDEV_OFFLOAD_XSTATS_DISABLE:
                nsim_dev_hwsdev_disable(hwsdev);
                break;
        case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
                nsim_dev_hwsdev_report_used(hwsdev, info);
                break;
        case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
                err = nsim_dev_hwsdev_report_delta(hwsdev, info);
                break;
        }

out:
        mutex_unlock(&hwstats->hwsdev_list_lock);
        return err;
}

static void nsim_dev_hwsdev_fini(struct nsim_dev_hwstats_netdev *hwsdev)
{
        dev_put(hwsdev->netdev);
        kfree(hwsdev);
}

static void
__nsim_dev_hwstats_event_unregister(struct nsim_dev_hwstats *hwstats,
                                    struct net_device *dev,
                                    enum netdev_offload_xstats_type type)
{
        struct nsim_dev_hwstats_netdev *hwsdev;
        struct list_head *hwsdev_list;

        hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);
        if (WARN_ON(!hwsdev_list))
                return;

        hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, dev->ifindex);
        if (!hwsdev)
                return;

        list_del(&hwsdev->list);
        nsim_dev_hwsdev_fini(hwsdev);
}

static void nsim_dev_hwstats_event_unregister(struct nsim_dev_hwstats *hwstats,
                                              struct net_device *dev)
{
        mutex_lock(&hwstats->hwsdev_list_lock);
        __nsim_dev_hwstats_event_unregister(hwstats, dev,
                                            NETDEV_OFFLOAD_XSTATS_TYPE_L3);
        mutex_unlock(&hwstats->hwsdev_list_lock);
}

static int nsim_dev_hwstats_event(struct nsim_dev_hwstats *hwstats,
                                  struct net_device *dev,
                                  unsigned long event, void *ptr)
{
        switch (event) {
        case NETDEV_OFFLOAD_XSTATS_ENABLE:
        case NETDEV_OFFLOAD_XSTATS_DISABLE:
        case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
        case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
                return nsim_dev_hwstats_event_off_xstats(hwstats, dev,
                                                         event, ptr);
        case NETDEV_UNREGISTER:
                nsim_dev_hwstats_event_unregister(hwstats, dev);
                break;
        }

        return 0;
}

static int nsim_dev_netdevice_event(struct notifier_block *nb,
                                    unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct nsim_dev_hwstats *hwstats;
        int err = 0;

        hwstats = container_of(nb, struct nsim_dev_hwstats, netdevice_nb);
        err = nsim_dev_hwstats_event(hwstats, dev, event, ptr);
        if (err)
                return notifier_from_errno(err);

        return NOTIFY_OK;
}

static int
nsim_dev_hwstats_enable_ifindex(struct nsim_dev_hwstats *hwstats,
                                int ifindex,
                                enum netdev_offload_xstats_type type,
                                struct list_head *hwsdev_list)
{
        struct nsim_dev_hwstats_netdev *hwsdev;
        struct nsim_dev *nsim_dev;
        struct net_device *netdev;
        bool notify = false;
        struct net *net;
        int err = 0;

        nsim_dev = container_of(hwstats, struct nsim_dev, hwstats);
        net = nsim_dev_net(nsim_dev);

        rtnl_lock();
        mutex_lock(&hwstats->hwsdev_list_lock);
        hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);
        if (hwsdev)
                goto out_unlock_list;

        netdev = dev_get_by_index(net, ifindex);
        if (!netdev) {
                err = -ENODEV;
                goto out_unlock_list;
        }

        hwsdev = kzalloc(sizeof(*hwsdev), GFP_KERNEL);
        if (!hwsdev) {
                err = -ENOMEM;
                goto out_put_netdev;
        }

        hwsdev->netdev = netdev;
        list_add_tail(&hwsdev->list, hwsdev_list);
        mutex_unlock(&hwstats->hwsdev_list_lock);

        if (netdev_offload_xstats_enabled(netdev, type)) {
                nsim_dev_hwsdev_enable(hwsdev, NULL);
                notify = true;
        }

        if (notify)
                rtnl_offload_xstats_notify(netdev);
        rtnl_unlock();
        return err;

out_put_netdev:
        dev_put(netdev);
out_unlock_list:
        mutex_unlock(&hwstats->hwsdev_list_lock);
        rtnl_unlock();
        return err;
}

static int
nsim_dev_hwstats_disable_ifindex(struct nsim_dev_hwstats *hwstats,
                                 int ifindex,
                                 enum netdev_offload_xstats_type type,
                                 struct list_head *hwsdev_list)
{
        struct nsim_dev_hwstats_netdev *hwsdev;
        int err = 0;

        rtnl_lock();
        mutex_lock(&hwstats->hwsdev_list_lock);
        hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);
        if (hwsdev)
                list_del(&hwsdev->list);
        mutex_unlock(&hwstats->hwsdev_list_lock);

        if (!hwsdev) {
                err = -ENOENT;
                goto unlock_out;
        }

        if (netdev_offload_xstats_enabled(hwsdev->netdev, type)) {
                netdev_offload_xstats_push_delta(hwsdev->netdev, type,
                                                 &hwsdev->stats);
                rtnl_offload_xstats_notify(hwsdev->netdev);
        }
        nsim_dev_hwsdev_fini(hwsdev);

unlock_out:
        rtnl_unlock();
        return err;
}

static int
nsim_dev_hwstats_fail_ifindex(struct nsim_dev_hwstats *hwstats,
                              int ifindex,
                              enum netdev_offload_xstats_type type,
                              struct list_head *hwsdev_list)
{
        struct nsim_dev_hwstats_netdev *hwsdev;
        int err = 0;

        mutex_lock(&hwstats->hwsdev_list_lock);

        hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);
        if (!hwsdev) {
                err = -ENOENT;
                goto err_hwsdev_list_unlock;
        }

        hwsdev->fail_enable = true;

err_hwsdev_list_unlock:
        mutex_unlock(&hwstats->hwsdev_list_lock);
        return err;
}

enum nsim_dev_hwstats_do {
        NSIM_DEV_HWSTATS_DO_DISABLE,
        NSIM_DEV_HWSTATS_DO_ENABLE,
        NSIM_DEV_HWSTATS_DO_FAIL,
};

struct nsim_dev_hwstats_fops {
        const struct file_operations fops;
        enum nsim_dev_hwstats_do action;
        enum netdev_offload_xstats_type type;
};

static ssize_t
nsim_dev_hwstats_do_write(struct file *file,
                          const char __user *data,
                          size_t count, loff_t *ppos)
{
        struct nsim_dev_hwstats *hwstats = file->private_data;
        struct nsim_dev_hwstats_fops *hwsfops;
        struct list_head *hwsdev_list;
        int ifindex;
        int err;

        hwsfops = container_of(debugfs_real_fops(file),
                               struct nsim_dev_hwstats_fops, fops);

        err = kstrtoint_from_user(data, count, 0, &ifindex);
        if (err)
                return err;

        hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, hwsfops->type);
        if (WARN_ON(!hwsdev_list))
                return -EINVAL;

        switch (hwsfops->action) {
        case NSIM_DEV_HWSTATS_DO_DISABLE:
                err = nsim_dev_hwstats_disable_ifindex(hwstats, ifindex,
                                                       hwsfops->type,
                                                       hwsdev_list);
                break;
        case NSIM_DEV_HWSTATS_DO_ENABLE:
                err = nsim_dev_hwstats_enable_ifindex(hwstats, ifindex,
                                                      hwsfops->type,
                                                      hwsdev_list);
                break;
        case NSIM_DEV_HWSTATS_DO_FAIL:
                err = nsim_dev_hwstats_fail_ifindex(hwstats, ifindex,
                                                    hwsfops->type,
                                                    hwsdev_list);
                break;
        }
        if (err)
                return err;

        return count;
}

#define NSIM_DEV_HWSTATS_FOPS(ACTION, TYPE)                        \
        {                                                        \
                .fops = {                                        \
                        .open = simple_open,                        \
                        .write = nsim_dev_hwstats_do_write,        \
                        .llseek = generic_file_llseek,                \
                        .owner = THIS_MODULE,                        \
                },                                                \
                .action = ACTION,                                \
                .type = TYPE,                                        \
        }

static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_disable_fops =
        NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_DISABLE,
                              NETDEV_OFFLOAD_XSTATS_TYPE_L3);

static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_enable_fops =
        NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_ENABLE,
                              NETDEV_OFFLOAD_XSTATS_TYPE_L3);

static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_fail_fops =
        NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_FAIL,
                              NETDEV_OFFLOAD_XSTATS_TYPE_L3);

#undef NSIM_DEV_HWSTATS_FOPS

int nsim_dev_hwstats_init(struct nsim_dev *nsim_dev)
{
        struct nsim_dev_hwstats *hwstats = &nsim_dev->hwstats;
        struct net *net = nsim_dev_net(nsim_dev);
        int err;

        mutex_init(&hwstats->hwsdev_list_lock);
        INIT_LIST_HEAD(&hwstats->l3_list);

        hwstats->netdevice_nb.notifier_call = nsim_dev_netdevice_event;
        err = register_netdevice_notifier_net(net, &hwstats->netdevice_nb);
        if (err)
                goto err_mutex_destroy;

        hwstats->ddir = debugfs_create_dir("hwstats", nsim_dev->ddir);
        if (IS_ERR(hwstats->ddir)) {
                err = PTR_ERR(hwstats->ddir);
                goto err_unregister_notifier;
        }

        hwstats->l3_ddir = debugfs_create_dir("l3", hwstats->ddir);
        if (IS_ERR(hwstats->l3_ddir)) {
                err = PTR_ERR(hwstats->l3_ddir);
                goto err_remove_hwstats_recursive;
        }

        debugfs_create_file("enable_ifindex", 0200, hwstats->l3_ddir, hwstats,
                            &nsim_dev_hwstats_l3_enable_fops.fops);
        debugfs_create_file("disable_ifindex", 0200, hwstats->l3_ddir, hwstats,
                            &nsim_dev_hwstats_l3_disable_fops.fops);
        debugfs_create_file("fail_next_enable", 0200, hwstats->l3_ddir, hwstats,
                            &nsim_dev_hwstats_l3_fail_fops.fops);

        INIT_DELAYED_WORK(&hwstats->traffic_dw,
                          &nsim_dev_hwstats_traffic_work);
        schedule_delayed_work(&hwstats->traffic_dw,
                              msecs_to_jiffies(NSIM_DEV_HWSTATS_TRAFFIC_MS));
        return 0;

err_remove_hwstats_recursive:
        debugfs_remove_recursive(hwstats->ddir);
err_unregister_notifier:
        unregister_netdevice_notifier_net(net, &hwstats->netdevice_nb);
err_mutex_destroy:
        mutex_destroy(&hwstats->hwsdev_list_lock);
        return err;
}

static void nsim_dev_hwsdev_list_wipe(struct nsim_dev_hwstats *hwstats,
                                      enum netdev_offload_xstats_type type)
{
        struct nsim_dev_hwstats_netdev *hwsdev, *tmp;
        struct list_head *hwsdev_list;

        hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);
        if (WARN_ON(!hwsdev_list))
                return;

        mutex_lock(&hwstats->hwsdev_list_lock);
        list_for_each_entry_safe(hwsdev, tmp, hwsdev_list, list) {
                list_del(&hwsdev->list);
                nsim_dev_hwsdev_fini(hwsdev);
        }
        mutex_unlock(&hwstats->hwsdev_list_lock);
}

void nsim_dev_hwstats_exit(struct nsim_dev *nsim_dev)
{
        struct nsim_dev_hwstats *hwstats = &nsim_dev->hwstats;
        struct net *net = nsim_dev_net(nsim_dev);

        cancel_delayed_work_sync(&hwstats->traffic_dw);
        debugfs_remove_recursive(hwstats->ddir);
        unregister_netdevice_notifier_net(net, &hwstats->netdevice_nb);
        nsim_dev_hwsdev_list_wipe(hwstats, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
        mutex_destroy(&hwstats->hwsdev_list_lock);
}







































































































































































    8 





    8 






    8 






    8 
    8 












    8 
    8 







    8 




















    8 




























    8 










    8 









    8 









    8 













    8 

























    8 



    8 


    8 











    8 


    8 
    8 


    8 
    8 
    8 


    8 
    8 










































    8 



    8 







    8 





































    8 











    8 

    8 

    5 



    8 


















































































































    8 


















    8 



































































































































































































































    8 



























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
// SPDX-License-Identifier: GPL-2.0-only
/*
 * In-kernel rpcbind client supporting versions 2, 3, and 4 of the rpcbind
 * protocol
 *
 * Based on RFC 1833: "Binding Protocols for ONC RPC Version 2" and
 * RFC 3530: "Network File System (NFS) version 4 Protocol"
 *
 * Original: Gilles Quillard, Bull Open Source, 2005 <gilles.quillard@bull.net>
 * Updated: Chuck Lever, Oracle Corporation, 2007 <chuck.lever@oracle.com>
 *
 * Descended from net/sunrpc/pmap_clnt.c,
 *  Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/module.h>

#include <linux/types.h>
#include <linux/socket.h>
#include <linux/un.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <net/ipv6.h>

#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/addr.h>
#include <linux/sunrpc/sched.h>
#include <linux/sunrpc/xprtsock.h>

#include <trace/events/sunrpc.h>

#include "netns.h"

#define RPCBIND_SOCK_PATHNAME        "/var/run/rpcbind.sock"
#define RPCBIND_SOCK_ABSTRACT_NAME "\0/run/rpcbind.sock"

#define RPCBIND_PROGRAM                (100000u)
#define RPCBIND_PORT                (111u)

#define RPCBVERS_2                (2u)
#define RPCBVERS_3                (3u)
#define RPCBVERS_4                (4u)

enum {
        RPCBPROC_NULL,
        RPCBPROC_SET,
        RPCBPROC_UNSET,
        RPCBPROC_GETPORT,
        RPCBPROC_GETADDR = 3,                /* alias for GETPORT */
        RPCBPROC_DUMP,
        RPCBPROC_CALLIT,
        RPCBPROC_BCAST = 5,                /* alias for CALLIT */
        RPCBPROC_GETTIME,
        RPCBPROC_UADDR2TADDR,
        RPCBPROC_TADDR2UADDR,
        RPCBPROC_GETVERSADDR,
        RPCBPROC_INDIRECT,
        RPCBPROC_GETADDRLIST,
        RPCBPROC_GETSTAT,
};

/*
 * r_owner
 *
 * The "owner" is allowed to unset a service in the rpcbind database.
 *
 * For AF_LOCAL SET/UNSET requests, rpcbind treats this string as a
 * UID which it maps to a local user name via a password lookup.
 * In all other cases it is ignored.
 *
 * For SET/UNSET requests, user space provides a value, even for
 * network requests, and GETADDR uses an empty string.  We follow
 * those precedents here.
 */
#define RPCB_OWNER_STRING        "0"
#define RPCB_MAXOWNERLEN        sizeof(RPCB_OWNER_STRING)

/*
 * XDR data type sizes
 */
#define RPCB_program_sz                (1)
#define RPCB_version_sz                (1)
#define RPCB_protocol_sz        (1)
#define RPCB_port_sz                (1)
#define RPCB_boolean_sz                (1)

#define RPCB_netid_sz                (1 + XDR_QUADLEN(RPCBIND_MAXNETIDLEN))
#define RPCB_addr_sz                (1 + XDR_QUADLEN(RPCBIND_MAXUADDRLEN))
#define RPCB_ownerstring_sz        (1 + XDR_QUADLEN(RPCB_MAXOWNERLEN))

/*
 * XDR argument and result sizes
 */
#define RPCB_mappingargs_sz        (RPCB_program_sz + RPCB_version_sz + \
                                RPCB_protocol_sz + RPCB_port_sz)
#define RPCB_getaddrargs_sz        (RPCB_program_sz + RPCB_version_sz + \
                                RPCB_netid_sz + RPCB_addr_sz + \
                                RPCB_ownerstring_sz)

#define RPCB_getportres_sz        RPCB_port_sz
#define RPCB_setres_sz                RPCB_boolean_sz

/*
 * Note that RFC 1833 does not put any size restrictions on the
 * address string returned by the remote rpcbind database.
 */
#define RPCB_getaddrres_sz        RPCB_addr_sz

static void                        rpcb_getport_done(struct rpc_task *, void *);
static void                        rpcb_map_release(void *data);
static const struct rpc_program        rpcb_program;

struct rpcbind_args {
        struct rpc_xprt *        r_xprt;

        u32                        r_prog;
        u32                        r_vers;
        u32                        r_prot;
        unsigned short                r_port;
        const char *                r_netid;
        const char *                r_addr;
        const char *                r_owner;

        int                        r_status;
};

static const struct rpc_procinfo rpcb_procedures2[];
static const struct rpc_procinfo rpcb_procedures3[];
static const struct rpc_procinfo rpcb_procedures4[];

struct rpcb_info {
        u32                        rpc_vers;
        const struct rpc_procinfo *rpc_proc;
};

static const struct rpcb_info rpcb_next_version[];
static const struct rpcb_info rpcb_next_version6[];

static const struct rpc_call_ops rpcb_getport_ops = {
        .rpc_call_done                = rpcb_getport_done,
        .rpc_release                = rpcb_map_release,
};

static void rpcb_wake_rpcbind_waiters(struct rpc_xprt *xprt, int status)
{
        xprt_clear_binding(xprt);
        rpc_wake_up_status(&xprt->binding, status);
}

static void rpcb_map_release(void *data)
{
        struct rpcbind_args *map = data;

        rpcb_wake_rpcbind_waiters(map->r_xprt, map->r_status);
        xprt_put(map->r_xprt);
        kfree(map->r_addr);
        kfree(map);
}

static int rpcb_get_local(struct net *net)
{
        int cnt;
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);

        spin_lock(&sn->rpcb_clnt_lock);
        if (sn->rpcb_users)
                sn->rpcb_users++;
        cnt = sn->rpcb_users;
        spin_unlock(&sn->rpcb_clnt_lock);

        return cnt;
}

void rpcb_put_local(struct net *net)
{
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
        struct rpc_clnt *clnt = sn->rpcb_local_clnt;
        struct rpc_clnt *clnt4 = sn->rpcb_local_clnt4;
        int shutdown = 0;

        spin_lock(&sn->rpcb_clnt_lock);
        if (sn->rpcb_users) {
                if (--sn->rpcb_users == 0) {
                        sn->rpcb_local_clnt = NULL;
                        sn->rpcb_local_clnt4 = NULL;
                }
                shutdown = !sn->rpcb_users;
        }
        spin_unlock(&sn->rpcb_clnt_lock);

        if (shutdown) {
                /*
                 * cleanup_rpcb_clnt - remove xprtsock's sysctls, unregister
                 */
                if (clnt4)
                        rpc_shutdown_client(clnt4);
                if (clnt)
                        rpc_shutdown_client(clnt);
        }
}

static void rpcb_set_local(struct net *net, struct rpc_clnt *clnt,
                        struct rpc_clnt *clnt4,
                        bool is_af_local)
{
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);

        /* Protected by rpcb_create_local_mutex */
        sn->rpcb_local_clnt = clnt;
        sn->rpcb_local_clnt4 = clnt4;
        sn->rpcb_is_af_local = is_af_local ? 1 : 0;
        smp_wmb();
        sn->rpcb_users = 1;
}

/* Evaluate to actual length of the `sockaddr_un' structure.  */
# define SUN_LEN(ptr) (offsetof(struct sockaddr_un, sun_path)                \
                      + 1 + strlen((ptr)->sun_path + 1))

/*
 * Returns zero on success, otherwise a negative errno value
 * is returned.
 */
static int rpcb_create_af_local(struct net *net,
                                const struct sockaddr_un *addr)
{
        struct rpc_create_args args = {
                .net                = net,
                .protocol        = XPRT_TRANSPORT_LOCAL,
                .address        = (struct sockaddr *)addr,
                .addrsize        = SUN_LEN(addr),
                .servername        = "localhost",
                .program        = &rpcb_program,
                .version        = RPCBVERS_2,
                .authflavor        = RPC_AUTH_NULL,
                .cred                = current_cred(),
                /*
                 * We turn off the idle timeout to prevent the kernel
                 * from automatically disconnecting the socket.
                 * Otherwise, we'd have to cache the mount namespace
                 * of the caller and somehow pass that to the socket
                 * reconnect code.
                 */
                .flags                = RPC_CLNT_CREATE_NO_IDLE_TIMEOUT,
        };
        struct rpc_clnt *clnt, *clnt4;
        int result = 0;

        /*
         * Because we requested an RPC PING at transport creation time,
         * this works only if the user space portmapper is rpcbind, and
         * it's listening on AF_LOCAL on the named socket.
         */
        clnt = rpc_create(&args);
        if (IS_ERR(clnt)) {
                result = PTR_ERR(clnt);
                goto out;
        }

        clnt4 = rpc_bind_new_program(clnt, &rpcb_program, RPCBVERS_4);
        if (IS_ERR(clnt4))
                clnt4 = NULL;

        rpcb_set_local(net, clnt, clnt4, true);

out:
        return result;
}

static int rpcb_create_local_abstract(struct net *net)
{
        static const struct sockaddr_un rpcb_localaddr_abstract = {
                .sun_family                = AF_LOCAL,
                .sun_path                = RPCBIND_SOCK_ABSTRACT_NAME,
        };

        return rpcb_create_af_local(net, &rpcb_localaddr_abstract);
}

static int rpcb_create_local_unix(struct net *net)
{
        static const struct sockaddr_un rpcb_localaddr_unix = {
                .sun_family                = AF_LOCAL,
                .sun_path                = RPCBIND_SOCK_PATHNAME,
        };

        return rpcb_create_af_local(net, &rpcb_localaddr_unix);
}

/*
 * Returns zero on success, otherwise a negative errno value
 * is returned.
 */
static int rpcb_create_local_net(struct net *net)
{
        static const struct sockaddr_in rpcb_inaddr_loopback = {
                .sin_family                = AF_INET,
                .sin_addr.s_addr        = htonl(INADDR_LOOPBACK),
                .sin_port                = htons(RPCBIND_PORT),
        };
        struct rpc_create_args args = {
                .net                = net,
                .protocol        = XPRT_TRANSPORT_TCP,
                .address        = (struct sockaddr *)&rpcb_inaddr_loopback,
                .addrsize        = sizeof(rpcb_inaddr_loopback),
                .servername        = "localhost",
                .program        = &rpcb_program,
                .version        = RPCBVERS_2,
                .authflavor        = RPC_AUTH_UNIX,
                .cred                = current_cred(),
                .flags                = RPC_CLNT_CREATE_NOPING,
        };
        struct rpc_clnt *clnt, *clnt4;
        int result = 0;

        clnt = rpc_create(&args);
        if (IS_ERR(clnt)) {
                result = PTR_ERR(clnt);
                goto out;
        }

        /*
         * This results in an RPC ping.  On systems running portmapper,
         * the v4 ping will fail.  Proceed anyway, but disallow rpcb
         * v4 upcalls.
         */
        clnt4 = rpc_bind_new_program(clnt, &rpcb_program, RPCBVERS_4);
        if (IS_ERR(clnt4))
                clnt4 = NULL;

        rpcb_set_local(net, clnt, clnt4, false);

out:
        return result;
}

/*
 * Returns zero on success, otherwise a negative errno value
 * is returned.
 */
int rpcb_create_local(struct net *net)
{
        static DEFINE_MUTEX(rpcb_create_local_mutex);
        int result = 0;

        if (rpcb_get_local(net))
                return result;

        mutex_lock(&rpcb_create_local_mutex);
        if (rpcb_get_local(net))
                goto out;

        if (rpcb_create_local_abstract(net) != 0 &&
            rpcb_create_local_unix(net) != 0)
                result = rpcb_create_local_net(net);

out:
        mutex_unlock(&rpcb_create_local_mutex);
        return result;
}

static struct rpc_clnt *rpcb_create(struct net *net, const char *nodename,
                                    const char *hostname,
                                    struct sockaddr *srvaddr, size_t salen,
                                    int proto, u32 version,
                                    const struct cred *cred,
                                    const struct rpc_timeout *timeo)
{
        struct rpc_create_args args = {
                .net                = net,
                .protocol        = proto,
                .address        = srvaddr,
                .addrsize        = salen,
                .timeout        = timeo,
                .servername        = hostname,
                .nodename        = nodename,
                .program        = &rpcb_program,
                .version        = version,
                .authflavor        = RPC_AUTH_UNIX,
                .cred                = cred,
                .flags                = (RPC_CLNT_CREATE_NOPING |
                                        RPC_CLNT_CREATE_NONPRIVPORT),
        };

        switch (srvaddr->sa_family) {
        case AF_INET:
                ((struct sockaddr_in *)srvaddr)->sin_port = htons(RPCBIND_PORT);
                break;
        case AF_INET6:
                ((struct sockaddr_in6 *)srvaddr)->sin6_port = htons(RPCBIND_PORT);
                break;
        default:
                return ERR_PTR(-EAFNOSUPPORT);
        }

        return rpc_create(&args);
}

static int rpcb_register_call(struct sunrpc_net *sn, struct rpc_clnt *clnt, struct rpc_message *msg, bool is_set)
{
        int flags = RPC_TASK_NOCONNECT;
        int error, result = 0;

        if (is_set || !sn->rpcb_is_af_local)
                flags = RPC_TASK_SOFTCONN;
        msg->rpc_resp = &result;

        error = rpc_call_sync(clnt, msg, flags);
        if (error < 0)
                return error;

        if (!result)
                return -EACCES;
        return 0;
}

/**
 * rpcb_register - set or unset a port registration with the local rpcbind svc
 * @net: target network namespace
 * @prog: RPC program number to bind
 * @vers: RPC version number to bind
 * @prot: transport protocol to register
 * @port: port value to register
 *
 * Returns zero if the registration request was dispatched successfully
 * and the rpcbind daemon returned success.  Otherwise, returns an errno
 * value that reflects the nature of the error (request could not be
 * dispatched, timed out, or rpcbind returned an error).
 *
 * RPC services invoke this function to advertise their contact
 * information via the system's rpcbind daemon.  RPC services
 * invoke this function once for each [program, version, transport]
 * tuple they wish to advertise.
 *
 * Callers may also unregister RPC services that are no longer
 * available by setting the passed-in port to zero.  This removes
 * all registered transports for [program, version] from the local
 * rpcbind database.
 *
 * This function uses rpcbind protocol version 2 to contact the
 * local rpcbind daemon.
 *
 * Registration works over both AF_INET and AF_INET6, and services
 * registered via this function are advertised as available for any
 * address.  If the local rpcbind daemon is listening on AF_INET6,
 * services registered via this function will be advertised on
 * IN6ADDR_ANY (ie available for all AF_INET and AF_INET6
 * addresses).
 */
int rpcb_register(struct net *net, u32 prog, u32 vers, int prot, unsigned short port)
{
        struct rpcbind_args map = {
                .r_prog                = prog,
                .r_vers                = vers,
                .r_prot                = prot,
                .r_port                = port,
        };
        struct rpc_message msg = {
                .rpc_argp        = &map,
        };
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);
        bool is_set = false;

        trace_pmap_register(prog, vers, prot, port);

        msg.rpc_proc = &rpcb_procedures2[RPCBPROC_UNSET];
        if (port != 0) {
                msg.rpc_proc = &rpcb_procedures2[RPCBPROC_SET];
                is_set = true;
        }

        return rpcb_register_call(sn, sn->rpcb_local_clnt, &msg, is_set);
}

/*
 * Fill in AF_INET family-specific arguments to register
 */
static int rpcb_register_inet4(struct sunrpc_net *sn,
                               const struct sockaddr *sap,
                               struct rpc_message *msg)
{
        const struct sockaddr_in *sin = (const struct sockaddr_in *)sap;
        struct rpcbind_args *map = msg->rpc_argp;
        unsigned short port = ntohs(sin->sin_port);
        bool is_set = false;
        int result;

        map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL);

        msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET];
        if (port != 0) {
                msg->rpc_proc = &rpcb_procedures4[RPCBPROC_SET];
                is_set = true;
        }

        result = rpcb_register_call(sn, sn->rpcb_local_clnt4, msg, is_set);
        kfree(map->r_addr);
        return result;
}

/*
 * Fill in AF_INET6 family-specific arguments to register
 */
static int rpcb_register_inet6(struct sunrpc_net *sn,
                               const struct sockaddr *sap,
                               struct rpc_message *msg)
{
        const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)sap;
        struct rpcbind_args *map = msg->rpc_argp;
        unsigned short port = ntohs(sin6->sin6_port);
        bool is_set = false;
        int result;

        map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL);

        msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET];
        if (port != 0) {
                msg->rpc_proc = &rpcb_procedures4[RPCBPROC_SET];
                is_set = true;
        }

        result = rpcb_register_call(sn, sn->rpcb_local_clnt4, msg, is_set);
        kfree(map->r_addr);
        return result;
}

static int rpcb_unregister_all_protofamilies(struct sunrpc_net *sn,
                                             struct rpc_message *msg)
{
        struct rpcbind_args *map = msg->rpc_argp;

        trace_rpcb_unregister(map->r_prog, map->r_vers, map->r_netid);

        map->r_addr = "";
        msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET];

        return rpcb_register_call(sn, sn->rpcb_local_clnt4, msg, false);
}

/**
 * rpcb_v4_register - set or unset a port registration with the local rpcbind
 * @net: target network namespace
 * @program: RPC program number of service to (un)register
 * @version: RPC version number of service to (un)register
 * @address: address family, IP address, and port to (un)register
 * @netid: netid of transport protocol to (un)register
 *
 * Returns zero if the registration request was dispatched successfully
 * and the rpcbind daemon returned success.  Otherwise, returns an errno
 * value that reflects the nature of the error (request could not be
 * dispatched, timed out, or rpcbind returned an error).
 *
 * RPC services invoke this function to advertise their contact
 * information via the system's rpcbind daemon.  RPC services
 * invoke this function once for each [program, version, address,
 * netid] tuple they wish to advertise.
 *
 * Callers may also unregister RPC services that are registered at a
 * specific address by setting the port number in @address to zero.
 * They may unregister all registered protocol families at once for
 * a service by passing a NULL @address argument.  If @netid is ""
 * then all netids for [program, version, address] are unregistered.
 *
 * This function uses rpcbind protocol version 4 to contact the
 * local rpcbind daemon.  The local rpcbind daemon must support
 * version 4 of the rpcbind protocol in order for these functions
 * to register a service successfully.
 *
 * Supported netids include "udp" and "tcp" for UDP and TCP over
 * IPv4, and "udp6" and "tcp6" for UDP and TCP over IPv6,
 * respectively.
 *
 * The contents of @address determine the address family and the
 * port to be registered.  The usual practice is to pass INADDR_ANY
 * as the raw address, but specifying a non-zero address is also
 * supported by this API if the caller wishes to advertise an RPC
 * service on a specific network interface.
 *
 * Note that passing in INADDR_ANY does not create the same service
 * registration as IN6ADDR_ANY.  The former advertises an RPC
 * service on any IPv4 address, but not on IPv6.  The latter
 * advertises the service on all IPv4 and IPv6 addresses.
 */
int rpcb_v4_register(struct net *net, const u32 program, const u32 version,
                     const struct sockaddr *address, const char *netid)
{
        struct rpcbind_args map = {
                .r_prog                = program,
                .r_vers                = version,
                .r_netid        = netid,
                .r_owner        = RPCB_OWNER_STRING,
        };
        struct rpc_message msg = {
                .rpc_argp        = &map,
        };
        struct sunrpc_net *sn = net_generic(net, sunrpc_net_id);

        if (sn->rpcb_local_clnt4 == NULL)
                return -EPROTONOSUPPORT;

        if (address == NULL)
                return rpcb_unregister_all_protofamilies(sn, &msg);

        trace_rpcb_register(map.r_prog, map.r_vers, map.r_addr, map.r_netid);

        switch (address->sa_family) {
        case AF_INET:
                return rpcb_register_inet4(sn, address, &msg);
        case AF_INET6:
                return rpcb_register_inet6(sn, address, &msg);
        }

        return -EAFNOSUPPORT;
}

static struct rpc_task *rpcb_call_async(struct rpc_clnt *rpcb_clnt,
                struct rpcbind_args *map, const struct rpc_procinfo *proc)
{
        struct rpc_message msg = {
                .rpc_proc = proc,
                .rpc_argp = map,
                .rpc_resp = map,
        };
        struct rpc_task_setup task_setup_data = {
                .rpc_client = rpcb_clnt,
                .rpc_message = &msg,
                .callback_ops = &rpcb_getport_ops,
                .callback_data = map,
                .flags = RPC_TASK_ASYNC | RPC_TASK_SOFTCONN,
        };

        return rpc_run_task(&task_setup_data);
}

/*
 * In the case where rpc clients have been cloned, we want to make
 * sure that we use the program number/version etc of the actual
 * owner of the xprt. To do so, we walk back up the tree of parents
 * to find whoever created the transport and/or whoever has the
 * autobind flag set.
 */
static struct rpc_clnt *rpcb_find_transport_owner(struct rpc_clnt *clnt)
{
        struct rpc_clnt *parent = clnt->cl_parent;
        struct rpc_xprt_switch *xps = rcu_access_pointer(clnt->cl_xpi.xpi_xpswitch);

        while (parent != clnt) {
                if (rcu_access_pointer(parent->cl_xpi.xpi_xpswitch) != xps)
                        break;
                if (clnt->cl_autobind)
                        break;
                clnt = parent;
                parent = parent->cl_parent;
        }
        return clnt;
}

/**
 * rpcb_getport_async - obtain the port for a given RPC service on a given host
 * @task: task that is waiting for portmapper request
 *
 * This one can be called for an ongoing RPC request, and can be used in
 * an async (rpciod) context.
 */
void rpcb_getport_async(struct rpc_task *task)
{
        struct rpc_clnt *clnt;
        const struct rpc_procinfo *proc;
        u32 bind_version;
        struct rpc_xprt *xprt;
        struct rpc_clnt        *rpcb_clnt;
        struct rpcbind_args *map;
        struct rpc_task        *child;
        struct sockaddr_storage addr;
        struct sockaddr *sap = (struct sockaddr *)&addr;
        size_t salen;
        int status;

        rcu_read_lock();
        clnt = rpcb_find_transport_owner(task->tk_client);
        rcu_read_unlock();
        xprt = xprt_get(task->tk_xprt);

        /* Put self on the wait queue to ensure we get notified if
         * some other task is already attempting to bind the port */
        rpc_sleep_on_timeout(&xprt->binding, task,
                        NULL, jiffies + xprt->bind_timeout);

        if (xprt_test_and_set_binding(xprt)) {
                xprt_put(xprt);
                return;
        }

        /* Someone else may have bound if we slept */
        if (xprt_bound(xprt)) {
                status = 0;
                goto bailout_nofree;
        }

        /* Parent transport's destination address */
        salen = rpc_peeraddr(clnt, sap, sizeof(addr));

        /* Don't ever use rpcbind v2 for AF_INET6 requests */
        switch (sap->sa_family) {
        case AF_INET:
                proc = rpcb_next_version[xprt->bind_index].rpc_proc;
                bind_version = rpcb_next_version[xprt->bind_index].rpc_vers;
                break;
        case AF_INET6:
                proc = rpcb_next_version6[xprt->bind_index].rpc_proc;
                bind_version = rpcb_next_version6[xprt->bind_index].rpc_vers;
                break;
        default:
                status = -EAFNOSUPPORT;
                goto bailout_nofree;
        }
        if (proc == NULL) {
                xprt->bind_index = 0;
                status = -EPFNOSUPPORT;
                goto bailout_nofree;
        }

        trace_rpcb_getport(clnt, task, bind_version);

        rpcb_clnt = rpcb_create(xprt->xprt_net,
                                clnt->cl_nodename,
                                xprt->servername, sap, salen,
                                xprt->prot, bind_version,
                                clnt->cl_cred,
                                task->tk_client->cl_timeout);
        if (IS_ERR(rpcb_clnt)) {
                status = PTR_ERR(rpcb_clnt);
                goto bailout_nofree;
        }

        map = kzalloc(sizeof(struct rpcbind_args), rpc_task_gfp_mask());
        if (!map) {
                status = -ENOMEM;
                goto bailout_release_client;
        }
        map->r_prog = clnt->cl_prog;
        map->r_vers = clnt->cl_vers;
        map->r_prot = xprt->prot;
        map->r_port = 0;
        map->r_xprt = xprt;
        map->r_status = -EIO;

        switch (bind_version) {
        case RPCBVERS_4:
        case RPCBVERS_3:
                map->r_netid = xprt->address_strings[RPC_DISPLAY_NETID];
                map->r_addr = rpc_sockaddr2uaddr(sap, rpc_task_gfp_mask());
                if (!map->r_addr) {
                        status = -ENOMEM;
                        goto bailout_free_args;
                }
                map->r_owner = "";
                break;
        case RPCBVERS_2:
                map->r_addr = NULL;
                break;
        default:
                BUG();
        }

        child = rpcb_call_async(rpcb_clnt, map, proc);
        rpc_release_client(rpcb_clnt);
        if (IS_ERR(child)) {
                /* rpcb_map_release() has freed the arguments */
                return;
        }

        xprt->stat.bind_count++;
        rpc_put_task(child);
        return;

bailout_free_args:
        kfree(map);
bailout_release_client:
        rpc_release_client(rpcb_clnt);
bailout_nofree:
        rpcb_wake_rpcbind_waiters(xprt, status);
        task->tk_status = status;
        xprt_put(xprt);
}
EXPORT_SYMBOL_GPL(rpcb_getport_async);

/*
 * Rpcbind child task calls this callback via tk_exit.
 */
static void rpcb_getport_done(struct rpc_task *child, void *data)
{
        struct rpcbind_args *map = data;
        struct rpc_xprt *xprt = map->r_xprt;

        map->r_status = child->tk_status;

        /* Garbage reply: retry with a lesser rpcbind version */
        if (map->r_status == -EIO)
                map->r_status = -EPROTONOSUPPORT;

        /* rpcbind server doesn't support this rpcbind protocol version */
        if (map->r_status == -EPROTONOSUPPORT)
                xprt->bind_index++;

        if (map->r_status < 0) {
                /* rpcbind server not available on remote host? */
                map->r_port = 0;

        } else if (map->r_port == 0) {
                /* Requested RPC service wasn't registered on remote host */
                map->r_status = -EACCES;
        } else {
                /* Succeeded */
                map->r_status = 0;
        }

        trace_rpcb_setport(child, map->r_status, map->r_port);
        xprt->ops->set_port(xprt, map->r_port);
        if (map->r_port)
                xprt_set_bound(xprt);
}

/*
 * XDR functions for rpcbind
 */

static void rpcb_enc_mapping(struct rpc_rqst *req, struct xdr_stream *xdr,
                             const void *data)
{
        const struct rpcbind_args *rpcb = data;
        __be32 *p;

        p = xdr_reserve_space(xdr, RPCB_mappingargs_sz << 2);
        *p++ = cpu_to_be32(rpcb->r_prog);
        *p++ = cpu_to_be32(rpcb->r_vers);
        *p++ = cpu_to_be32(rpcb->r_prot);
        *p   = cpu_to_be32(rpcb->r_port);
}

static int rpcb_dec_getport(struct rpc_rqst *req, struct xdr_stream *xdr,
                            void *data)
{
        struct rpcbind_args *rpcb = data;
        unsigned long port;
        __be32 *p;

        rpcb->r_port = 0;

        p = xdr_inline_decode(xdr, 4);
        if (unlikely(p == NULL))
                return -EIO;

        port = be32_to_cpup(p);
        if (unlikely(port > USHRT_MAX))
                return -EIO;

        rpcb->r_port = port;
        return 0;
}

static int rpcb_dec_set(struct rpc_rqst *req, struct xdr_stream *xdr,
                        void *data)
{
        unsigned int *boolp = data;
        __be32 *p;

        p = xdr_inline_decode(xdr, 4);
        if (unlikely(p == NULL))
                return -EIO;

        *boolp = 0;
        if (*p != xdr_zero)
                *boolp = 1;
        return 0;
}

static void encode_rpcb_string(struct xdr_stream *xdr, const char *string,
                               const u32 maxstrlen)
{
        __be32 *p;
        u32 len;

        len = strlen(string);
        WARN_ON_ONCE(len > maxstrlen);
        if (len > maxstrlen)
                /* truncate and hope for the best */
                len = maxstrlen;
        p = xdr_reserve_space(xdr, 4 + len);
        xdr_encode_opaque(p, string, len);
}

static void rpcb_enc_getaddr(struct rpc_rqst *req, struct xdr_stream *xdr,
                             const void *data)
{
        const struct rpcbind_args *rpcb = data;
        __be32 *p;

        p = xdr_reserve_space(xdr, (RPCB_program_sz + RPCB_version_sz) << 2);
        *p++ = cpu_to_be32(rpcb->r_prog);
        *p = cpu_to_be32(rpcb->r_vers);

        encode_rpcb_string(xdr, rpcb->r_netid, RPCBIND_MAXNETIDLEN);
        encode_rpcb_string(xdr, rpcb->r_addr, RPCBIND_MAXUADDRLEN);
        encode_rpcb_string(xdr, rpcb->r_owner, RPCB_MAXOWNERLEN);
}

static int rpcb_dec_getaddr(struct rpc_rqst *req, struct xdr_stream *xdr,
                            void *data)
{
        struct rpcbind_args *rpcb = data;
        struct sockaddr_storage address;
        struct sockaddr *sap = (struct sockaddr *)&address;
        __be32 *p;
        u32 len;

        rpcb->r_port = 0;

        p = xdr_inline_decode(xdr, 4);
        if (unlikely(p == NULL))
                goto out_fail;
        len = be32_to_cpup(p);

        /*
         * If the returned universal address is a null string,
         * the requested RPC service was not registered.
         */
        if (len == 0)
                return 0;

        if (unlikely(len > RPCBIND_MAXUADDRLEN))
                goto out_fail;

        p = xdr_inline_decode(xdr, len);
        if (unlikely(p == NULL))
                goto out_fail;

        if (rpc_uaddr2sockaddr(req->rq_xprt->xprt_net, (char *)p, len,
                                sap, sizeof(address)) == 0)
                goto out_fail;
        rpcb->r_port = rpc_get_port(sap);

        return 0;

out_fail:
        return -EIO;
}

/*
 * Not all rpcbind procedures described in RFC 1833 are implemented
 * since the Linux kernel RPC code requires only these.
 */

static const struct rpc_procinfo rpcb_procedures2[] = {
        [RPCBPROC_SET] = {
                .p_proc                = RPCBPROC_SET,
                .p_encode        = rpcb_enc_mapping,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_mappingargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_SET,
                .p_timer        = 0,
                .p_name                = "SET",
        },
        [RPCBPROC_UNSET] = {
                .p_proc                = RPCBPROC_UNSET,
                .p_encode        = rpcb_enc_mapping,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_mappingargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_UNSET,
                .p_timer        = 0,
                .p_name                = "UNSET",
        },
        [RPCBPROC_GETPORT] = {
                .p_proc                = RPCBPROC_GETPORT,
                .p_encode        = rpcb_enc_mapping,
                .p_decode        = rpcb_dec_getport,
                .p_arglen        = RPCB_mappingargs_sz,
                .p_replen        = RPCB_getportres_sz,
                .p_statidx        = RPCBPROC_GETPORT,
                .p_timer        = 0,
                .p_name                = "GETPORT",
        },
};

static const struct rpc_procinfo rpcb_procedures3[] = {
        [RPCBPROC_SET] = {
                .p_proc                = RPCBPROC_SET,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_SET,
                .p_timer        = 0,
                .p_name                = "SET",
        },
        [RPCBPROC_UNSET] = {
                .p_proc                = RPCBPROC_UNSET,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_UNSET,
                .p_timer        = 0,
                .p_name                = "UNSET",
        },
        [RPCBPROC_GETADDR] = {
                .p_proc                = RPCBPROC_GETADDR,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_getaddr,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_getaddrres_sz,
                .p_statidx        = RPCBPROC_GETADDR,
                .p_timer        = 0,
                .p_name                = "GETADDR",
        },
};

static const struct rpc_procinfo rpcb_procedures4[] = {
        [RPCBPROC_SET] = {
                .p_proc                = RPCBPROC_SET,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_SET,
                .p_timer        = 0,
                .p_name                = "SET",
        },
        [RPCBPROC_UNSET] = {
                .p_proc                = RPCBPROC_UNSET,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_set,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_setres_sz,
                .p_statidx        = RPCBPROC_UNSET,
                .p_timer        = 0,
                .p_name                = "UNSET",
        },
        [RPCBPROC_GETADDR] = {
                .p_proc                = RPCBPROC_GETADDR,
                .p_encode        = rpcb_enc_getaddr,
                .p_decode        = rpcb_dec_getaddr,
                .p_arglen        = RPCB_getaddrargs_sz,
                .p_replen        = RPCB_getaddrres_sz,
                .p_statidx        = RPCBPROC_GETADDR,
                .p_timer        = 0,
                .p_name                = "GETADDR",
        },
};

static const struct rpcb_info rpcb_next_version[] = {
        {
                .rpc_vers        = RPCBVERS_2,
                .rpc_proc        = &rpcb_procedures2[RPCBPROC_GETPORT],
        },
        {
                .rpc_proc        = NULL,
        },
};

static const struct rpcb_info rpcb_next_version6[] = {
        {
                .rpc_vers        = RPCBVERS_4,
                .rpc_proc        = &rpcb_procedures4[RPCBPROC_GETADDR],
        },
        {
                .rpc_vers        = RPCBVERS_3,
                .rpc_proc        = &rpcb_procedures3[RPCBPROC_GETADDR],
        },
        {
                .rpc_proc        = NULL,
        },
};

static unsigned int rpcb_version2_counts[ARRAY_SIZE(rpcb_procedures2)];
static const struct rpc_version rpcb_version2 = {
        .number                = RPCBVERS_2,
        .nrprocs        = ARRAY_SIZE(rpcb_procedures2),
        .procs                = rpcb_procedures2,
        .counts                = rpcb_version2_counts,
};

static unsigned int rpcb_version3_counts[ARRAY_SIZE(rpcb_procedures3)];
static const struct rpc_version rpcb_version3 = {
        .number                = RPCBVERS_3,
        .nrprocs        = ARRAY_SIZE(rpcb_procedures3),
        .procs                = rpcb_procedures3,
        .counts                = rpcb_version3_counts,
};

static unsigned int rpcb_version4_counts[ARRAY_SIZE(rpcb_procedures4)];
static const struct rpc_version rpcb_version4 = {
        .number                = RPCBVERS_4,
        .nrprocs        = ARRAY_SIZE(rpcb_procedures4),
        .procs                = rpcb_procedures4,
        .counts                = rpcb_version4_counts,
};

static const struct rpc_version *rpcb_version[] = {
        NULL,
        NULL,
        &rpcb_version2,
        &rpcb_version3,
        &rpcb_version4
};

static struct rpc_stat rpcb_stats;

static const struct rpc_program rpcb_program = {
        .name                = "rpcbind",
        .number                = RPCBIND_PROGRAM,
        .nrvers                = ARRAY_SIZE(rpcb_version),
        .version        = rpcb_version,
        .stats                = &rpcb_stats,
};





























   15 
   15 










   81 

   80 
   81 
   79 



    1 

    2 


    2 
    1 

    1 
    1 


    2 



























































































































































  208 



  206 
  209 
  209 

  208 
  209 





  209 






   36 

   36 
   36 


   36 
   35 


























  107 

  106 
  107 













































































































































   15 


   15 
   15 
   13 

    2 


   15 
   10 



    5 








   15 














    2 


    2 











    7 




    7 
    7 
    4 


    4 

    7 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Pluggable TCP congestion control support and newReno
 * congestion control.
 * Based on ideas from I/O scheduler support and Web100.
 *
 * Copyright (C) 2005 Stephen Hemminger <shemminger@osdl.org>
 */

#define pr_fmt(fmt) "TCP: " fmt

#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/list.h>
#include <linux/gfp.h>
#include <linux/jhash.h>
#include <net/tcp.h>
#include <trace/events/tcp.h>

static DEFINE_SPINLOCK(tcp_cong_list_lock);
static LIST_HEAD(tcp_cong_list);

/* Simple linear search, don't expect many entries! */
struct tcp_congestion_ops *tcp_ca_find(const char *name)
{
        struct tcp_congestion_ops *e;

        list_for_each_entry_rcu(e, &tcp_cong_list, list) {
                if (strcmp(e->name, name) == 0)
                        return e;
        }

        return NULL;
}

void tcp_set_ca_state(struct sock *sk, const u8 ca_state)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        trace_tcp_cong_state_set(sk, ca_state);

        if (icsk->icsk_ca_ops->set_state)
                icsk->icsk_ca_ops->set_state(sk, ca_state);
        icsk->icsk_ca_state = ca_state;
}

/* Must be called with rcu lock held */
static struct tcp_congestion_ops *tcp_ca_find_autoload(const char *name)
{
        struct tcp_congestion_ops *ca = tcp_ca_find(name);

#ifdef CONFIG_MODULES
        if (!ca && capable(CAP_NET_ADMIN)) {
                rcu_read_unlock();
                request_module("tcp_%s", name);
                rcu_read_lock();
                ca = tcp_ca_find(name);
        }
#endif
        return ca;
}

/* Simple linear search, not much in here. */
struct tcp_congestion_ops *tcp_ca_find_key(u32 key)
{
        struct tcp_congestion_ops *e;

        list_for_each_entry_rcu(e, &tcp_cong_list, list) {
                if (e->key == key)
                        return e;
        }

        return NULL;
}

int tcp_validate_congestion_control(struct tcp_congestion_ops *ca)
{
        /* all algorithms must implement these */
        if (!ca->ssthresh || !ca->undo_cwnd ||
            !(ca->cong_avoid || ca->cong_control)) {
                pr_err("%s does not implement required ops\n", ca->name);
                return -EINVAL;
        }

        return 0;
}

/* Attach new congestion control algorithm to the list
 * of available options.
 */
int tcp_register_congestion_control(struct tcp_congestion_ops *ca)
{
        int ret;

        ret = tcp_validate_congestion_control(ca);
        if (ret)
                return ret;

        ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));

        spin_lock(&tcp_cong_list_lock);
        if (ca->key == TCP_CA_UNSPEC || tcp_ca_find_key(ca->key)) {
                pr_notice("%s already registered or non-unique key\n",
                          ca->name);
                ret = -EEXIST;
        } else {
                list_add_tail_rcu(&ca->list, &tcp_cong_list);
                pr_debug("%s registered\n", ca->name);
        }
        spin_unlock(&tcp_cong_list_lock);

        return ret;
}
EXPORT_SYMBOL_GPL(tcp_register_congestion_control);

/*
 * Remove congestion control algorithm, called from
 * the module's remove function.  Module ref counts are used
 * to ensure that this can't be done till all sockets using
 * that method are closed.
 */
void tcp_unregister_congestion_control(struct tcp_congestion_ops *ca)
{
        spin_lock(&tcp_cong_list_lock);
        list_del_rcu(&ca->list);
        spin_unlock(&tcp_cong_list_lock);

        /* Wait for outstanding readers to complete before the
         * module gets removed entirely.
         *
         * A try_module_get() should fail by now as our module is
         * in "going" state since no refs are held anymore and
         * module_exit() handler being called.
         */
        synchronize_rcu();
}
EXPORT_SYMBOL_GPL(tcp_unregister_congestion_control);

/* Replace a registered old ca with a new one.
 *
 * The new ca must have the same name as the old one, that has been
 * registered.
 */
int tcp_update_congestion_control(struct tcp_congestion_ops *ca, struct tcp_congestion_ops *old_ca)
{
        struct tcp_congestion_ops *existing;
        int ret = 0;

        ca->key = jhash(ca->name, sizeof(ca->name), strlen(ca->name));

        spin_lock(&tcp_cong_list_lock);
        existing = tcp_ca_find_key(old_ca->key);
        if (ca->key == TCP_CA_UNSPEC || !existing || strcmp(existing->name, ca->name)) {
                pr_notice("%s not registered or non-unique key\n",
                          ca->name);
                ret = -EINVAL;
        } else if (existing != old_ca) {
                pr_notice("invalid old congestion control algorithm to replace\n");
                ret = -EINVAL;
        } else {
                /* Add the new one before removing the old one to keep
                 * one implementation available all the time.
                 */
                list_add_tail_rcu(&ca->list, &tcp_cong_list);
                list_del_rcu(&existing->list);
                pr_debug("%s updated\n", ca->name);
        }
        spin_unlock(&tcp_cong_list_lock);

        /* Wait for outstanding readers to complete before the
         * module or struct_ops gets removed entirely.
         */
        if (!ret)
                synchronize_rcu();

        return ret;
}

u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca)
{
        const struct tcp_congestion_ops *ca;
        u32 key = TCP_CA_UNSPEC;

        might_sleep();

        rcu_read_lock();
        ca = tcp_ca_find_autoload(name);
        if (ca) {
                key = ca->key;
                *ecn_ca = ca->flags & TCP_CONG_NEEDS_ECN;
        }
        rcu_read_unlock();

        return key;
}

char *tcp_ca_get_name_by_key(u32 key, char *buffer)
{
        const struct tcp_congestion_ops *ca;
        char *ret = NULL;

        rcu_read_lock();
        ca = tcp_ca_find_key(key);
        if (ca) {
                strscpy(buffer, ca->name, TCP_CA_NAME_MAX);
                ret = buffer;
        }
        rcu_read_unlock();

        return ret;
}

/* Assign choice of congestion control. */
void tcp_assign_congestion_control(struct sock *sk)
{
        struct net *net = sock_net(sk);
        struct inet_connection_sock *icsk = inet_csk(sk);
        const struct tcp_congestion_ops *ca;

        rcu_read_lock();
        ca = rcu_dereference(net->ipv4.tcp_congestion_control);
        if (unlikely(!bpf_try_module_get(ca, ca->owner)))
                ca = &tcp_reno;
        icsk->icsk_ca_ops = ca;
        rcu_read_unlock();

        memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
        if (ca->flags & TCP_CONG_NEEDS_ECN)
                INET_ECN_xmit(sk);
        else
                INET_ECN_dontxmit(sk);
}

void tcp_init_congestion_control(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        tcp_sk(sk)->prior_ssthresh = 0;
        if (icsk->icsk_ca_ops->init)
                icsk->icsk_ca_ops->init(sk);
        if (tcp_ca_needs_ecn(sk))
                INET_ECN_xmit(sk);
        else
                INET_ECN_dontxmit(sk);
        icsk->icsk_ca_initialized = 1;
}

static void tcp_reinit_congestion_control(struct sock *sk,
                                          const struct tcp_congestion_ops *ca)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        tcp_cleanup_congestion_control(sk);
        icsk->icsk_ca_ops = ca;
        icsk->icsk_ca_setsockopt = 1;
        memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));

        if (ca->flags & TCP_CONG_NEEDS_ECN)
                INET_ECN_xmit(sk);
        else
                INET_ECN_dontxmit(sk);

        if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
                tcp_init_congestion_control(sk);
}

/* Manage refcounts on socket close. */
void tcp_cleanup_congestion_control(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        if (icsk->icsk_ca_initialized && icsk->icsk_ca_ops->release)
                icsk->icsk_ca_ops->release(sk);
        icsk->icsk_ca_initialized = 0;
        bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
}

/* Used by sysctl to change default congestion control */
int tcp_set_default_congestion_control(struct net *net, const char *name)
{
        struct tcp_congestion_ops *ca;
        const struct tcp_congestion_ops *prev;
        int ret;

        rcu_read_lock();
        ca = tcp_ca_find_autoload(name);
        if (!ca) {
                ret = -ENOENT;
        } else if (!bpf_try_module_get(ca, ca->owner)) {
                ret = -EBUSY;
        } else if (!net_eq(net, &init_net) &&
                        !(ca->flags & TCP_CONG_NON_RESTRICTED)) {
                /* Only init netns can set default to a restricted algorithm */
                ret = -EPERM;
        } else {
                prev = xchg(&net->ipv4.tcp_congestion_control, ca);
                if (prev)
                        bpf_module_put(prev, prev->owner);

                ca->flags |= TCP_CONG_NON_RESTRICTED;
                ret = 0;
        }
        rcu_read_unlock();

        return ret;
}

/* Set default value from kernel configuration at bootup */
static int __init tcp_congestion_default(void)
{
        return tcp_set_default_congestion_control(&init_net,
                                                  CONFIG_DEFAULT_TCP_CONG);
}
late_initcall(tcp_congestion_default);

/* Build string with list of available congestion control values */
void tcp_get_available_congestion_control(char *buf, size_t maxlen)
{
        struct tcp_congestion_ops *ca;
        size_t offs = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
                offs += snprintf(buf + offs, maxlen - offs,
                                 "%s%s",
                                 offs == 0 ? "" : " ", ca->name);

                if (WARN_ON_ONCE(offs >= maxlen))
                        break;
        }
        rcu_read_unlock();
}

/* Get current default congestion control */
void tcp_get_default_congestion_control(struct net *net, char *name)
{
        const struct tcp_congestion_ops *ca;

        rcu_read_lock();
        ca = rcu_dereference(net->ipv4.tcp_congestion_control);
        strscpy(name, ca->name, TCP_CA_NAME_MAX);
        rcu_read_unlock();
}

/* Built list of non-restricted congestion control values */
void tcp_get_allowed_congestion_control(char *buf, size_t maxlen)
{
        struct tcp_congestion_ops *ca;
        size_t offs = 0;

        *buf = '\0';
        rcu_read_lock();
        list_for_each_entry_rcu(ca, &tcp_cong_list, list) {
                if (!(ca->flags & TCP_CONG_NON_RESTRICTED))
                        continue;
                offs += snprintf(buf + offs, maxlen - offs,
                                 "%s%s",
                                 offs == 0 ? "" : " ", ca->name);

                if (WARN_ON_ONCE(offs >= maxlen))
                        break;
        }
        rcu_read_unlock();
}

/* Change list of non-restricted congestion control */
int tcp_set_allowed_congestion_control(char *val)
{
        struct tcp_congestion_ops *ca;
        char *saved_clone, *clone, *name;
        int ret = 0;

        saved_clone = clone = kstrdup(val, GFP_USER);
        if (!clone)
                return -ENOMEM;

        spin_lock(&tcp_cong_list_lock);
        /* pass 1 check for bad entries */
        while ((name = strsep(&clone, " ")) && *name) {
                ca = tcp_ca_find(name);
                if (!ca) {
                        ret = -ENOENT;
                        goto out;
                }
        }

        /* pass 2 clear old values */
        list_for_each_entry_rcu(ca, &tcp_cong_list, list)
                ca->flags &= ~TCP_CONG_NON_RESTRICTED;

        /* pass 3 mark as allowed */
        while ((name = strsep(&val, " ")) && *name) {
                ca = tcp_ca_find(name);
                WARN_ON(!ca);
                if (ca)
                        ca->flags |= TCP_CONG_NON_RESTRICTED;
        }
out:
        spin_unlock(&tcp_cong_list_lock);
        kfree(saved_clone);

        return ret;
}

/* Change congestion control for socket. If load is false, then it is the
 * responsibility of the caller to call tcp_init_congestion_control or
 * tcp_reinit_congestion_control (if the current congestion control was
 * already initialized.
 */
int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
                               bool cap_net_admin)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        const struct tcp_congestion_ops *ca;
        int err = 0;

        if (icsk->icsk_ca_dst_locked)
                return -EPERM;

        rcu_read_lock();
        if (!load)
                ca = tcp_ca_find(name);
        else
                ca = tcp_ca_find_autoload(name);

        /* No change asking for existing value */
        if (ca == icsk->icsk_ca_ops) {
                icsk->icsk_ca_setsockopt = 1;
                goto out;
        }

        if (!ca)
                err = -ENOENT;
        else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) || cap_net_admin))
                err = -EPERM;
        else if (!bpf_try_module_get(ca, ca->owner))
                err = -EBUSY;
        else
                tcp_reinit_congestion_control(sk, ca);
 out:
        rcu_read_unlock();
        return err;
}

/* Slow start is used when congestion window is no greater than the slow start
 * threshold. We base on RFC2581 and also handle stretch ACKs properly.
 * We do not implement RFC3465 Appropriate Byte Counting (ABC) per se but
 * something better;) a packet is only considered (s)acked in its entirety to
 * defend the ACK attacks described in the RFC. Slow start processes a stretch
 * ACK of degree N as if N acks of degree 1 are received back to back except
 * ABC caps N to 2. Slow start exits when cwnd grows over ssthresh and
 * returns the leftover acks to adjust cwnd in congestion avoidance mode.
 */
__bpf_kfunc u32 tcp_slow_start(struct tcp_sock *tp, u32 acked)
{
        u32 cwnd = min(tcp_snd_cwnd(tp) + acked, tp->snd_ssthresh);

        acked -= cwnd - tcp_snd_cwnd(tp);
        tcp_snd_cwnd_set(tp, min(cwnd, tp->snd_cwnd_clamp));

        return acked;
}
EXPORT_SYMBOL_GPL(tcp_slow_start);

/* In theory this is tp->snd_cwnd += 1 / tp->snd_cwnd (or alternative w),
 * for every packet that was ACKed.
 */
__bpf_kfunc void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked)
{
        /* If credits accumulated at a higher w, apply them gently now. */
        if (tp->snd_cwnd_cnt >= w) {
                tp->snd_cwnd_cnt = 0;
                tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
        }

        tp->snd_cwnd_cnt += acked;
        if (tp->snd_cwnd_cnt >= w) {
                u32 delta = tp->snd_cwnd_cnt / w;

                tp->snd_cwnd_cnt -= delta * w;
                tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + delta);
        }
        tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), tp->snd_cwnd_clamp));
}
EXPORT_SYMBOL_GPL(tcp_cong_avoid_ai);

/*
 * TCP Reno congestion control
 * This is special case used for fallback as well.
 */
/* This is Jacobson's slow start and congestion avoidance.
 * SIGCOMM '88, p. 328.
 */
__bpf_kfunc void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked)
{
        struct tcp_sock *tp = tcp_sk(sk);

        if (!tcp_is_cwnd_limited(sk))
                return;

        /* In "safe" area, increase. */
        if (tcp_in_slow_start(tp)) {
                acked = tcp_slow_start(tp, acked);
                if (!acked)
                        return;
        }
        /* In dangerous area, increase slowly. */
        tcp_cong_avoid_ai(tp, tcp_snd_cwnd(tp), acked);
}
EXPORT_SYMBOL_GPL(tcp_reno_cong_avoid);

/* Slow start threshold is half the congestion window (min 2) */
__bpf_kfunc u32 tcp_reno_ssthresh(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);

        return max(tcp_snd_cwnd(tp) >> 1U, 2U);
}
EXPORT_SYMBOL_GPL(tcp_reno_ssthresh);

__bpf_kfunc u32 tcp_reno_undo_cwnd(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);

        return max(tcp_snd_cwnd(tp), tp->prior_cwnd);
}
EXPORT_SYMBOL_GPL(tcp_reno_undo_cwnd);

struct tcp_congestion_ops tcp_reno = {
        .flags                = TCP_CONG_NON_RESTRICTED,
        .name                = "reno",
        .owner                = THIS_MODULE,
        .ssthresh        = tcp_reno_ssthresh,
        .cong_avoid        = tcp_reno_cong_avoid,
        .undo_cwnd        = tcp_reno_undo_cwnd,
};





























































































































































































   80 



   44 
   81 




    3 


   32 
    6 











  113 










  106 





    3 

  106 
  107 
  107 
  106 




  107 



  107 
  107 


    1 


















  106 
   29 

   78 
   52 

   29 
   22 
    7 



  105 
   51 



   70 
   85 

   22 

  107 









   23 









   23 


   23 




   22 
















   23 






   23 
   23 
   23 



   23 


   23 
   23 
   22 
















   23 


    1 

   23 






   23 






   23 
   12 
   23 

































































































































































   32 






   32 



   32 
   32 
   32 




   32 
   32 




   32 












   31 

   31 











































   31 

   22 
   31 

   31 

   32 


    1 






   22 








   22 












































































































































































































































































































    2 





    3 
    3 



    2 
    2 



    2 


    1 










    1 
    3 

    2 














    3 

    1 
    3 







    3 




    9 





    9 
    1 

    8 


    3 









    9 










    1 
    1 
    1 











    3 

    3 
    1 




    2 
    2 


    5 

















    3 
    6 



























































    3 




    2 

    2 

    2 


























    6 
















    6 








    4 


    3 




    1 











    4 
    4 












   18 






   13 



   17 




   18 


   16 


   15 
    9 



   14 
   11 
    3 


   13 

    5 
    4 
    9 


    4 
    9 



    4 
    4 
    4 












  203 






    3 




   27 
    8 

   27 



   27 

   27 






  203 












   57 


































































































































   70 


   70 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
// SPDX-License-Identifier: GPL-2.0-or-later
/* linux/net/ipv4/arp.c
 *
 * Copyright (C) 1994 by Florian  La Roche
 *
 * This module implements the Address Resolution Protocol ARP (RFC 826),
 * which is used to convert IP addresses (or in the future maybe other
 * high-level addresses) into a low-level hardware address (like an Ethernet
 * address).
 *
 * Fixes:
 *                Alan Cox        :        Removed the Ethernet assumptions in
 *                                        Florian's code
 *                Alan Cox        :        Fixed some small errors in the ARP
 *                                        logic
 *                Alan Cox        :        Allow >4K in /proc
 *                Alan Cox        :        Make ARP add its own protocol entry
 *                Ross Martin     :       Rewrote arp_rcv() and arp_get_info()
 *                Stephen Henson        :        Add AX25 support to arp_get_info()
 *                Alan Cox        :        Drop data when a device is downed.
 *                Alan Cox        :        Use init_timer().
 *                Alan Cox        :        Double lock fixes.
 *                Martin Seine        :        Move the arphdr structure
 *                                        to if_arp.h for compatibility.
 *                                        with BSD based programs.
 *                Andrew Tridgell :       Added ARP netmask code and
 *                                        re-arranged proxy handling.
 *                Alan Cox        :        Changed to use notifiers.
 *                Niibe Yutaka        :        Reply for this device or proxies only.
 *                Alan Cox        :        Don't proxy across hardware types!
 *                Jonathan Naylor :        Added support for NET/ROM.
 *                Mike Shaver     :       RFC1122 checks.
 *                Jonathan Naylor :        Only lookup the hardware address for
 *                                        the correct hardware type.
 *                Germano Caronni        :        Assorted subtle races.
 *                Craig Schlenter :        Don't modify permanent entry
 *                                        during arp_rcv.
 *                Russ Nelson        :        Tidied up a few bits.
 *                Alexey Kuznetsov:        Major changes to caching and behaviour,
 *                                        eg intelligent arp probing and
 *                                        generation
 *                                        of host down events.
 *                Alan Cox        :        Missing unlock in device events.
 *                Eckes                :        ARP ioctl control errors.
 *                Alexey Kuznetsov:        Arp free fix.
 *                Manuel Rodriguez:        Gratuitous ARP.
 *              Jonathan Layes  :       Added arpd support through kerneld
 *                                      message queue (960314)
 *                Mike Shaver        :        /proc/sys/net/ipv4/arp_* support
 *                Mike McLagan    :        Routing by source
 *                Stuart Cheshire        :        Metricom and grat arp fixes
 *                                        *** FOR 2.1 clean this up ***
 *                Lawrence V. Stefani: (08/12/96) Added FDDI support.
 *                Alan Cox        :        Took the AP1000 nasty FDDI hack and
 *                                        folded into the mainstream FDDI code.
 *                                        Ack spit, Linus how did you allow that
 *                                        one in...
 *                Jes Sorensen        :        Make FDDI work again in 2.1.x and
 *                                        clean up the APFDDI & gen. FDDI bits.
 *                Alexey Kuznetsov:        new arp state machine;
 *                                        now it is in net/core/neighbour.c.
 *                Krzysztof Halasa:        Added Frame Relay ARP support.
 *                Arnaldo C. Melo :        convert /proc/net/arp to seq_file
 *                Shmulik Hen:                Split arp_send to arp_create and
 *                                        arp_xmit so intermediate drivers like
 *                                        bonding can change the skb before
 *                                        sending (e.g. insert 8021q tag).
 *                Harald Welte        :        convert to make use of jenkins hash
 *                Jesper D. Brouer:       Proxy ARP PVLAN RFC 3069 support.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/capability.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/mm.h>
#include <linux/inet.h>
#include <linux/inetdevice.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/fddidevice.h>
#include <linux/if_arp.h>
#include <linux/skbuff.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/stat.h>
#include <linux/init.h>
#include <linux/net.h>
#include <linux/rcupdate.h>
#include <linux/slab.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

#include <net/net_namespace.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/route.h>
#include <net/protocol.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/ax25.h>
#include <net/netrom.h>
#include <net/dst_metadata.h>
#include <net/ip_tunnels.h>

#include <linux/uaccess.h>

#include <linux/netfilter_arp.h>

/*
 *        Interface to generic neighbour cache.
 */
static u32 arp_hash(const void *pkey, const struct net_device *dev, __u32 *hash_rnd);
static bool arp_key_eq(const struct neighbour *n, const void *pkey);
static int arp_constructor(struct neighbour *neigh);
static void arp_solicit(struct neighbour *neigh, struct sk_buff *skb);
static void arp_error_report(struct neighbour *neigh, struct sk_buff *skb);
static void parp_redo(struct sk_buff *skb);
static int arp_is_multicast(const void *pkey);

static const struct neigh_ops arp_generic_ops = {
        .family =                AF_INET,
        .solicit =                arp_solicit,
        .error_report =                arp_error_report,
        .output =                neigh_resolve_output,
        .connected_output =        neigh_connected_output,
};

static const struct neigh_ops arp_hh_ops = {
        .family =                AF_INET,
        .solicit =                arp_solicit,
        .error_report =                arp_error_report,
        .output =                neigh_resolve_output,
        .connected_output =        neigh_resolve_output,
};

static const struct neigh_ops arp_direct_ops = {
        .family =                AF_INET,
        .output =                neigh_direct_output,
        .connected_output =        neigh_direct_output,
};

struct neigh_table arp_tbl = {
        .family                = AF_INET,
        .key_len        = 4,
        .protocol        = cpu_to_be16(ETH_P_IP),
        .hash                = arp_hash,
        .key_eq                = arp_key_eq,
        .constructor        = arp_constructor,
        .proxy_redo        = parp_redo,
        .is_multicast        = arp_is_multicast,
        .id                = "arp_cache",
        .parms                = {
                .tbl                        = &arp_tbl,
                .reachable_time                = 30 * HZ,
                .data        = {
                        [NEIGH_VAR_MCAST_PROBES] = 3,
                        [NEIGH_VAR_UCAST_PROBES] = 3,
                        [NEIGH_VAR_RETRANS_TIME] = 1 * HZ,
                        [NEIGH_VAR_BASE_REACHABLE_TIME] = 30 * HZ,
                        [NEIGH_VAR_DELAY_PROBE_TIME] = 5 * HZ,
                        [NEIGH_VAR_INTERVAL_PROBE_TIME_MS] = 5 * HZ,
                        [NEIGH_VAR_GC_STALETIME] = 60 * HZ,
                        [NEIGH_VAR_QUEUE_LEN_BYTES] = SK_WMEM_MAX,
                        [NEIGH_VAR_PROXY_QLEN] = 64,
                        [NEIGH_VAR_ANYCAST_DELAY] = 1 * HZ,
                        [NEIGH_VAR_PROXY_DELAY]        = (8 * HZ) / 10,
                        [NEIGH_VAR_LOCKTIME] = 1 * HZ,
                },
        },
        .gc_interval        = 30 * HZ,
        .gc_thresh1        = 128,
        .gc_thresh2        = 512,
        .gc_thresh3        = 1024,
};
EXPORT_SYMBOL(arp_tbl);

int arp_mc_map(__be32 addr, u8 *haddr, struct net_device *dev, int dir)
{
        switch (dev->type) {
        case ARPHRD_ETHER:
        case ARPHRD_FDDI:
        case ARPHRD_IEEE802:
                ip_eth_mc_map(addr, haddr);
                return 0;
        case ARPHRD_INFINIBAND:
                ip_ib_mc_map(addr, dev->broadcast, haddr);
                return 0;
        case ARPHRD_IPGRE:
                ip_ipgre_mc_map(addr, dev->broadcast, haddr);
                return 0;
        default:
                if (dir) {
                        memcpy(haddr, dev->broadcast, dev->addr_len);
                        return 0;
                }
        }
        return -EINVAL;
}


static u32 arp_hash(const void *pkey,
                    const struct net_device *dev,
                    __u32 *hash_rnd)
{
        return arp_hashfn(pkey, dev, hash_rnd);
}

static bool arp_key_eq(const struct neighbour *neigh, const void *pkey)
{
        return neigh_key_eq32(neigh, pkey);
}

static int arp_constructor(struct neighbour *neigh)
{
        __be32 addr;
        struct net_device *dev = neigh->dev;
        struct in_device *in_dev;
        struct neigh_parms *parms;
        u32 inaddr_any = INADDR_ANY;

        if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
                memcpy(neigh->primary_key, &inaddr_any, arp_tbl.key_len);

        addr = *(__be32 *)neigh->primary_key;
        rcu_read_lock();
        in_dev = __in_dev_get_rcu(dev);
        if (!in_dev) {
                rcu_read_unlock();
                return -EINVAL;
        }

        neigh->type = inet_addr_type_dev_table(dev_net(dev), dev, addr);

        parms = in_dev->arp_parms;
        __neigh_parms_put(neigh->parms);
        neigh->parms = neigh_parms_clone(parms);
        rcu_read_unlock();

        if (!dev->header_ops) {
                neigh->nud_state = NUD_NOARP;
                neigh->ops = &arp_direct_ops;
                neigh->output = neigh_direct_output;
        } else {
                /* Good devices (checked by reading texts, but only Ethernet is
                   tested)

                   ARPHRD_ETHER: (ethernet, apfddi)
                   ARPHRD_FDDI: (fddi)
                   ARPHRD_IEEE802: (tr)
                   ARPHRD_METRICOM: (strip)
                   ARPHRD_ARCNET:
                   etc. etc. etc.

                   ARPHRD_IPDDP will also work, if author repairs it.
                   I did not it, because this driver does not work even
                   in old paradigm.
                 */

                if (neigh->type == RTN_MULTICAST) {
                        neigh->nud_state = NUD_NOARP;
                        arp_mc_map(addr, neigh->ha, dev, 1);
                } else if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) {
                        neigh->nud_state = NUD_NOARP;
                        memcpy(neigh->ha, dev->dev_addr, dev->addr_len);
                } else if (neigh->type == RTN_BROADCAST ||
                           (dev->flags & IFF_POINTOPOINT)) {
                        neigh->nud_state = NUD_NOARP;
                        memcpy(neigh->ha, dev->broadcast, dev->addr_len);
                }

                if (dev->header_ops->cache)
                        neigh->ops = &arp_hh_ops;
                else
                        neigh->ops = &arp_generic_ops;

                if (neigh->nud_state & NUD_VALID)
                        neigh->output = neigh->ops->connected_output;
                else
                        neigh->output = neigh->ops->output;
        }
        return 0;
}

static void arp_error_report(struct neighbour *neigh, struct sk_buff *skb)
{
        dst_link_failure(skb);
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_FAILED);
}

/* Create and send an arp packet. */
static void arp_send_dst(int type, int ptype, __be32 dest_ip,
                         struct net_device *dev, __be32 src_ip,
                         const unsigned char *dest_hw,
                         const unsigned char *src_hw,
                         const unsigned char *target_hw,
                         struct dst_entry *dst)
{
        struct sk_buff *skb;

        /* arp on this interface. */
        if (dev->flags & IFF_NOARP)
                return;

        skb = arp_create(type, ptype, dest_ip, dev, src_ip,
                         dest_hw, src_hw, target_hw);
        if (!skb)
                return;

        skb_dst_set(skb, dst_clone(dst));
        arp_xmit(skb);
}

void arp_send(int type, int ptype, __be32 dest_ip,
              struct net_device *dev, __be32 src_ip,
              const unsigned char *dest_hw, const unsigned char *src_hw,
              const unsigned char *target_hw)
{
        arp_send_dst(type, ptype, dest_ip, dev, src_ip, dest_hw, src_hw,
                     target_hw, NULL);
}
EXPORT_SYMBOL(arp_send);

static void arp_solicit(struct neighbour *neigh, struct sk_buff *skb)
{
        __be32 saddr = 0;
        u8 dst_ha[MAX_ADDR_LEN], *dst_hw = NULL;
        struct net_device *dev = neigh->dev;
        __be32 target = *(__be32 *)neigh->primary_key;
        int probes = atomic_read(&neigh->probes);
        struct in_device *in_dev;
        struct dst_entry *dst = NULL;

        rcu_read_lock();
        in_dev = __in_dev_get_rcu(dev);
        if (!in_dev) {
                rcu_read_unlock();
                return;
        }
        switch (IN_DEV_ARP_ANNOUNCE(in_dev)) {
        default:
        case 0:                /* By default announce any local IP */
                if (skb && inet_addr_type_dev_table(dev_net(dev), dev,
                                          ip_hdr(skb)->saddr) == RTN_LOCAL)
                        saddr = ip_hdr(skb)->saddr;
                break;
        case 1:                /* Restrict announcements of saddr in same subnet */
                if (!skb)
                        break;
                saddr = ip_hdr(skb)->saddr;
                if (inet_addr_type_dev_table(dev_net(dev), dev,
                                             saddr) == RTN_LOCAL) {
                        /* saddr should be known to target */
                        if (inet_addr_onlink(in_dev, target, saddr))
                                break;
                }
                saddr = 0;
                break;
        case 2:                /* Avoid secondary IPs, get a primary/preferred one */
                break;
        }
        rcu_read_unlock();

        if (!saddr)
                saddr = inet_select_addr(dev, target, RT_SCOPE_LINK);

        probes -= NEIGH_VAR(neigh->parms, UCAST_PROBES);
        if (probes < 0) {
                if (!(READ_ONCE(neigh->nud_state) & NUD_VALID))
                        pr_debug("trying to ucast probe in NUD_INVALID\n");
                neigh_ha_snapshot(dst_ha, neigh, dev);
                dst_hw = dst_ha;
        } else {
                probes -= NEIGH_VAR(neigh->parms, APP_PROBES);
                if (probes < 0) {
                        neigh_app_ns(neigh);
                        return;
                }
        }

        if (skb && !(dev->priv_flags & IFF_XMIT_DST_RELEASE))
                dst = skb_dst(skb);
        arp_send_dst(ARPOP_REQUEST, ETH_P_ARP, target, dev, saddr,
                     dst_hw, dev->dev_addr, NULL, dst);
}

static int arp_ignore(struct in_device *in_dev, __be32 sip, __be32 tip)
{
        struct net *net = dev_net(in_dev->dev);
        int scope;

        switch (IN_DEV_ARP_IGNORE(in_dev)) {
        case 0:        /* Reply, the tip is already validated */
                return 0;
        case 1:        /* Reply only if tip is configured on the incoming interface */
                sip = 0;
                scope = RT_SCOPE_HOST;
                break;
        case 2:        /*
                 * Reply only if tip is configured on the incoming interface
                 * and is in same subnet as sip
                 */
                scope = RT_SCOPE_HOST;
                break;
        case 3:        /* Do not reply for scope host addresses */
                sip = 0;
                scope = RT_SCOPE_LINK;
                in_dev = NULL;
                break;
        case 4:        /* Reserved */
        case 5:
        case 6:
        case 7:
                return 0;
        case 8:        /* Do not reply */
                return 1;
        default:
                return 0;
        }
        return !inet_confirm_addr(net, in_dev, sip, tip, scope);
}

static int arp_accept(struct in_device *in_dev, __be32 sip)
{
        struct net *net = dev_net(in_dev->dev);
        int scope = RT_SCOPE_LINK;

        switch (IN_DEV_ARP_ACCEPT(in_dev)) {
        case 0: /* Don't create new entries from garp */
                return 0;
        case 1: /* Create new entries from garp */
                return 1;
        case 2: /* Create a neighbor in the arp table only if sip
                 * is in the same subnet as an address configured
                 * on the interface that received the garp message
                 */
                return !!inet_confirm_addr(net, in_dev, sip, 0, scope);
        default:
                return 0;
        }
}

static int arp_filter(__be32 sip, __be32 tip, struct net_device *dev)
{
        struct rtable *rt;
        int flag = 0;
        /*unsigned long now; */
        struct net *net = dev_net(dev);

        rt = ip_route_output(net, sip, tip, 0, l3mdev_master_ifindex_rcu(dev),
                             RT_SCOPE_UNIVERSE);
        if (IS_ERR(rt))
                return 1;
        if (rt->dst.dev != dev) {
                __NET_INC_STATS(net, LINUX_MIB_ARPFILTER);
                flag = 1;
        }
        ip_rt_put(rt);
        return flag;
}

/*
 * Check if we can use proxy ARP for this path
 */
static inline int arp_fwd_proxy(struct in_device *in_dev,
                                struct net_device *dev,        struct rtable *rt)
{
        struct in_device *out_dev;
        int imi, omi = -1;

        if (rt->dst.dev == dev)
                return 0;

        if (!IN_DEV_PROXY_ARP(in_dev))
                return 0;
        imi = IN_DEV_MEDIUM_ID(in_dev);
        if (imi == 0)
                return 1;
        if (imi == -1)
                return 0;

        /* place to check for proxy_arp for routes */

        out_dev = __in_dev_get_rcu(rt->dst.dev);
        if (out_dev)
                omi = IN_DEV_MEDIUM_ID(out_dev);

        return omi != imi && omi != -1;
}

/*
 * Check for RFC3069 proxy arp private VLAN (allow to send back to same dev)
 *
 * RFC3069 supports proxy arp replies back to the same interface.  This
 * is done to support (ethernet) switch features, like RFC 3069, where
 * the individual ports are not allowed to communicate with each
 * other, BUT they are allowed to talk to the upstream router.  As
 * described in RFC 3069, it is possible to allow these hosts to
 * communicate through the upstream router, by proxy_arp'ing.
 *
 * RFC 3069: "VLAN Aggregation for Efficient IP Address Allocation"
 *
 *  This technology is known by different names:
 *    In RFC 3069 it is called VLAN Aggregation.
 *    Cisco and Allied Telesyn call it Private VLAN.
 *    Hewlett-Packard call it Source-Port filtering or port-isolation.
 *    Ericsson call it MAC-Forced Forwarding (RFC Draft).
 *
 */
static inline int arp_fwd_pvlan(struct in_device *in_dev,
                                struct net_device *dev,        struct rtable *rt,
                                __be32 sip, __be32 tip)
{
        /* Private VLAN is only concerned about the same ethernet segment */
        if (rt->dst.dev != dev)
                return 0;

        /* Don't reply on self probes (often done by windowz boxes)*/
        if (sip == tip)
                return 0;

        if (IN_DEV_PROXY_ARP_PVLAN(in_dev))
                return 1;
        else
                return 0;
}

/*
 *        Interface to link layer: send routine and receive handler.
 */

/*
 *        Create an arp packet. If dest_hw is not set, we create a broadcast
 *        message.
 */
struct sk_buff *arp_create(int type, int ptype, __be32 dest_ip,
                           struct net_device *dev, __be32 src_ip,
                           const unsigned char *dest_hw,
                           const unsigned char *src_hw,
                           const unsigned char *target_hw)
{
        struct sk_buff *skb;
        struct arphdr *arp;
        unsigned char *arp_ptr;
        int hlen = LL_RESERVED_SPACE(dev);
        int tlen = dev->needed_tailroom;

        /*
         *        Allocate a buffer
         */

        skb = alloc_skb(arp_hdr_len(dev) + hlen + tlen, GFP_ATOMIC);
        if (!skb)
                return NULL;

        skb_reserve(skb, hlen);
        skb_reset_network_header(skb);
        arp = skb_put(skb, arp_hdr_len(dev));
        skb->dev = dev;
        skb->protocol = htons(ETH_P_ARP);
        if (!src_hw)
                src_hw = dev->dev_addr;
        if (!dest_hw)
                dest_hw = dev->broadcast;

        /*
         *        Fill the device header for the ARP frame
         */
        if (dev_hard_header(skb, dev, ptype, dest_hw, src_hw, skb->len) < 0)
                goto out;

        /*
         * Fill out the arp protocol part.
         *
         * The arp hardware type should match the device type, except for FDDI,
         * which (according to RFC 1390) should always equal 1 (Ethernet).
         */
        /*
         *        Exceptions everywhere. AX.25 uses the AX.25 PID value not the
         *        DIX code for the protocol. Make these device structure fields.
         */
        switch (dev->type) {
        default:
                arp->ar_hrd = htons(dev->type);
                arp->ar_pro = htons(ETH_P_IP);
                break;

#if IS_ENABLED(CONFIG_AX25)
        case ARPHRD_AX25:
                arp->ar_hrd = htons(ARPHRD_AX25);
                arp->ar_pro = htons(AX25_P_IP);
                break;

#if IS_ENABLED(CONFIG_NETROM)
        case ARPHRD_NETROM:
                arp->ar_hrd = htons(ARPHRD_NETROM);
                arp->ar_pro = htons(AX25_P_IP);
                break;
#endif
#endif

#if IS_ENABLED(CONFIG_FDDI)
        case ARPHRD_FDDI:
                arp->ar_hrd = htons(ARPHRD_ETHER);
                arp->ar_pro = htons(ETH_P_IP);
                break;
#endif
        }

        arp->ar_hln = dev->addr_len;
        arp->ar_pln = 4;
        arp->ar_op = htons(type);

        arp_ptr = (unsigned char *)(arp + 1);

        memcpy(arp_ptr, src_hw, dev->addr_len);
        arp_ptr += dev->addr_len;
        memcpy(arp_ptr, &src_ip, 4);
        arp_ptr += 4;

        switch (dev->type) {
#if IS_ENABLED(CONFIG_FIREWIRE_NET)
        case ARPHRD_IEEE1394:
                break;
#endif
        default:
                if (target_hw)
                        memcpy(arp_ptr, target_hw, dev->addr_len);
                else
                        memset(arp_ptr, 0, dev->addr_len);
                arp_ptr += dev->addr_len;
        }
        memcpy(arp_ptr, &dest_ip, 4);

        return skb;

out:
        kfree_skb(skb);
        return NULL;
}
EXPORT_SYMBOL(arp_create);

static int arp_xmit_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        return dev_queue_xmit(skb);
}

/*
 *        Send an arp packet.
 */
void arp_xmit(struct sk_buff *skb)
{
        /* Send it off, maybe filter it using firewalling first.  */
        NF_HOOK(NFPROTO_ARP, NF_ARP_OUT,
                dev_net(skb->dev), NULL, skb, NULL, skb->dev,
                arp_xmit_finish);
}
EXPORT_SYMBOL(arp_xmit);

static bool arp_is_garp(struct net *net, struct net_device *dev,
                        int *addr_type, __be16 ar_op,
                        __be32 sip, __be32 tip,
                        unsigned char *sha, unsigned char *tha)
{
        bool is_garp = tip == sip;

        /* Gratuitous ARP _replies_ also require target hwaddr to be
         * the same as source.
         */
        if (is_garp && ar_op == htons(ARPOP_REPLY))
                is_garp =
                        /* IPv4 over IEEE 1394 doesn't provide target
                         * hardware address field in its ARP payload.
                         */
                        tha &&
                        !memcmp(tha, sha, dev->addr_len);

        if (is_garp) {
                *addr_type = inet_addr_type_dev_table(net, dev, sip);
                if (*addr_type != RTN_UNICAST)
                        is_garp = false;
        }
        return is_garp;
}

/*
 *        Process an arp request.
 */

static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct net_device *dev = skb->dev;
        struct in_device *in_dev = __in_dev_get_rcu(dev);
        struct arphdr *arp;
        unsigned char *arp_ptr;
        struct rtable *rt;
        unsigned char *sha;
        unsigned char *tha = NULL;
        __be32 sip, tip;
        u16 dev_type = dev->type;
        int addr_type;
        struct neighbour *n;
        struct dst_entry *reply_dst = NULL;
        bool is_garp = false;

        /* arp_rcv below verifies the ARP header and verifies the device
         * is ARP'able.
         */

        if (!in_dev)
                goto out_free_skb;

        arp = arp_hdr(skb);

        switch (dev_type) {
        default:
                if (arp->ar_pro != htons(ETH_P_IP) ||
                    htons(dev_type) != arp->ar_hrd)
                        goto out_free_skb;
                break;
        case ARPHRD_ETHER:
        case ARPHRD_FDDI:
        case ARPHRD_IEEE802:
                /*
                 * ETHERNET, and Fibre Channel (which are IEEE 802
                 * devices, according to RFC 2625) devices will accept ARP
                 * hardware types of either 1 (Ethernet) or 6 (IEEE 802.2).
                 * This is the case also of FDDI, where the RFC 1390 says that
                 * FDDI devices should accept ARP hardware of (1) Ethernet,
                 * however, to be more robust, we'll accept both 1 (Ethernet)
                 * or 6 (IEEE 802.2)
                 */
                if ((arp->ar_hrd != htons(ARPHRD_ETHER) &&
                     arp->ar_hrd != htons(ARPHRD_IEEE802)) ||
                    arp->ar_pro != htons(ETH_P_IP))
                        goto out_free_skb;
                break;
        case ARPHRD_AX25:
                if (arp->ar_pro != htons(AX25_P_IP) ||
                    arp->ar_hrd != htons(ARPHRD_AX25))
                        goto out_free_skb;
                break;
        case ARPHRD_NETROM:
                if (arp->ar_pro != htons(AX25_P_IP) ||
                    arp->ar_hrd != htons(ARPHRD_NETROM))
                        goto out_free_skb;
                break;
        }

        /* Understand only these message types */

        if (arp->ar_op != htons(ARPOP_REPLY) &&
            arp->ar_op != htons(ARPOP_REQUEST))
                goto out_free_skb;

/*
 *        Extract fields
 */
        arp_ptr = (unsigned char *)(arp + 1);
        sha        = arp_ptr;
        arp_ptr += dev->addr_len;
        memcpy(&sip, arp_ptr, 4);
        arp_ptr += 4;
        switch (dev_type) {
#if IS_ENABLED(CONFIG_FIREWIRE_NET)
        case ARPHRD_IEEE1394:
                break;
#endif
        default:
                tha = arp_ptr;
                arp_ptr += dev->addr_len;
        }
        memcpy(&tip, arp_ptr, 4);
/*
 *        Check for bad requests for 127.x.x.x and requests for multicast
 *        addresses.  If this is one such, delete it.
 */
        if (ipv4_is_multicast(tip) ||
            (!IN_DEV_ROUTE_LOCALNET(in_dev) && ipv4_is_loopback(tip)))
                goto out_free_skb;

 /*
  *        For some 802.11 wireless deployments (and possibly other networks),
  *        there will be an ARP proxy and gratuitous ARP frames are attacks
  *        and thus should not be accepted.
  */
        if (sip == tip && IN_DEV_ORCONF(in_dev, DROP_GRATUITOUS_ARP))
                goto out_free_skb;

/*
 *     Special case: We must set Frame Relay source Q.922 address
 */
        if (dev_type == ARPHRD_DLCI)
                sha = dev->broadcast;

/*
 *  Process entry.  The idea here is we want to send a reply if it is a
 *  request for us or if it is a request for someone else that we hold
 *  a proxy for.  We want to add an entry to our cache if it is a reply
 *  to us or if it is a request for our address.
 *  (The assumption for this last is that if someone is requesting our
 *  address, they are probably intending to talk to us, so it saves time
 *  if we cache their address.  Their address is also probably not in
 *  our cache, since ours is not in their cache.)
 *
 *  Putting this another way, we only care about replies if they are to
 *  us, in which case we add them to the cache.  For requests, we care
 *  about those for us and those for our proxies.  We reply to both,
 *  and in the case of requests for us we add the requester to the arp
 *  cache.
 */

        if (arp->ar_op == htons(ARPOP_REQUEST) && skb_metadata_dst(skb))
                reply_dst = (struct dst_entry *)
                            iptunnel_metadata_reply(skb_metadata_dst(skb),
                                                    GFP_ATOMIC);

        /* Special case: IPv4 duplicate address detection packet (RFC2131) */
        if (sip == 0) {
                if (arp->ar_op == htons(ARPOP_REQUEST) &&
                    inet_addr_type_dev_table(net, dev, tip) == RTN_LOCAL &&
                    !arp_ignore(in_dev, sip, tip))
                        arp_send_dst(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip,
                                     sha, dev->dev_addr, sha, reply_dst);
                goto out_consume_skb;
        }

        if (arp->ar_op == htons(ARPOP_REQUEST) &&
            ip_route_input_noref(skb, tip, sip, 0, dev) == 0) {

                rt = skb_rtable(skb);
                addr_type = rt->rt_type;

                if (addr_type == RTN_LOCAL) {
                        int dont_send;

                        dont_send = arp_ignore(in_dev, sip, tip);
                        if (!dont_send && IN_DEV_ARPFILTER(in_dev))
                                dont_send = arp_filter(sip, tip, dev);
                        if (!dont_send) {
                                n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
                                if (n) {
                                        arp_send_dst(ARPOP_REPLY, ETH_P_ARP,
                                                     sip, dev, tip, sha,
                                                     dev->dev_addr, sha,
                                                     reply_dst);
                                        neigh_release(n);
                                }
                        }
                        goto out_consume_skb;
                } else if (IN_DEV_FORWARD(in_dev)) {
                        if (addr_type == RTN_UNICAST  &&
                            (arp_fwd_proxy(in_dev, dev, rt) ||
                             arp_fwd_pvlan(in_dev, dev, rt, sip, tip) ||
                             (rt->dst.dev != dev &&
                              pneigh_lookup(&arp_tbl, net, &tip, dev, 0)))) {
                                n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
                                if (n)
                                        neigh_release(n);

                                if (NEIGH_CB(skb)->flags & LOCALLY_ENQUEUED ||
                                    skb->pkt_type == PACKET_HOST ||
                                    NEIGH_VAR(in_dev->arp_parms, PROXY_DELAY) == 0) {
                                        arp_send_dst(ARPOP_REPLY, ETH_P_ARP,
                                                     sip, dev, tip, sha,
                                                     dev->dev_addr, sha,
                                                     reply_dst);
                                } else {
                                        pneigh_enqueue(&arp_tbl,
                                                       in_dev->arp_parms, skb);
                                        goto out_free_dst;
                                }
                                goto out_consume_skb;
                        }
                }
        }

        /* Update our ARP tables */

        n = __neigh_lookup(&arp_tbl, &sip, dev, 0);

        addr_type = -1;
        if (n || arp_accept(in_dev, sip)) {
                is_garp = arp_is_garp(net, dev, &addr_type, arp->ar_op,
                                      sip, tip, sha, tha);
        }

        if (arp_accept(in_dev, sip)) {
                /* Unsolicited ARP is not accepted by default.
                   It is possible, that this option should be enabled for some
                   devices (strip is candidate)
                 */
                if (!n &&
                    (is_garp ||
                     (arp->ar_op == htons(ARPOP_REPLY) &&
                      (addr_type == RTN_UNICAST ||
                       (addr_type < 0 &&
                        /* postpone calculation to as late as possible */
                        inet_addr_type_dev_table(net, dev, sip) ==
                                RTN_UNICAST)))))
                        n = __neigh_lookup(&arp_tbl, &sip, dev, 1);
        }

        if (n) {
                int state = NUD_REACHABLE;
                int override;

                /* If several different ARP replies follows back-to-back,
                   use the FIRST one. It is possible, if several proxy
                   agents are active. Taking the first reply prevents
                   arp trashing and chooses the fastest router.
                 */
                override = time_after(jiffies,
                                      n->updated +
                                      NEIGH_VAR(n->parms, LOCKTIME)) ||
                           is_garp;

                /* Broadcast replies and request packets
                   do not assert neighbour reachability.
                 */
                if (arp->ar_op != htons(ARPOP_REPLY) ||
                    skb->pkt_type != PACKET_HOST)
                        state = NUD_STALE;
                neigh_update(n, sha, state,
                             override ? NEIGH_UPDATE_F_OVERRIDE : 0, 0);
                neigh_release(n);
        }

out_consume_skb:
        consume_skb(skb);

out_free_dst:
        dst_release(reply_dst);
        return NET_RX_SUCCESS;

out_free_skb:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static void parp_redo(struct sk_buff *skb)
{
        arp_process(dev_net(skb->dev), NULL, skb);
}

static int arp_is_multicast(const void *pkey)
{
        return ipv4_is_multicast(*((__be32 *)pkey));
}

/*
 *        Receive an arp request from the device layer.
 */

static int arp_rcv(struct sk_buff *skb, struct net_device *dev,
                   struct packet_type *pt, struct net_device *orig_dev)
{
        const struct arphdr *arp;

        /* do not tweak dropwatch on an ARP we will ignore */
        if (dev->flags & IFF_NOARP ||
            skb->pkt_type == PACKET_OTHERHOST ||
            skb->pkt_type == PACKET_LOOPBACK)
                goto consumeskb;

        skb = skb_share_check(skb, GFP_ATOMIC);
        if (!skb)
                goto out_of_mem;

        /* ARP header, plus 2 device addresses, plus 2 IP addresses.  */
        if (!pskb_may_pull(skb, arp_hdr_len(dev)))
                goto freeskb;

        arp = arp_hdr(skb);
        if (arp->ar_hln != dev->addr_len || arp->ar_pln != 4)
                goto freeskb;

        memset(NEIGH_CB(skb), 0, sizeof(struct neighbour_cb));

        return NF_HOOK(NFPROTO_ARP, NF_ARP_IN,
                       dev_net(dev), NULL, skb, dev, NULL,
                       arp_process);

consumeskb:
        consume_skb(skb);
        return NET_RX_SUCCESS;
freeskb:
        kfree_skb(skb);
out_of_mem:
        return NET_RX_DROP;
}

/*
 *        User level interface (ioctl)
 */

static struct net_device *arp_req_dev_by_name(struct net *net, struct arpreq *r,
                                              bool getarp)
{
        struct net_device *dev;

        if (getarp)
                dev = dev_get_by_name_rcu(net, r->arp_dev);
        else
                dev = __dev_get_by_name(net, r->arp_dev);
        if (!dev)
                return ERR_PTR(-ENODEV);

        /* Mmmm... It is wrong... ARPHRD_NETROM == 0 */
        if (!r->arp_ha.sa_family)
                r->arp_ha.sa_family = dev->type;

        if ((r->arp_flags & ATF_COM) && r->arp_ha.sa_family != dev->type)
                return ERR_PTR(-EINVAL);

        return dev;
}

static struct net_device *arp_req_dev(struct net *net, struct arpreq *r)
{
        struct net_device *dev;
        struct rtable *rt;
        __be32 ip;

        if (r->arp_dev[0])
                return arp_req_dev_by_name(net, r, false);

        if (r->arp_flags & ATF_PUBL)
                return NULL;

        ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;

        rt = ip_route_output(net, ip, 0, 0, 0, RT_SCOPE_LINK);
        if (IS_ERR(rt))
                return ERR_CAST(rt);

        dev = rt->dst.dev;
        ip_rt_put(rt);

        if (!dev)
                return ERR_PTR(-EINVAL);

        return dev;
}

/*
 *        Set (create) an ARP cache entry.
 */

static int arp_req_set_proxy(struct net *net, struct net_device *dev, int on)
{
        if (!dev) {
                IPV4_DEVCONF_ALL(net, PROXY_ARP) = on;
                return 0;
        }
        if (__in_dev_get_rtnl(dev)) {
                IN_DEV_CONF_SET(__in_dev_get_rtnl(dev), PROXY_ARP, on);
                return 0;
        }
        return -ENXIO;
}

static int arp_req_set_public(struct net *net, struct arpreq *r,
                struct net_device *dev)
{
        __be32 mask = ((struct sockaddr_in *)&r->arp_netmask)->sin_addr.s_addr;

        if (!dev && (r->arp_flags & ATF_COM)) {
                dev = dev_getbyhwaddr_rcu(net, r->arp_ha.sa_family,
                                      r->arp_ha.sa_data);
                if (!dev)
                        return -ENODEV;
        }
        if (mask) {
                __be32 ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;

                if (!pneigh_lookup(&arp_tbl, net, &ip, dev, 1))
                        return -ENOBUFS;
                return 0;
        }

        return arp_req_set_proxy(net, dev, 1);
}

static int arp_req_set(struct net *net, struct arpreq *r)
{
        struct neighbour *neigh;
        struct net_device *dev;
        __be32 ip;
        int err;

        dev = arp_req_dev(net, r);
        if (IS_ERR(dev))
                return PTR_ERR(dev);

        if (r->arp_flags & ATF_PUBL)
                return arp_req_set_public(net, r, dev);

        switch (dev->type) {
#if IS_ENABLED(CONFIG_FDDI)
        case ARPHRD_FDDI:
                /*
                 * According to RFC 1390, FDDI devices should accept ARP
                 * hardware types of 1 (Ethernet).  However, to be more
                 * robust, we'll accept hardware types of either 1 (Ethernet)
                 * or 6 (IEEE 802.2).
                 */
                if (r->arp_ha.sa_family != ARPHRD_FDDI &&
                    r->arp_ha.sa_family != ARPHRD_ETHER &&
                    r->arp_ha.sa_family != ARPHRD_IEEE802)
                        return -EINVAL;
                break;
#endif
        default:
                if (r->arp_ha.sa_family != dev->type)
                        return -EINVAL;
                break;
        }

        ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;

        neigh = __neigh_lookup_errno(&arp_tbl, &ip, dev);
        err = PTR_ERR(neigh);
        if (!IS_ERR(neigh)) {
                unsigned int state = NUD_STALE;

                if (r->arp_flags & ATF_PERM) {
                        r->arp_flags |= ATF_COM;
                        state = NUD_PERMANENT;
                }

                err = neigh_update(neigh, (r->arp_flags & ATF_COM) ?
                                   r->arp_ha.sa_data : NULL, state,
                                   NEIGH_UPDATE_F_OVERRIDE |
                                   NEIGH_UPDATE_F_ADMIN, 0);
                neigh_release(neigh);
        }
        return err;
}

static unsigned int arp_state_to_flags(struct neighbour *neigh)
{
        if (neigh->nud_state&NUD_PERMANENT)
                return ATF_PERM | ATF_COM;
        else if (neigh->nud_state&NUD_VALID)
                return ATF_COM;
        else
                return 0;
}

/*
 *        Get an ARP cache entry.
 */

static int arp_req_get(struct net *net, struct arpreq *r)
{
        __be32 ip = ((struct sockaddr_in *) &r->arp_pa)->sin_addr.s_addr;
        struct neighbour *neigh;
        struct net_device *dev;

        if (!r->arp_dev[0])
                return -ENODEV;

        dev = arp_req_dev_by_name(net, r, true);
        if (IS_ERR(dev))
                return PTR_ERR(dev);

        neigh = neigh_lookup(&arp_tbl, &ip, dev);
        if (!neigh)
                return -ENXIO;

        if (READ_ONCE(neigh->nud_state) & NUD_NOARP) {
                neigh_release(neigh);
                return -ENXIO;
        }

        read_lock_bh(&neigh->lock);
        memcpy(r->arp_ha.sa_data, neigh->ha,
               min(dev->addr_len, sizeof(r->arp_ha.sa_data_min)));
        r->arp_flags = arp_state_to_flags(neigh);
        read_unlock_bh(&neigh->lock);

        neigh_release(neigh);

        r->arp_ha.sa_family = dev->type;
        netdev_copy_name(dev, r->arp_dev);

        return 0;
}

int arp_invalidate(struct net_device *dev, __be32 ip, bool force)
{
        struct neighbour *neigh = neigh_lookup(&arp_tbl, &ip, dev);
        int err = -ENXIO;
        struct neigh_table *tbl = &arp_tbl;

        if (neigh) {
                if ((READ_ONCE(neigh->nud_state) & NUD_VALID) && !force) {
                        neigh_release(neigh);
                        return 0;
                }

                if (READ_ONCE(neigh->nud_state) & ~NUD_NOARP)
                        err = neigh_update(neigh, NULL, NUD_FAILED,
                                           NEIGH_UPDATE_F_OVERRIDE|
                                           NEIGH_UPDATE_F_ADMIN, 0);
                write_lock_bh(&tbl->lock);
                neigh_release(neigh);
                neigh_remove_one(neigh);
                write_unlock_bh(&tbl->lock);
        }

        return err;
}

static int arp_req_delete_public(struct net *net, struct arpreq *r,
                struct net_device *dev)
{
        __be32 mask = ((struct sockaddr_in *)&r->arp_netmask)->sin_addr.s_addr;

        if (mask) {
                __be32 ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;

                return pneigh_delete(&arp_tbl, net, &ip, dev);
        }

        return arp_req_set_proxy(net, dev, 0);
}

static int arp_req_delete(struct net *net, struct arpreq *r)
{
        struct net_device *dev;
        __be32 ip;

        dev = arp_req_dev(net, r);
        if (IS_ERR(dev))
                return PTR_ERR(dev);

        if (r->arp_flags & ATF_PUBL)
                return arp_req_delete_public(net, r, dev);

        ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;

        return arp_invalidate(dev, ip, true);
}

/*
 *        Handle an ARP layer I/O control request.
 */

int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg)
{
        struct arpreq r;
        __be32 *netmask;
        int err;

        switch (cmd) {
        case SIOCDARP:
        case SIOCSARP:
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                fallthrough;
        case SIOCGARP:
                err = copy_from_user(&r, arg, sizeof(struct arpreq));
                if (err)
                        return -EFAULT;
                break;
        default:
                return -EINVAL;
        }

        if (r.arp_pa.sa_family != AF_INET)
                return -EPFNOSUPPORT;

        if (!(r.arp_flags & ATF_PUBL) &&
            (r.arp_flags & (ATF_NETMASK | ATF_DONTPUB)))
                return -EINVAL;

        netmask = &((struct sockaddr_in *)&r.arp_netmask)->sin_addr.s_addr;
        if (!(r.arp_flags & ATF_NETMASK))
                *netmask = htonl(0xFFFFFFFFUL);
        else if (*netmask && *netmask != htonl(0xFFFFFFFFUL))
                return -EINVAL;

        switch (cmd) {
        case SIOCDARP:
                rtnl_lock();
                err = arp_req_delete(net, &r);
                rtnl_unlock();
                break;
        case SIOCSARP:
                rtnl_lock();
                err = arp_req_set(net, &r);
                rtnl_unlock();
                break;
        case SIOCGARP:
                rcu_read_lock();
                err = arp_req_get(net, &r);
                rcu_read_unlock();

                if (!err && copy_to_user(arg, &r, sizeof(r)))
                        err = -EFAULT;
                break;
        }

        return err;
}

static int arp_netdev_event(struct notifier_block *this, unsigned long event,
                            void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct netdev_notifier_change_info *change_info;
        struct in_device *in_dev;
        bool evict_nocarrier;

        switch (event) {
        case NETDEV_CHANGEADDR:
                neigh_changeaddr(&arp_tbl, dev);
                rt_cache_flush(dev_net(dev));
                break;
        case NETDEV_CHANGE:
                change_info = ptr;
                if (change_info->flags_changed & IFF_NOARP)
                        neigh_changeaddr(&arp_tbl, dev);

                in_dev = __in_dev_get_rtnl(dev);
                if (!in_dev)
                        evict_nocarrier = true;
                else
                        evict_nocarrier = IN_DEV_ARP_EVICT_NOCARRIER(in_dev);

                if (evict_nocarrier && !netif_carrier_ok(dev))
                        neigh_carrier_down(&arp_tbl, dev);
                break;
        default:
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block arp_netdev_notifier = {
        .notifier_call = arp_netdev_event,
};

/* Note, that it is not on notifier chain.
   It is necessary, that this routine was called after route cache will be
   flushed.
 */
void arp_ifdown(struct net_device *dev)
{
        neigh_ifdown(&arp_tbl, dev);
}


/*
 *        Called once on startup.
 */

static struct packet_type arp_packet_type __read_mostly = {
        .type =        cpu_to_be16(ETH_P_ARP),
        .func =        arp_rcv,
};

#ifdef CONFIG_PROC_FS
#if IS_ENABLED(CONFIG_AX25)

/*
 *        ax25 -> ASCII conversion
 */
static void ax2asc2(ax25_address *a, char *buf)
{
        char c, *s;
        int n;

        for (n = 0, s = buf; n < 6; n++) {
                c = (a->ax25_call[n] >> 1) & 0x7F;

                if (c != ' ')
                        *s++ = c;
        }

        *s++ = '-';
        n = (a->ax25_call[6] >> 1) & 0x0F;
        if (n > 9) {
                *s++ = '1';
                n -= 10;
        }

        *s++ = n + '0';
        *s++ = '\0';

        if (*buf == '\0' || *buf == '-') {
                buf[0] = '*';
                buf[1] = '\0';
        }
}
#endif /* CONFIG_AX25 */

#define HBUFFERLEN 30

static void arp_format_neigh_entry(struct seq_file *seq,
                                   struct neighbour *n)
{
        char hbuffer[HBUFFERLEN];
        int k, j;
        char tbuf[16];
        struct net_device *dev = n->dev;
        int hatype = dev->type;

        read_lock(&n->lock);
        /* Convert hardware address to XX:XX:XX:XX ... form. */
#if IS_ENABLED(CONFIG_AX25)
        if (hatype == ARPHRD_AX25 || hatype == ARPHRD_NETROM)
                ax2asc2((ax25_address *)n->ha, hbuffer);
        else {
#endif
        for (k = 0, j = 0; k < HBUFFERLEN - 3 && j < dev->addr_len; j++) {
                hbuffer[k++] = hex_asc_hi(n->ha[j]);
                hbuffer[k++] = hex_asc_lo(n->ha[j]);
                hbuffer[k++] = ':';
        }
        if (k != 0)
                --k;
        hbuffer[k] = 0;
#if IS_ENABLED(CONFIG_AX25)
        }
#endif
        sprintf(tbuf, "%pI4", n->primary_key);
        seq_printf(seq, "%-16s 0x%-10x0x%-10x%-17s     *        %s\n",
                   tbuf, hatype, arp_state_to_flags(n), hbuffer, dev->name);
        read_unlock(&n->lock);
}

static void arp_format_pneigh_entry(struct seq_file *seq,
                                    struct pneigh_entry *n)
{
        struct net_device *dev = n->dev;
        int hatype = dev ? dev->type : 0;
        char tbuf[16];

        sprintf(tbuf, "%pI4", n->key);
        seq_printf(seq, "%-16s 0x%-10x0x%-10x%s     *        %s\n",
                   tbuf, hatype, ATF_PUBL | ATF_PERM, "00:00:00:00:00:00",
                   dev ? dev->name : "*");
}

static int arp_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, "IP address       HW type     Flags       "
                              "HW address            Mask     Device\n");
        } else {
                struct neigh_seq_state *state = seq->private;

                if (state->flags & NEIGH_SEQ_IS_PNEIGH)
                        arp_format_pneigh_entry(seq, v);
                else
                        arp_format_neigh_entry(seq, v);
        }

        return 0;
}

static void *arp_seq_start(struct seq_file *seq, loff_t *pos)
{
        /* Don't want to confuse "arp -a" w/ magic entries,
         * so we tell the generic iterator to skip NUD_NOARP.
         */
        return neigh_seq_start(seq, pos, &arp_tbl, NEIGH_SEQ_SKIP_NOARP);
}

static const struct seq_operations arp_seq_ops = {
        .start        = arp_seq_start,
        .next        = neigh_seq_next,
        .stop        = neigh_seq_stop,
        .show        = arp_seq_show,
};
#endif /* CONFIG_PROC_FS */

static int __net_init arp_net_init(struct net *net)
{
        if (!proc_create_net("arp", 0444, net->proc_net, &arp_seq_ops,
                        sizeof(struct neigh_seq_state)))
                return -ENOMEM;
        return 0;
}

static void __net_exit arp_net_exit(struct net *net)
{
        remove_proc_entry("arp", net->proc_net);
}

static struct pernet_operations arp_net_ops = {
        .init = arp_net_init,
        .exit = arp_net_exit,
};

void __init arp_init(void)
{
        neigh_table_init(NEIGH_ARP_TABLE, &arp_tbl);

        dev_add_pack(&arp_packet_type);
        register_pernet_subsys(&arp_net_ops);
#ifdef CONFIG_SYSCTL
        neigh_sysctl_register(NULL, &arp_tbl.parms, NULL);
#endif
        register_netdevice_notifier(&arp_netdev_notifier);
}







































































































































































































































































































































































    3 






































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  CLC (connection layer control) handshake over initial TCP socket to
 *  prepare for RDMA traffic
 *
 *  Copyright IBM Corp. 2016
 *
 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#ifndef _SMC_CLC_H
#define _SMC_CLC_H

#include <rdma/ib_verbs.h>
#include <linux/smc.h>

#include "smc.h"
#include "smc_netlink.h"

#define SMC_CLC_PROPOSAL        0x01
#define SMC_CLC_ACCEPT                0x02
#define SMC_CLC_CONFIRM                0x03
#define SMC_CLC_DECLINE                0x04

#define SMC_TYPE_R                0                /* SMC-R only                      */
#define SMC_TYPE_D                1                /* SMC-D only                      */
#define SMC_TYPE_N                2                /* neither SMC-R nor SMC-D    */
#define SMC_TYPE_B                3                /* SMC-R and SMC-D              */
#define CLC_WAIT_TIME                (6 * HZ)        /* max. wait time on clcsock  */
#define CLC_WAIT_TIME_SHORT        HZ                /* short wait time on clcsock */
#define SMC_CLC_DECL_MEM        0x01010000  /* insufficient memory resources  */
#define SMC_CLC_DECL_TIMEOUT_CL        0x02010000  /* timeout w4 QP confirm link     */
#define SMC_CLC_DECL_TIMEOUT_AL        0x02020000  /* timeout w4 QP add link              */
#define SMC_CLC_DECL_CNFERR        0x03000000  /* configuration error            */
#define SMC_CLC_DECL_PEERNOSMC        0x03010000  /* peer did not indicate SMC      */
#define SMC_CLC_DECL_IPSEC        0x03020000  /* IPsec usage                      */
#define SMC_CLC_DECL_NOSMCDEV        0x03030000  /* no SMC device found (R or D)   */
#define SMC_CLC_DECL_NOSMCDDEV        0x03030001  /* no SMC-D device found              */
#define SMC_CLC_DECL_NOSMCRDEV        0x03030002  /* no SMC-R device found              */
#define SMC_CLC_DECL_NOISM2SUPP        0x03030003  /* hardware has no ISMv2 support  */
#define SMC_CLC_DECL_NOV2EXT        0x03030004  /* peer sent no clc v2 extension  */
#define SMC_CLC_DECL_NOV2DEXT        0x03030005  /* peer sent no clc SMC-Dv2 ext.  */
#define SMC_CLC_DECL_NOSEID        0x03030006  /* peer sent no SEID              */
#define SMC_CLC_DECL_NOSMCD2DEV        0x03030007  /* no SMC-Dv2 device found              */
#define SMC_CLC_DECL_NOUEID        0x03030008  /* peer sent no UEID              */
#define SMC_CLC_DECL_RELEASEERR        0x03030009  /* release version negotiate failed */
#define SMC_CLC_DECL_MAXCONNERR        0x0303000a  /* max connections negotiate failed */
#define SMC_CLC_DECL_MAXLINKERR        0x0303000b  /* max links negotiate failed */
#define SMC_CLC_DECL_MODEUNSUPP        0x03040000  /* smc modes do not match (R or D)*/
#define SMC_CLC_DECL_RMBE_EC        0x03050000  /* peer has eyecatcher in RMBE    */
#define SMC_CLC_DECL_OPTUNSUPP        0x03060000  /* fastopen sockopt not supported */
#define SMC_CLC_DECL_DIFFPREFIX        0x03070000  /* IP prefix / subnet mismatch    */
#define SMC_CLC_DECL_GETVLANERR        0x03080000  /* err to get vlan id of ip device*/
#define SMC_CLC_DECL_ISMVLANERR        0x03090000  /* err to reg vlan id on ism dev  */
#define SMC_CLC_DECL_NOACTLINK        0x030a0000  /* no active smc-r link in lgr    */
#define SMC_CLC_DECL_NOSRVLINK        0x030b0000  /* SMC-R link from srv not found  */
#define SMC_CLC_DECL_VERSMISMAT        0x030c0000  /* SMC version mismatch              */
#define SMC_CLC_DECL_MAX_DMB        0x030d0000  /* SMC-D DMB limit exceeded       */
#define SMC_CLC_DECL_NOROUTE        0x030e0000  /* SMC-Rv2 conn. no route to peer */
#define SMC_CLC_DECL_NOINDIRECT        0x030f0000  /* SMC-Rv2 conn. indirect mismatch*/
#define SMC_CLC_DECL_SYNCERR        0x04000000  /* synchronization error          */
#define SMC_CLC_DECL_PEERDECL        0x05000000  /* peer declined during handshake */
#define SMC_CLC_DECL_INTERR        0x09990000  /* internal error                      */
#define SMC_CLC_DECL_ERR_RTOK        0x09990001  /*         rtoken handling failed       */
#define SMC_CLC_DECL_ERR_RDYLNK        0x09990002  /*         ib ready link failed              */
#define SMC_CLC_DECL_ERR_REGBUF        0x09990003  /*         reg rdma bufs failed              */

#define SMC_FIRST_CONTACT_MASK        0b10        /* first contact bit within typev2 */

struct smc_clc_msg_hdr {        /* header1 of clc messages */
        u8 eyecatcher[4];        /* eye catcher */
        u8 type;                /* proposal / accept / confirm / decline */
        __be16 length;
#if defined(__BIG_ENDIAN_BITFIELD)
        u8 version : 4,
           typev2  : 2,
           typev1  : 2;
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 typev1  : 2,
           typev2  : 2,
           version : 4;
#endif
} __packed;                        /* format defined in RFC7609 */

struct smc_clc_msg_trail {        /* trailer of clc messages */
        u8 eyecatcher[4];
};

struct smc_clc_msg_local {        /* header2 of clc messages */
        u8 id_for_peer[SMC_SYSTEMID_LEN]; /* unique system id */
        u8 gid[16];                /* gid of ib_device port */
        u8 mac[6];                /* mac of ib_device port */
};

/* Struct would be 4 byte aligned, but it is used in an array that is sent
 * to peers and must conform to RFC7609, hence we need to use packed here.
 */
struct smc_clc_ipv6_prefix {
        struct in6_addr prefix;
        u8 prefix_len;
} __packed;                        /* format defined in RFC7609 */

#if defined(__BIG_ENDIAN_BITFIELD)
struct smc_clc_v2_flag {
        u8 release : 4,
           rsvd    : 3,
           seid    : 1;
};
#elif defined(__LITTLE_ENDIAN_BITFIELD)
struct smc_clc_v2_flag {
        u8 seid   : 1,
        rsvd      : 3,
        release   : 4;
};
#endif

struct smc_clnt_opts_area_hdr {
        u8 eid_cnt;                /* number of user defined EIDs */
        u8 ism_gid_cnt;                /* number of ISMv2 GIDs */
        u8 reserved1;
        struct smc_clc_v2_flag flag;
        u8 reserved2[2];
        __be16 smcd_v2_ext_offset; /* SMC-Dv2 Extension Offset */
};

struct smc_clc_smcd_gid_chid {
        __be64 gid;                /* ISM GID */
        __be16 chid;                /* ISMv2 CHID */
} __packed;                /* format defined in
                         * IBM Shared Memory Communications Version 2
                         * (https://www.ibm.com/support/pages/node/6326337)
                         */

struct smc_clc_v2_extension {
        /* New members must be added within the struct_group() macro below. */
        struct_group_tagged(smc_clc_v2_extension_fixed, fixed,
                struct smc_clnt_opts_area_hdr hdr;
                u8 roce[16];                /* RoCEv2 GID */
                u8 max_conns;
                u8 max_links;
                __be16 feature_mask;
                u8 reserved[12];
        );
        u8 user_eids[][SMC_MAX_EID_LEN];
};
static_assert(offsetof(struct smc_clc_v2_extension, user_eids) == sizeof(struct smc_clc_v2_extension_fixed),
              "struct member likely outside of struct_group_tagged()");

struct smc_clc_msg_proposal_prefix {        /* prefix part of clc proposal message*/
        __be32 outgoing_subnet;        /* subnet mask */
        u8 prefix_len;                /* number of significant bits in mask */
        u8 reserved[2];
        u8 ipv6_prefixes_cnt;        /* number of IPv6 prefixes in prefix array */
} __aligned(4);

struct smc_clc_msg_smcd {        /* SMC-D GID information */
        struct smc_clc_smcd_gid_chid ism; /* ISM native GID+CHID of requester */
        __be16 v2_ext_offset;        /* SMC Version 2 Extension Offset */
        u8 vendor_oui[3];        /* vendor organizationally unique identifier */
        u8 vendor_exp_options[5];
        u8 reserved[20];
};

struct smc_clc_smcd_v2_extension {
        /* New members must be added within the struct_group() macro below. */
        struct_group_tagged(smc_clc_smcd_v2_extension_fixed, fixed,
                u8 system_eid[SMC_MAX_EID_LEN];
                u8 reserved[16];
        );
        struct smc_clc_smcd_gid_chid gidchid[];
};
static_assert(offsetof(struct smc_clc_smcd_v2_extension, gidchid) == sizeof(struct smc_clc_smcd_v2_extension_fixed),
              "struct member likely outside of struct_group_tagged()");

struct smc_clc_msg_proposal {        /* clc proposal message sent by Linux */
        struct smc_clc_msg_hdr hdr;
        struct smc_clc_msg_local lcl;
        __be16 iparea_offset;        /* offset to IP address information area */
} __aligned(4);

#define SMC_CLC_MAX_V6_PREFIX                8
#define SMC_CLC_MAX_UEID                8
#define SMCD_CLC_MAX_V2_GID_ENTRIES        8 /* max # of CHID-GID entries in CLC
                                           * proposal SMC-Dv2 extension.
                                           * each ISM device takes one entry and
                                           * each Emulated-ISM takes two entries
                                           */

struct smc_clc_msg_proposal_area {
        struct smc_clc_msg_proposal                pclc_base;
        struct smc_clc_msg_smcd                        pclc_smcd;
        struct smc_clc_msg_proposal_prefix        pclc_prfx;
        struct smc_clc_ipv6_prefix        pclc_prfx_ipv6[SMC_CLC_MAX_V6_PREFIX];
        struct smc_clc_v2_extension_fixed        pclc_v2_ext;
        u8                        user_eids[SMC_CLC_MAX_UEID][SMC_MAX_EID_LEN];
        struct smc_clc_smcd_v2_extension_fixed        pclc_smcd_v2_ext;
        struct smc_clc_smcd_gid_chid
                                pclc_gidchids[SMCD_CLC_MAX_V2_GID_ENTRIES];
        struct smc_clc_msg_trail                pclc_trl;
};

struct smcr_clc_msg_accept_confirm {        /* SMCR accept/confirm */
        struct smc_clc_msg_local lcl;
        u8 qpn[3];                        /* QP number */
        __be32 rmb_rkey;                /* RMB rkey */
        u8 rmbe_idx;                        /* Index of RMBE in RMB */
        __be32 rmbe_alert_token;        /* unique connection id */
 #if defined(__BIG_ENDIAN_BITFIELD)
        u8 rmbe_size : 4,                /* buf size (compressed) */
           qp_mtu   : 4;                /* QP mtu */
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 qp_mtu   : 4,
           rmbe_size : 4;
#endif
        u8 reserved;
        __be64 rmb_dma_addr;        /* RMB virtual address */
        u8 reserved2;
        u8 psn[3];                /* packet sequence number */
} __packed;

struct smcd_clc_msg_accept_confirm_common {        /* SMCD accept/confirm */
        __be64 gid;                /* Sender GID */
        __be64 token;                /* DMB token */
        u8 dmbe_idx;                /* DMBE index */
#if defined(__BIG_ENDIAN_BITFIELD)
        u8 dmbe_size : 4,        /* buf size (compressed) */
           reserved3 : 4;
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 reserved3 : 4,
           dmbe_size : 4;
#endif
        u16 reserved4;
        __be32 linkid;                /* Link identifier */
} __packed;

#define SMC_CLC_OS_ZOS                1
#define SMC_CLC_OS_LINUX        2
#define SMC_CLC_OS_AIX                3

struct smc_clc_first_contact_ext {
#if defined(__BIG_ENDIAN_BITFIELD)
        u8 v2_direct : 1,
           reserved  : 7;
        u8 os_type : 4,
           release : 4;
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 reserved  : 7,
           v2_direct : 1;
        u8 release : 4,
           os_type : 4;
#endif
        u8 reserved2[2];
        u8 hostname[SMC_MAX_HOSTNAME_LEN];
};

struct smc_clc_first_contact_ext_v2x {
        struct smc_clc_first_contact_ext fce_v2_base;
        union {
                struct {
                        u8 max_conns; /* for SMC-R only */
                        u8 max_links; /* for SMC-R only */
                };
                u8 reserved3[2];        /* for SMC-D only */
        };
        __be16 feature_mask;
        __be32 vendor_exp_options;
        u8 reserved4[8];
} __packed;                /* format defined in
                         * IBM Shared Memory Communications Version 2 (Third Edition)
                         * (https://www.ibm.com/support/pages/node/7009315)
                         */

struct smc_clc_fce_gid_ext {
        u8 gid_cnt;
        u8 reserved2[3];
        u8 gid[][SMC_GID_SIZE];
};

struct smc_clc_msg_accept_confirm {        /* clc accept / confirm message */
        struct smc_clc_msg_hdr hdr;
        union {
                struct { /* SMC-R */
                        struct smcr_clc_msg_accept_confirm r0;
                        struct { /* v2 only */
                                u8 eid[SMC_MAX_EID_LEN];
                                u8 reserved6[8];
                        } __packed r1;
                };
                struct { /* SMC-D */
                        struct smcd_clc_msg_accept_confirm_common d0;
                        struct { /* v2 only, but 12 bytes reserved in v1 */
                                __be16 chid;
                                u8 eid[SMC_MAX_EID_LEN];
                                __be64 gid_ext;
                        } __packed d1;
                };
        };
};

struct smc_clc_msg_decline {        /* clc decline message */
        struct smc_clc_msg_hdr hdr;
        u8 id_for_peer[SMC_SYSTEMID_LEN]; /* sender peer_id */
        __be32 peer_diagnosis;        /* diagnosis information */
#if defined(__BIG_ENDIAN_BITFIELD)
        u8 os_type  : 4,
           reserved : 4;
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 reserved : 4,
           os_type  : 4;
#endif
        u8 reserved2[3];
        struct smc_clc_msg_trail trl; /* eye catcher "SMCD" or "SMCR" EBCDIC */
} __aligned(4);

#define SMC_DECL_DIAG_COUNT_V2        4 /* no. of additional peer diagnosis codes */

struct smc_clc_msg_decline_v2 {        /* clc decline message */
        struct smc_clc_msg_hdr hdr;
        u8 id_for_peer[SMC_SYSTEMID_LEN]; /* sender peer_id */
        __be32 peer_diagnosis;        /* diagnosis information */
#if defined(__BIG_ENDIAN_BITFIELD)
        u8 os_type  : 4,
           reserved : 4;
#elif defined(__LITTLE_ENDIAN_BITFIELD)
        u8 reserved : 4,
           os_type  : 4;
#endif
        u8 reserved2[3];
        __be32 peer_diagnosis_v2[SMC_DECL_DIAG_COUNT_V2];
        struct smc_clc_msg_trail trl; /* eye catcher "SMCD" or "SMCR" EBCDIC */
} __aligned(4);

/* determine start of the prefix area within the proposal message */
static inline struct smc_clc_msg_proposal_prefix *
smc_clc_proposal_get_prefix(struct smc_clc_msg_proposal *pclc)
{
        return (struct smc_clc_msg_proposal_prefix *)
               ((u8 *)pclc + sizeof(*pclc) + ntohs(pclc->iparea_offset));
}

static inline bool smcr_indicated(int smc_type)
{
        return smc_type == SMC_TYPE_R || smc_type == SMC_TYPE_B;
}

static inline bool smcd_indicated(int smc_type)
{
        return smc_type == SMC_TYPE_D || smc_type == SMC_TYPE_B;
}

static inline u8 smc_indicated_type(int is_smcd, int is_smcr)
{
        if (is_smcd && is_smcr)
                return SMC_TYPE_B;
        if (is_smcd)
                return SMC_TYPE_D;
        if (is_smcr)
                return SMC_TYPE_R;
        return SMC_TYPE_N;
}

/* get SMC-D info from proposal message */
static inline struct smc_clc_msg_smcd *
smc_get_clc_msg_smcd(struct smc_clc_msg_proposal *prop)
{
        if (smcd_indicated(prop->hdr.typev1) &&
            ntohs(prop->iparea_offset) != sizeof(struct smc_clc_msg_smcd))
                return NULL;

        return (struct smc_clc_msg_smcd *)(prop + 1);
}

static inline struct smc_clc_v2_extension *
smc_get_clc_v2_ext(struct smc_clc_msg_proposal *prop)
{
        struct smc_clc_msg_smcd *prop_smcd = smc_get_clc_msg_smcd(prop);

        if (!prop_smcd || !ntohs(prop_smcd->v2_ext_offset))
                return NULL;

        return (struct smc_clc_v2_extension *)
               ((u8 *)prop_smcd +
               offsetof(struct smc_clc_msg_smcd, v2_ext_offset) +
               sizeof(prop_smcd->v2_ext_offset) +
               ntohs(prop_smcd->v2_ext_offset));
}

static inline struct smc_clc_smcd_v2_extension *
smc_get_clc_smcd_v2_ext(struct smc_clc_v2_extension *prop_v2ext)
{
        if (!prop_v2ext)
                return NULL;
        if (!ntohs(prop_v2ext->hdr.smcd_v2_ext_offset))
                return NULL;

        return (struct smc_clc_smcd_v2_extension *)
                ((u8 *)prop_v2ext +
                 offsetof(struct smc_clc_v2_extension, hdr) +
                 offsetof(struct smc_clnt_opts_area_hdr, smcd_v2_ext_offset) +
                 sizeof(prop_v2ext->hdr.smcd_v2_ext_offset) +
                 ntohs(prop_v2ext->hdr.smcd_v2_ext_offset));
}

static inline struct smc_clc_first_contact_ext *
smc_get_clc_first_contact_ext(struct smc_clc_msg_accept_confirm *clc,
                              bool is_smcd)
{
        int clc_v2_len;

        if (clc->hdr.version == SMC_V1 ||
            !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
                return NULL;

        if (is_smcd)
                clc_v2_len =
                        offsetofend(struct smc_clc_msg_accept_confirm, d1);
        else
                clc_v2_len =
                        offsetofend(struct smc_clc_msg_accept_confirm, r1);

        return (struct smc_clc_first_contact_ext *)(((u8 *)clc) + clc_v2_len);
}

struct smcd_dev;
struct smc_init_info;

int smc_clc_prfx_match(struct socket *clcsock,
                       struct smc_clc_msg_proposal_prefix *prop);
int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen,
                     u8 expected_type, unsigned long timeout);
int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info, u8 version);
int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini);
int smc_clc_send_confirm(struct smc_sock *smc, bool clnt_first_contact,
                         u8 version, u8 *eid, struct smc_init_info *ini);
int smc_clc_send_accept(struct smc_sock *smc, bool srv_first_contact,
                        u8 version, u8 *negotiated_eid, struct smc_init_info *ini);
int smc_clc_srv_v2x_features_validate(struct smc_sock *smc,
                                      struct smc_clc_msg_proposal *pclc,
                                      struct smc_init_info *ini);
int smc_clc_clnt_v2x_features_validate(struct smc_clc_first_contact_ext *fce,
                                       struct smc_init_info *ini);
int smc_clc_v2x_features_confirm_check(struct smc_clc_msg_accept_confirm *cclc,
                                       struct smc_init_info *ini);
void smc_clc_init(void) __init;
void smc_clc_exit(void);
void smc_clc_get_hostname(u8 **host);
bool smc_clc_match_eid(u8 *negotiated_eid,
                       struct smc_clc_v2_extension *smc_v2_ext,
                       u8 *peer_eid, u8 *local_eid);
int smc_clc_ueid_count(void);
int smc_nl_dump_ueid(struct sk_buff *skb, struct netlink_callback *cb);
int smc_nl_add_ueid(struct sk_buff *skb, struct genl_info *info);
int smc_nl_remove_ueid(struct sk_buff *skb, struct genl_info *info);
int smc_nl_flush_ueid(struct sk_buff *skb, struct genl_info *info);
int smc_nl_dump_seid(struct sk_buff *skb, struct netlink_callback *cb);
int smc_nl_enable_seid(struct sk_buff *skb, struct genl_info *info);
int smc_nl_disable_seid(struct sk_buff *skb, struct genl_info *info);

#endif




















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 
    1 


































   85 
























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* AF_RXRPC internal definitions
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/atomic.h>
#include <linux/seqlock.h>
#include <linux/win_minmax.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include <keys/rxrpc-type.h>
#include "protocol.h"

#define FCRYPT_BSIZE 8
struct rxrpc_crypt {
        union {
                u8        x[FCRYPT_BSIZE];
                __be32        n[2];
        };
} __attribute__((aligned(8)));

#define rxrpc_queue_work(WS)        queue_work(rxrpc_workqueue, (WS))
#define rxrpc_queue_delayed_work(WS,D)        \
        queue_delayed_work(rxrpc_workqueue, (WS), (D))

struct key_preparsed_payload;
struct rxrpc_connection;
struct rxrpc_txbuf;

/*
 * Mark applied to socket buffers in skb->mark.  skb->priority is used
 * to pass supplementary information.
 */
enum rxrpc_skb_mark {
        RXRPC_SKB_MARK_PACKET,                /* Received packet */
        RXRPC_SKB_MARK_ERROR,                /* Error notification */
        RXRPC_SKB_MARK_SERVICE_CONN_SECURED, /* Service connection response has been verified */
        RXRPC_SKB_MARK_REJECT_BUSY,        /* Reject with BUSY */
        RXRPC_SKB_MARK_REJECT_ABORT,        /* Reject with ABORT (code in skb->priority) */
};

/*
 * sk_state for RxRPC sockets
 */
enum {
        RXRPC_UNBOUND = 0,
        RXRPC_CLIENT_UNBOUND,                /* Unbound socket used as client */
        RXRPC_CLIENT_BOUND,                /* client local address bound */
        RXRPC_SERVER_BOUND,                /* server local address bound */
        RXRPC_SERVER_BOUND2,                /* second server local address bound */
        RXRPC_SERVER_LISTENING,                /* server listening for connections */
        RXRPC_SERVER_LISTEN_DISABLED,        /* server listening disabled */
        RXRPC_CLOSE,                        /* socket is being closed */
};

/*
 * Per-network namespace data.
 */
struct rxrpc_net {
        struct proc_dir_entry        *proc_net;        /* Subdir in /proc/net */
        u32                        epoch;                /* Local epoch for detecting local-end reset */
        struct list_head        calls;                /* List of calls active in this namespace */
        spinlock_t                call_lock;        /* Lock for ->calls */
        atomic_t                nr_calls;        /* Count of allocated calls */

        atomic_t                nr_conns;
        struct list_head        bundle_proc_list; /* List of bundles for proc */
        struct list_head        conn_proc_list;        /* List of conns in this namespace for proc */
        struct list_head        service_conns;        /* Service conns in this namespace */
        rwlock_t                conn_lock;        /* Lock for ->conn_proc_list, ->service_conns */
        struct work_struct        service_conn_reaper;
        struct timer_list        service_conn_reap_timer;

        bool                        live;

        atomic_t                nr_client_conns;

        struct hlist_head        local_endpoints;
        struct mutex                local_mutex;        /* Lock for ->local_endpoints */

        DECLARE_HASHTABLE        (peer_hash, 10);
        spinlock_t                peer_hash_lock;        /* Lock for ->peer_hash */

#define RXRPC_KEEPALIVE_TIME 20 /* NAT keepalive time in seconds */
        u8                        peer_keepalive_cursor;
        time64_t                peer_keepalive_base;
        struct list_head        peer_keepalive[32];
        struct list_head        peer_keepalive_new;
        struct timer_list        peer_keepalive_timer;
        struct work_struct        peer_keepalive_work;

        atomic_t                stat_tx_data;
        atomic_t                stat_tx_data_retrans;
        atomic_t                stat_tx_data_send;
        atomic_t                stat_tx_data_send_frag;
        atomic_t                stat_tx_data_send_fail;
        atomic_t                stat_tx_data_underflow;
        atomic_t                stat_tx_data_cwnd_reset;
        atomic_t                stat_rx_data;
        atomic_t                stat_rx_data_reqack;
        atomic_t                stat_rx_data_jumbo;

        atomic_t                stat_tx_ack_fill;
        atomic_t                stat_tx_ack_send;
        atomic_t                stat_tx_ack_skip;
        atomic_t                stat_tx_acks[256];
        atomic_t                stat_rx_acks[256];

        atomic_t                stat_why_req_ack[8];

        atomic_t                stat_io_loop;
};

/*
 * Service backlog preallocation.
 *
 * This contains circular buffers of preallocated peers, connections and calls
 * for incoming service calls and their head and tail pointers.  This allows
 * calls to be set up in the data_ready handler, thereby avoiding the need to
 * shuffle packets around so much.
 */
struct rxrpc_backlog {
        unsigned short                peer_backlog_head;
        unsigned short                peer_backlog_tail;
        unsigned short                conn_backlog_head;
        unsigned short                conn_backlog_tail;
        unsigned short                call_backlog_head;
        unsigned short                call_backlog_tail;
#define RXRPC_BACKLOG_MAX        32
        struct rxrpc_peer        *peer_backlog[RXRPC_BACKLOG_MAX];
        struct rxrpc_connection        *conn_backlog[RXRPC_BACKLOG_MAX];
        struct rxrpc_call        *call_backlog[RXRPC_BACKLOG_MAX];
};

/*
 * RxRPC socket definition
 */
struct rxrpc_sock {
        /* WARNING: sk has to be the first member */
        struct sock                sk;
        rxrpc_notify_new_call_t        notify_new_call; /* Func to notify of new call */
        rxrpc_discard_new_call_t discard_new_call; /* Func to discard a new call */
        struct rxrpc_local        *local;                /* local endpoint */
        struct rxrpc_backlog        *backlog;        /* Preallocation for services */
        spinlock_t                incoming_lock;        /* Incoming call vs service shutdown lock */
        struct list_head        sock_calls;        /* List of calls owned by this socket */
        struct list_head        to_be_accepted;        /* calls awaiting acceptance */
        struct list_head        recvmsg_q;        /* Calls awaiting recvmsg's attention  */
        spinlock_t                recvmsg_lock;        /* Lock for recvmsg_q */
        struct key                *key;                /* security for this socket */
        struct key                *securities;        /* list of server security descriptors */
        struct rb_root                calls;                /* User ID -> call mapping */
        unsigned long                flags;
#define RXRPC_SOCK_CONNECTED                0        /* connect_srx is set */
        rwlock_t                call_lock;        /* lock for calls */
        u32                        min_sec_level;        /* minimum security level */
#define RXRPC_SECURITY_MAX        RXRPC_SECURITY_ENCRYPT
        bool                        exclusive;        /* Exclusive connection for a client socket */
        u16                        second_service;        /* Additional service bound to the endpoint */
        struct {
                /* Service upgrade information */
                u16                from;                /* Service ID to upgrade (if not 0) */
                u16                to;                /* service ID to upgrade to */
        } service_upgrade;
        sa_family_t                family;                /* Protocol family created with */
        struct sockaddr_rxrpc        srx;                /* Primary Service/local addresses */
        struct sockaddr_rxrpc        connect_srx;        /* Default client address from connect() */
};

#define rxrpc_sk(__sk) container_of((__sk), struct rxrpc_sock, sk)

/*
 * CPU-byteorder normalised Rx packet header.
 */
struct rxrpc_host_header {
        u32                epoch;                /* client boot timestamp */
        u32                cid;                /* connection and channel ID */
        u32                callNumber;        /* call ID (0 for connection-level packets) */
        u32                seq;                /* sequence number of pkt in call stream */
        u32                serial;                /* serial number of pkt sent to network */
        u8                type;                /* packet type */
        u8                flags;                /* packet flags */
        u8                userStatus;        /* app-layer defined status */
        u8                securityIndex;        /* security protocol ID */
        union {
                u16        _rsvd;                /* reserved */
                u16        cksum;                /* kerberos security checksum */
        };
        u16                serviceId;        /* service ID */
} __packed;

/*
 * RxRPC socket buffer private variables
 * - max 48 bytes (struct sk_buff::cb)
 */
struct rxrpc_skb_priv {
        union {
                struct rxrpc_connection *conn;        /* Connection referred to (poke packet) */
                struct {
                        u16                offset;                /* Offset of data */
                        u16                len;                /* Length of data */
                        u8                flags;
#define RXRPC_RX_VERIFIED        0x01
                };
                struct {
                        rxrpc_seq_t        first_ack;        /* First packet in acks table */
                        rxrpc_seq_t        prev_ack;        /* Highest seq seen */
                        rxrpc_serial_t        acked_serial;        /* Packet in response to (or 0) */
                        u8                reason;                /* Reason for ack */
                        u8                nr_acks;        /* Number of acks+nacks */
                        u8                nr_nacks;        /* Number of nacks */
                } ack;
        };
        struct rxrpc_host_header hdr;        /* RxRPC packet header from this packet */
};

#define rxrpc_skb(__skb) ((struct rxrpc_skb_priv *) &(__skb)->cb)

/*
 * RxRPC security module interface
 */
struct rxrpc_security {
        const char                *name;                /* name of this service */
        u8                        security_index;        /* security type provided */
        u32                        no_key_abort;        /* Abort code indicating no key */

        /* Initialise a security service */
        int (*init)(void);

        /* Clean up a security service */
        void (*exit)(void);

        /* Parse the information from a server key */
        int (*preparse_server_key)(struct key_preparsed_payload *);

        /* Clean up the preparse buffer after parsing a server key */
        void (*free_preparse_server_key)(struct key_preparsed_payload *);

        /* Destroy the payload of a server key */
        void (*destroy_server_key)(struct key *);

        /* Describe a server key */
        void (*describe_server_key)(const struct key *, struct seq_file *);

        /* initialise a connection's security */
        int (*init_connection_security)(struct rxrpc_connection *,
                                        struct rxrpc_key_token *);

        /* Work out how much data we can store in a packet, given an estimate
         * of the amount of data remaining and allocate a data buffer.
         */
        struct rxrpc_txbuf *(*alloc_txbuf)(struct rxrpc_call *call, size_t remaining, gfp_t gfp);

        /* impose security on a packet */
        int (*secure_packet)(struct rxrpc_call *, struct rxrpc_txbuf *);

        /* verify the security on a received packet */
        int (*verify_packet)(struct rxrpc_call *, struct sk_buff *);

        /* Free crypto request on a call */
        void (*free_call_crypto)(struct rxrpc_call *);

        /* issue a challenge */
        int (*issue_challenge)(struct rxrpc_connection *);

        /* respond to a challenge */
        int (*respond_to_challenge)(struct rxrpc_connection *,
                                    struct sk_buff *);

        /* verify a response */
        int (*verify_response)(struct rxrpc_connection *,
                               struct sk_buff *);

        /* clear connection security */
        void (*clear)(struct rxrpc_connection *);
};

/*
 * RxRPC local transport endpoint description
 * - owned by a single AF_RXRPC socket
 * - pointed to by transport socket struct sk_user_data
 */
struct rxrpc_local {
        struct rcu_head                rcu;
        atomic_t                active_users;        /* Number of users of the local endpoint */
        refcount_t                ref;                /* Number of references to the structure */
        struct net                *net;                /* The network namespace */
        struct rxrpc_net        *rxnet;                /* Our bits in the network namespace */
        struct hlist_node        link;
        struct socket                *socket;        /* my UDP socket */
        struct task_struct        *io_thread;
        struct completion        io_thread_ready; /* Indication that the I/O thread started */
        struct page_frag_cache        tx_alloc;        /* Tx control packet allocation (I/O thread only) */
        struct rxrpc_sock        *service;        /* Service(s) listening on this endpoint */
#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
        struct sk_buff_head        rx_delay_queue;        /* Delay injection queue */
#endif
        struct sk_buff_head        rx_queue;        /* Received packets */
        struct list_head        conn_attend_q;        /* Conns requiring immediate attention */
        struct list_head        call_attend_q;        /* Calls requiring immediate attention */

        struct rb_root                client_bundles;        /* Client connection bundles by socket params */
        spinlock_t                client_bundles_lock; /* Lock for client_bundles */
        bool                        kill_all_client_conns;
        struct list_head        idle_client_conns;
        struct timer_list        client_conn_reap_timer;
        unsigned long                client_conn_flags;
#define RXRPC_CLIENT_CONN_REAP_TIMER        0        /* The client conn reap timer expired */

        spinlock_t                lock;                /* access lock */
        rwlock_t                services_lock;        /* lock for services list */
        int                        debug_id;        /* debug ID for printks */
        bool                        dead;
        bool                        service_closed;        /* Service socket closed */
        struct idr                conn_ids;        /* List of connection IDs */
        struct list_head        new_client_calls; /* Newly created client calls need connection */
        spinlock_t                client_call_lock; /* Lock for ->new_client_calls */
        struct sockaddr_rxrpc        srx;                /* local address */
};

/*
 * RxRPC remote transport endpoint definition
 * - matched by local endpoint, remote port, address and protocol type
 */
struct rxrpc_peer {
        struct rcu_head                rcu;                /* This must be first */
        refcount_t                ref;
        unsigned long                hash_key;
        struct hlist_node        hash_link;
        struct rxrpc_local        *local;
        struct hlist_head        error_targets;        /* targets for net error distribution */
        struct rb_root                service_conns;        /* Service connections */
        struct list_head        keepalive_link;        /* Link in net->peer_keepalive[] */
        time64_t                last_tx_at;        /* Last time packet sent here */
        seqlock_t                service_conn_lock;
        spinlock_t                lock;                /* access lock */
        unsigned int                if_mtu;                /* interface MTU for this peer */
        unsigned int                mtu;                /* network MTU for this peer */
        unsigned int                maxdata;        /* data size (MTU - hdrsize) */
        unsigned short                hdrsize;        /* header size (IP + UDP + RxRPC) */
        int                        debug_id;        /* debug ID for printks */
        struct sockaddr_rxrpc        srx;                /* remote address */

        /* calculated RTT cache */
#define RXRPC_RTT_CACHE_SIZE 32
        spinlock_t                rtt_input_lock;        /* RTT lock for input routine */
        ktime_t                        rtt_last_req;        /* Time of last RTT request */
        unsigned int                rtt_count;        /* Number of samples we've got */

        u32                        srtt_us;        /* smoothed round trip time << 3 in usecs */
        u32                        mdev_us;        /* medium deviation                        */
        u32                        mdev_max_us;        /* maximal mdev for the last rtt period        */
        u32                        rttvar_us;        /* smoothed mdev_max                        */
        u32                        rto_us;                /* Retransmission timeout in usec */
        u8                        backoff;        /* Backoff timeout (as shift) */

        u8                        cong_ssthresh;        /* Congestion slow-start threshold */
};

/*
 * Keys for matching a connection.
 */
struct rxrpc_conn_proto {
        union {
                struct {
                        u32        epoch;                /* epoch of this connection */
                        u32        cid;                /* connection ID */
                };
                u64                index_key;
        };
};

struct rxrpc_conn_parameters {
        struct rxrpc_local        *local;                /* Representation of local endpoint */
        struct rxrpc_peer        *peer;                /* Representation of remote endpoint */
        struct key                *key;                /* Security details */
        bool                        exclusive;        /* T if conn is exclusive */
        bool                        upgrade;        /* T if service ID can be upgraded */
        u16                        service_id;        /* Service ID for this connection */
        u32                        security_level;        /* Security level selected */
};

/*
 * Call completion condition (state == RXRPC_CALL_COMPLETE).
 */
enum rxrpc_call_completion {
        RXRPC_CALL_SUCCEEDED,                /* - Normal termination */
        RXRPC_CALL_REMOTELY_ABORTED,        /* - call aborted by peer */
        RXRPC_CALL_LOCALLY_ABORTED,        /* - call aborted locally on error or close */
        RXRPC_CALL_LOCAL_ERROR,                /* - call failed due to local error */
        RXRPC_CALL_NETWORK_ERROR,        /* - call terminated by network error */
        NR__RXRPC_CALL_COMPLETIONS
};

/*
 * Bits in the connection flags.
 */
enum rxrpc_conn_flag {
        RXRPC_CONN_IN_SERVICE_CONNS,        /* Conn is in peer->service_conns */
        RXRPC_CONN_DONT_REUSE,                /* Don't reuse this connection */
        RXRPC_CONN_PROBING_FOR_UPGRADE,        /* Probing for service upgrade */
        RXRPC_CONN_FINAL_ACK_0,                /* Need final ACK for channel 0 */
        RXRPC_CONN_FINAL_ACK_1,                /* Need final ACK for channel 1 */
        RXRPC_CONN_FINAL_ACK_2,                /* Need final ACK for channel 2 */
        RXRPC_CONN_FINAL_ACK_3,                /* Need final ACK for channel 3 */
};

#define RXRPC_CONN_FINAL_ACK_MASK ((1UL << RXRPC_CONN_FINAL_ACK_0) |        \
                                   (1UL << RXRPC_CONN_FINAL_ACK_1) |        \
                                   (1UL << RXRPC_CONN_FINAL_ACK_2) |        \
                                   (1UL << RXRPC_CONN_FINAL_ACK_3))

/*
 * Events that can be raised upon a connection.
 */
enum rxrpc_conn_event {
        RXRPC_CONN_EV_CHALLENGE,        /* Send challenge packet */
        RXRPC_CONN_EV_ABORT_CALLS,        /* Abort attached calls */
};

/*
 * The connection protocol state.
 */
enum rxrpc_conn_proto_state {
        RXRPC_CONN_UNUSED,                /* Connection not yet attempted */
        RXRPC_CONN_CLIENT_UNSECURED,        /* Client connection needs security init */
        RXRPC_CONN_CLIENT,                /* Client connection */
        RXRPC_CONN_SERVICE_PREALLOC,        /* Service connection preallocation */
        RXRPC_CONN_SERVICE_UNSECURED,        /* Service unsecured connection */
        RXRPC_CONN_SERVICE_CHALLENGING,        /* Service challenging for security */
        RXRPC_CONN_SERVICE,                /* Service secured connection */
        RXRPC_CONN_ABORTED,                /* Conn aborted */
        RXRPC_CONN__NR_STATES
};

/*
 * RxRPC client connection bundle.
 */
struct rxrpc_bundle {
        struct rxrpc_local        *local;                /* Representation of local endpoint */
        struct rxrpc_peer        *peer;                /* Remote endpoint */
        struct key                *key;                /* Security details */
        struct list_head        proc_link;        /* Link in net->bundle_proc_list */
        const struct rxrpc_security *security;        /* applied security module */
        refcount_t                ref;
        atomic_t                active;                /* Number of active users */
        unsigned int                debug_id;
        u32                        security_level;        /* Security level selected */
        u16                        service_id;        /* Service ID for this connection */
        bool                        try_upgrade;        /* True if the bundle is attempting upgrade */
        bool                        exclusive;        /* T if conn is exclusive */
        bool                        upgrade;        /* T if service ID can be upgraded */
        unsigned short                alloc_error;        /* Error from last conn allocation */
        struct rb_node                local_node;        /* Node in local->client_conns */
        struct list_head        waiting_calls;        /* Calls waiting for channels */
        unsigned long                avail_chans;        /* Mask of available channels */
        unsigned int                conn_ids[4];        /* Connection IDs. */
        struct rxrpc_connection        *conns[4];        /* The connections in the bundle (max 4) */
};

/*
 * RxRPC connection definition
 * - matched by { local, peer, epoch, conn_id, direction }
 * - each connection can only handle four simultaneous calls
 */
struct rxrpc_connection {
        struct rxrpc_conn_proto        proto;
        struct rxrpc_local        *local;                /* Representation of local endpoint */
        struct rxrpc_peer        *peer;                /* Remote endpoint */
        struct rxrpc_net        *rxnet;                /* Network namespace to which call belongs */
        struct key                *key;                /* Security details */
        struct list_head        attend_link;        /* Link in local->conn_attend_q */

        refcount_t                ref;
        atomic_t                active;                /* Active count for service conns */
        struct rcu_head                rcu;
        struct list_head        cache_link;

        unsigned char                act_chans;        /* Mask of active channels */
        struct rxrpc_channel {
                unsigned long                final_ack_at;        /* Time at which to issue final ACK */
                struct rxrpc_call        *call;                /* Active call */
                unsigned int                call_debug_id;        /* call->debug_id */
                u32                        call_id;        /* ID of current call */
                u32                        call_counter;        /* Call ID counter */
                u32                        last_call;        /* ID of last call */
                u8                        last_type;        /* Type of last packet */
                union {
                        u32                last_seq;
                        u32                last_abort;
                };
        } channels[RXRPC_MAXCALLS];

        struct timer_list        timer;                /* Conn event timer */
        struct work_struct        processor;        /* connection event processor */
        struct work_struct        destructor;        /* In-process-context destroyer */
        struct rxrpc_bundle        *bundle;        /* Client connection bundle */
        struct rb_node                service_node;        /* Node in peer->service_conns */
        struct list_head        proc_link;        /* link in procfs list */
        struct list_head        link;                /* link in master connection list */
        struct sk_buff_head        rx_queue;        /* received conn-level packets */
        struct page_frag_cache        tx_data_alloc;        /* Tx DATA packet allocation */
        struct mutex                tx_data_alloc_lock;

        struct mutex                security_lock;        /* Lock for security management */
        const struct rxrpc_security *security;        /* applied security module */
        union {
                struct {
                        struct crypto_sync_skcipher *cipher;        /* encryption handle */
                        struct rxrpc_crypt csum_iv;        /* packet checksum base */
                        u32        nonce;                /* response re-use preventer */
                } rxkad;
        };
        unsigned long                flags;
        unsigned long                events;
        unsigned long                idle_timestamp;        /* Time at which last became idle */
        spinlock_t                state_lock;        /* state-change lock */
        enum rxrpc_conn_proto_state state;        /* current state of connection */
        enum rxrpc_call_completion completion;        /* Completion condition */
        s32                        abort_code;        /* Abort code of connection abort */
        int                        debug_id;        /* debug ID for printks */
        rxrpc_serial_t                tx_serial;        /* Outgoing packet serial number counter */
        unsigned int                hi_serial;        /* highest serial number received */
        u32                        service_id;        /* Service ID, possibly upgraded */
        u32                        security_level;        /* Security level selected */
        u8                        security_ix;        /* security type */
        u8                        out_clientflag;        /* RXRPC_CLIENT_INITIATED if we are client */
        u8                        bundle_shift;        /* Index into bundle->avail_chans */
        bool                        exclusive;        /* T if conn is exclusive */
        bool                        upgrade;        /* T if service ID can be upgraded */
        u16                        orig_service_id; /* Originally requested service ID */
        short                        error;                /* Local error code */
};

static inline bool rxrpc_to_server(const struct rxrpc_skb_priv *sp)
{
        return sp->hdr.flags & RXRPC_CLIENT_INITIATED;
}

static inline bool rxrpc_to_client(const struct rxrpc_skb_priv *sp)
{
        return !rxrpc_to_server(sp);
}

/*
 * Flags in call->flags.
 */
enum rxrpc_call_flag {
        RXRPC_CALL_RELEASED,                /* call has been released - no more message to userspace */
        RXRPC_CALL_HAS_USERID,                /* has a user ID attached */
        RXRPC_CALL_IS_SERVICE,                /* Call is service call */
        RXRPC_CALL_EXPOSED,                /* The call was exposed to the world */
        RXRPC_CALL_RX_LAST,                /* Received the last packet (at rxtx_top) */
        RXRPC_CALL_TX_LAST,                /* Last packet in Tx buffer (at rxtx_top) */
        RXRPC_CALL_TX_ALL_ACKED,        /* Last packet has been hard-acked */
        RXRPC_CALL_SEND_PING,                /* A ping will need to be sent */
        RXRPC_CALL_RETRANS_TIMEOUT,        /* Retransmission due to timeout occurred */
        RXRPC_CALL_BEGAN_RX_TIMER,        /* We began the expect_rx_by timer */
        RXRPC_CALL_RX_HEARD,                /* The peer responded at least once to this call */
        RXRPC_CALL_DISCONNECTED,        /* The call has been disconnected */
        RXRPC_CALL_KERNEL,                /* The call was made by the kernel */
        RXRPC_CALL_UPGRADE,                /* Service upgrade was requested for the call */
        RXRPC_CALL_EXCLUSIVE,                /* The call uses a once-only connection */
        RXRPC_CALL_RX_IS_IDLE,                /* recvmsg() is idle - send an ACK */
        RXRPC_CALL_RECVMSG_READ_ALL,        /* recvmsg() read all of the received data */
};

/*
 * Events that can be raised on a call.
 */
enum rxrpc_call_event {
        RXRPC_CALL_EV_ACK_LOST,                /* ACK may be lost, send ping */
        RXRPC_CALL_EV_INITIAL_PING,        /* Send initial ping for a new service call */
};

/*
 * The states that a call can be in.
 */
enum rxrpc_call_state {
        RXRPC_CALL_UNINITIALISED,
        RXRPC_CALL_CLIENT_AWAIT_CONN,        /* - client waiting for connection to become available */
        RXRPC_CALL_CLIENT_SEND_REQUEST,        /* - client sending request phase */
        RXRPC_CALL_CLIENT_AWAIT_REPLY,        /* - client awaiting reply */
        RXRPC_CALL_CLIENT_RECV_REPLY,        /* - client receiving reply phase */
        RXRPC_CALL_SERVER_PREALLOC,        /* - service preallocation */
        RXRPC_CALL_SERVER_SECURING,        /* - server securing request connection */
        RXRPC_CALL_SERVER_RECV_REQUEST,        /* - server receiving request */
        RXRPC_CALL_SERVER_ACK_REQUEST,        /* - server pending ACK of request */
        RXRPC_CALL_SERVER_SEND_REPLY,        /* - server sending reply */
        RXRPC_CALL_SERVER_AWAIT_ACK,        /* - server awaiting final ACK */
        RXRPC_CALL_COMPLETE,                /* - call complete */
        NR__RXRPC_CALL_STATES
};

/*
 * Call Tx congestion management modes.
 */
enum rxrpc_congest_mode {
        RXRPC_CALL_SLOW_START,
        RXRPC_CALL_CONGEST_AVOIDANCE,
        RXRPC_CALL_PACKET_LOSS,
        RXRPC_CALL_FAST_RETRANSMIT,
        NR__RXRPC_CONGEST_MODES
};

/*
 * RxRPC call definition
 * - matched by { connection, call_id }
 */
struct rxrpc_call {
        struct rcu_head                rcu;
        struct rxrpc_connection        *conn;                /* connection carrying call */
        struct rxrpc_bundle        *bundle;        /* Connection bundle to use */
        struct rxrpc_peer        *peer;                /* Peer record for remote address */
        struct rxrpc_local        *local;                /* Representation of local endpoint */
        struct rxrpc_sock __rcu        *socket;        /* socket responsible */
        struct rxrpc_net        *rxnet;                /* Network namespace to which call belongs */
        struct key                *key;                /* Security details */
        const struct rxrpc_security *security;        /* applied security module */
        struct mutex                user_mutex;        /* User access mutex */
        struct sockaddr_rxrpc        dest_srx;        /* Destination address */
        ktime_t                        delay_ack_at;        /* When DELAY ACK needs to happen */
        ktime_t                        ack_lost_at;        /* When ACK is figured as lost */
        ktime_t                        resend_at;        /* When next resend needs to happen */
        ktime_t                        ping_at;        /* When next to send a ping */
        ktime_t                        keepalive_at;        /* When next to send a keepalive ping */
        ktime_t                        expect_rx_by;        /* When we expect to get a packet by */
        ktime_t                        expect_req_by;        /* When we expect to get a request DATA packet by */
        ktime_t                        expect_term_by;        /* When we expect call termination by */
        u32                        next_rx_timo;        /* Timeout for next Rx packet (ms) */
        u32                        next_req_timo;        /* Timeout for next Rx request packet (ms) */
        u32                        hard_timo;        /* Maximum lifetime or 0 (s) */
        struct timer_list        timer;                /* Combined event timer */
        struct work_struct        destroyer;        /* In-process-context destroyer */
        rxrpc_notify_rx_t        notify_rx;        /* kernel service Rx notification function */
        struct list_head        link;                /* link in master call list */
        struct list_head        wait_link;        /* Link in local->new_client_calls */
        struct hlist_node        error_link;        /* link in error distribution list */
        struct list_head        accept_link;        /* Link in rx->acceptq */
        struct list_head        recvmsg_link;        /* Link in rx->recvmsg_q */
        struct list_head        sock_link;        /* Link in rx->sock_calls */
        struct rb_node                sock_node;        /* Node in rx->calls */
        struct list_head        attend_link;        /* Link in local->call_attend_q */
        struct rxrpc_txbuf        *tx_pending;        /* Tx buffer being filled */
        wait_queue_head_t        waitq;                /* Wait queue for channel or Tx */
        s64                        tx_total_len;        /* Total length left to be transmitted (or -1) */
        unsigned long                user_call_ID;        /* user-defined call ID */
        unsigned long                flags;
        unsigned long                events;
        spinlock_t                notify_lock;        /* Kernel notification lock */
        unsigned int                send_abort_why; /* Why the abort [enum rxrpc_abort_reason] */
        s32                        send_abort;        /* Abort code to be sent */
        short                        send_abort_err;        /* Error to be associated with the abort */
        rxrpc_seq_t                send_abort_seq;        /* DATA packet that incurred the abort (or 0) */
        s32                        abort_code;        /* Local/remote abort code */
        int                        error;                /* Local error incurred */
        enum rxrpc_call_state        _state;                /* Current state of call (needs barrier) */
        enum rxrpc_call_completion completion;        /* Call completion condition */
        refcount_t                ref;
        u8                        security_ix;        /* Security type */
        enum rxrpc_interruptibility interruptibility; /* At what point call may be interrupted */
        u32                        call_id;        /* call ID on connection  */
        u32                        cid;                /* connection ID plus channel index */
        u32                        security_level;        /* Security level selected */
        int                        debug_id;        /* debug ID for printks */
        unsigned short                rx_pkt_offset;        /* Current recvmsg packet offset */
        unsigned short                rx_pkt_len;        /* Current recvmsg packet len */

        /* Transmitted data tracking. */
        spinlock_t                tx_lock;        /* Transmit queue lock */
        struct list_head        tx_sendmsg;        /* Sendmsg prepared packets */
        struct list_head        tx_buffer;        /* Buffer of transmissible packets */
        rxrpc_seq_t                tx_bottom;        /* First packet in buffer */
        rxrpc_seq_t                tx_transmitted;        /* Highest packet transmitted */
        rxrpc_seq_t                tx_prepared;        /* Highest Tx slot prepared. */
        rxrpc_seq_t                tx_top;                /* Highest Tx slot allocated. */
        u16                        tx_backoff;        /* Delay to insert due to Tx failure (ms) */
        u8                        tx_winsize;        /* Maximum size of Tx window */
#define RXRPC_TX_MAX_WINDOW        128
        ktime_t                        tx_last_sent;        /* Last time a transmission occurred */

        /* Received data tracking */
        struct sk_buff_head        recvmsg_queue;        /* Queue of packets ready for recvmsg() */
        struct sk_buff_head        rx_oos_queue;        /* Queue of out of sequence packets */

        rxrpc_seq_t                rx_highest_seq;        /* Higest sequence number received */
        rxrpc_seq_t                rx_consumed;        /* Highest packet consumed */
        rxrpc_serial_t                rx_serial;        /* Highest serial received for this call */
        u8                        rx_winsize;        /* Size of Rx window */

        /* TCP-style slow-start congestion control [RFC5681].  Since the SMSS
         * is fixed, we keep these numbers in terms of segments (ie. DATA
         * packets) rather than bytes.
         */
#define RXRPC_TX_SMSS                RXRPC_JUMBO_DATALEN
#define RXRPC_MIN_CWND                4
        u8                        cong_cwnd;        /* Congestion window size */
        u8                        cong_extra;        /* Extra to send for congestion management */
        u8                        cong_ssthresh;        /* Slow-start threshold */
        enum rxrpc_congest_mode        cong_mode:8;        /* Congestion management mode */
        u8                        cong_dup_acks;        /* Count of ACKs showing missing packets */
        u8                        cong_cumul_acks; /* Cumulative ACK count */
        ktime_t                        cong_tstamp;        /* Last time cwnd was changed */
        struct sk_buff                *cong_last_nack; /* Last ACK with nacks received */

        /* Receive-phase ACK management (ACKs we send). */
        u8                        ackr_reason;        /* reason to ACK */
        u16                        ackr_sack_base;        /* Starting slot in SACK table ring */
        rxrpc_seq_t                ackr_window;        /* Base of SACK window */
        rxrpc_seq_t                ackr_wtop;        /* Base of SACK window */
        unsigned int                ackr_nr_unacked; /* Number of unacked packets */
        atomic_t                ackr_nr_consumed; /* Number of packets needing hard ACK */
        struct {
#define RXRPC_SACK_SIZE 256
                 /* SACK table for soft-acked packets */
                u8                ackr_sack_table[RXRPC_SACK_SIZE];
        } __aligned(8);

        /* RTT management */
        rxrpc_serial_t                rtt_serial[4];        /* Serial number of DATA or PING sent */
        ktime_t                        rtt_sent_at[4];        /* Time packet sent */
        unsigned long                rtt_avail;        /* Mask of available slots in bits 0-3,
                                                 * Mask of pending samples in 8-11 */
#define RXRPC_CALL_RTT_AVAIL_MASK        0xf
#define RXRPC_CALL_RTT_PEND_SHIFT        8

        /* Transmission-phase ACK management (ACKs we've received). */
        ktime_t                        acks_latest_ts;        /* Timestamp of latest ACK received */
        rxrpc_seq_t                acks_first_seq;        /* first sequence number received */
        rxrpc_seq_t                acks_prev_seq;        /* Highest previousPacket received */
        rxrpc_seq_t                acks_hard_ack;        /* Latest hard-ack point */
        rxrpc_seq_t                acks_lowest_nak; /* Lowest NACK in the buffer (or ==tx_hard_ack) */
        rxrpc_serial_t                acks_highest_serial; /* Highest serial number ACK'd */
};

/*
 * Summary of a new ACK and the changes it made to the Tx buffer packet states.
 */
struct rxrpc_ack_summary {
        u16                        nr_acks;                /* Number of ACKs in packet */
        u16                        nr_new_acks;                /* Number of new ACKs in packet */
        u16                        nr_new_nacks;                /* Number of new nacks in packet */
        u16                        nr_retained_nacks;        /* Number of nacks retained between ACKs */
        u8                        ack_reason;
        bool                        saw_nacks;                /* Saw NACKs in packet */
        bool                        new_low_nack;                /* T if new low NACK found */
        bool                        retrans_timeo;                /* T if reTx due to timeout happened */
        u8                        flight_size;                /* Number of unreceived transmissions */
        /* Place to stash values for tracing */
        enum rxrpc_congest_mode        mode:8;
        u8                        cwnd;
        u8                        ssthresh;
        u8                        dup_acks;
        u8                        cumulative_acks;
};

/*
 * sendmsg() cmsg-specified parameters.
 */
enum rxrpc_command {
        RXRPC_CMD_SEND_DATA,                /* send data message */
        RXRPC_CMD_SEND_ABORT,                /* request abort generation */
        RXRPC_CMD_REJECT_BUSY,                /* [server] reject a call as busy */
        RXRPC_CMD_CHARGE_ACCEPT,        /* [server] charge accept preallocation */
};

struct rxrpc_call_params {
        s64                        tx_total_len;        /* Total Tx data length (if send data) */
        unsigned long                user_call_ID;        /* User's call ID */
        struct {
                u32                hard;                /* Maximum lifetime (sec) */
                u32                idle;                /* Max time since last data packet (msec) */
                u32                normal;                /* Max time since last call packet (msec) */
        } timeouts;
        u8                        nr_timeouts;        /* Number of timeouts specified */
        bool                        kernel;                /* T if kernel is making the call */
        enum rxrpc_interruptibility interruptibility; /* How is interruptible is the call? */
};

struct rxrpc_send_params {
        struct rxrpc_call_params call;
        u32                        abort_code;        /* Abort code to Tx (if abort) */
        enum rxrpc_command        command : 8;        /* The command to implement */
        bool                        exclusive;        /* Shared or exclusive call */
        bool                        upgrade;        /* If the connection is upgradeable */
};

/*
 * Buffer of data to be output as a packet.
 */
struct rxrpc_txbuf {
        struct list_head        call_link;        /* Link in call->tx_sendmsg/tx_buffer */
        struct list_head        tx_link;        /* Link in live Enc queue or Tx queue */
        ktime_t                        last_sent;        /* Time at which last transmitted */
        refcount_t                ref;
        rxrpc_seq_t                seq;                /* Sequence number of this packet */
        rxrpc_serial_t                serial;                /* Last serial number transmitted with */
        unsigned int                call_debug_id;
        unsigned int                debug_id;
        unsigned int                len;                /* Amount of data in buffer */
        unsigned int                space;                /* Remaining data space */
        unsigned int                offset;                /* Offset of fill point */
        unsigned int                flags;
#define RXRPC_TXBUF_WIRE_FLAGS        0xff                /* The wire protocol flags */
#define RXRPC_TXBUF_RESENT        0x100                /* Set if has been resent */
        __be16                        cksum;                /* Checksum to go in header */
        unsigned short                ack_rwind;        /* ACK receive window */
        u8 /*enum rxrpc_propose_ack_trace*/ ack_why;        /* If ack, why */
        u8                        nr_kvec;        /* Amount of kvec[] used */
        struct kvec                kvec[3];
};

static inline bool rxrpc_sending_to_server(const struct rxrpc_txbuf *txb)
{
        return txb->flags & RXRPC_CLIENT_INITIATED;
}

static inline bool rxrpc_sending_to_client(const struct rxrpc_txbuf *txb)
{
        return !rxrpc_sending_to_server(txb);
}

#include <trace/events/rxrpc.h>

/*
 * Allocate the next serial number on a connection.  0 must be skipped.
 */
static inline rxrpc_serial_t rxrpc_get_next_serial(struct rxrpc_connection *conn)
{
        rxrpc_serial_t serial;

        serial = conn->tx_serial;
        if (serial == 0)
                serial = 1;
        conn->tx_serial = serial + 1;
        return serial;
}

/*
 * af_rxrpc.c
 */
extern atomic_t rxrpc_n_rx_skbs;
extern struct workqueue_struct *rxrpc_workqueue;

/*
 * call_accept.c
 */
int rxrpc_service_prealloc(struct rxrpc_sock *, gfp_t);
void rxrpc_discard_prealloc(struct rxrpc_sock *);
bool rxrpc_new_incoming_call(struct rxrpc_local *local,
                             struct rxrpc_peer *peer,
                             struct rxrpc_connection *conn,
                             struct sockaddr_rxrpc *peer_srx,
                             struct sk_buff *skb);
int rxrpc_user_charge_accept(struct rxrpc_sock *, unsigned long);

/*
 * call_event.c
 */
void rxrpc_propose_ping(struct rxrpc_call *call, u32 serial,
                        enum rxrpc_propose_ack_trace why);
void rxrpc_propose_delay_ACK(struct rxrpc_call *, rxrpc_serial_t,
                             enum rxrpc_propose_ack_trace);
void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *);
void rxrpc_resend(struct rxrpc_call *call, struct sk_buff *ack_skb);

bool rxrpc_input_call_event(struct rxrpc_call *call, struct sk_buff *skb);

/*
 * call_object.c
 */
extern const char *const rxrpc_call_states[];
extern const char *const rxrpc_call_completions[];
extern struct kmem_cache *rxrpc_call_jar;

void rxrpc_poke_call(struct rxrpc_call *call, enum rxrpc_call_poke_trace what);
struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *, unsigned long);
struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *, gfp_t, unsigned int);
struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *,
                                         struct rxrpc_conn_parameters *,
                                         struct rxrpc_call_params *, gfp_t,
                                         unsigned int);
void rxrpc_start_call_timer(struct rxrpc_call *call);
void rxrpc_incoming_call(struct rxrpc_sock *, struct rxrpc_call *,
                         struct sk_buff *);
void rxrpc_release_call(struct rxrpc_sock *, struct rxrpc_call *);
void rxrpc_release_calls_on_socket(struct rxrpc_sock *);
void rxrpc_see_call(struct rxrpc_call *, enum rxrpc_call_trace);
struct rxrpc_call *rxrpc_try_get_call(struct rxrpc_call *, enum rxrpc_call_trace);
void rxrpc_get_call(struct rxrpc_call *, enum rxrpc_call_trace);
void rxrpc_put_call(struct rxrpc_call *, enum rxrpc_call_trace);
void rxrpc_cleanup_call(struct rxrpc_call *);
void rxrpc_destroy_all_calls(struct rxrpc_net *);

static inline bool rxrpc_is_service_call(const struct rxrpc_call *call)
{
        return test_bit(RXRPC_CALL_IS_SERVICE, &call->flags);
}

static inline bool rxrpc_is_client_call(const struct rxrpc_call *call)
{
        return !rxrpc_is_service_call(call);
}

/*
 * call_state.c
 */
bool rxrpc_set_call_completion(struct rxrpc_call *call,
                               enum rxrpc_call_completion compl,
                               u32 abort_code,
                               int error);
bool rxrpc_call_completed(struct rxrpc_call *call);
bool rxrpc_abort_call(struct rxrpc_call *call, rxrpc_seq_t seq,
                      u32 abort_code, int error, enum rxrpc_abort_reason why);
void rxrpc_prefail_call(struct rxrpc_call *call, enum rxrpc_call_completion compl,
                        int error);

static inline void rxrpc_set_call_state(struct rxrpc_call *call,
                                        enum rxrpc_call_state state)
{
        /* Order write of completion info before write of ->state. */
        smp_store_release(&call->_state, state);
        wake_up(&call->waitq);
}

static inline enum rxrpc_call_state __rxrpc_call_state(const struct rxrpc_call *call)
{
        return call->_state; /* Only inside I/O thread */
}

static inline bool __rxrpc_call_is_complete(const struct rxrpc_call *call)
{
        return __rxrpc_call_state(call) == RXRPC_CALL_COMPLETE;
}

static inline enum rxrpc_call_state rxrpc_call_state(const struct rxrpc_call *call)
{
        /* Order read ->state before read of completion info. */
        return smp_load_acquire(&call->_state);
}

static inline bool rxrpc_call_is_complete(const struct rxrpc_call *call)
{
        return rxrpc_call_state(call) == RXRPC_CALL_COMPLETE;
}

static inline bool rxrpc_call_has_failed(const struct rxrpc_call *call)
{
        return rxrpc_call_is_complete(call) && call->completion != RXRPC_CALL_SUCCEEDED;
}

/*
 * conn_client.c
 */
extern unsigned int rxrpc_reap_client_connections;
extern unsigned long rxrpc_conn_idle_client_expiry;
extern unsigned long rxrpc_conn_idle_client_fast_expiry;

void rxrpc_purge_client_connections(struct rxrpc_local *local);
struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *, enum rxrpc_bundle_trace);
void rxrpc_put_bundle(struct rxrpc_bundle *, enum rxrpc_bundle_trace);
int rxrpc_look_up_bundle(struct rxrpc_call *call, gfp_t gfp);
void rxrpc_connect_client_calls(struct rxrpc_local *local);
void rxrpc_expose_client_call(struct rxrpc_call *);
void rxrpc_disconnect_client_call(struct rxrpc_bundle *, struct rxrpc_call *);
void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle);
void rxrpc_discard_expired_client_conns(struct rxrpc_local *local);
void rxrpc_clean_up_local_conns(struct rxrpc_local *);

/*
 * conn_event.c
 */
void rxrpc_conn_retransmit_call(struct rxrpc_connection *conn, struct sk_buff *skb,
                                unsigned int channel);
int rxrpc_abort_conn(struct rxrpc_connection *conn, struct sk_buff *skb,
                     s32 abort_code, int err, enum rxrpc_abort_reason why);
void rxrpc_process_connection(struct work_struct *);
void rxrpc_process_delayed_final_acks(struct rxrpc_connection *, bool);
bool rxrpc_input_conn_packet(struct rxrpc_connection *conn, struct sk_buff *skb);
void rxrpc_input_conn_event(struct rxrpc_connection *conn, struct sk_buff *skb);

static inline bool rxrpc_is_conn_aborted(const struct rxrpc_connection *conn)
{
        /* Order reading the abort info after the state check. */
        return smp_load_acquire(&conn->state) == RXRPC_CONN_ABORTED;
}

/*
 * conn_object.c
 */
extern unsigned int rxrpc_connection_expiry;
extern unsigned int rxrpc_closed_conn_expiry;

void rxrpc_poke_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why);
struct rxrpc_connection *rxrpc_alloc_connection(struct rxrpc_net *, gfp_t);
struct rxrpc_connection *rxrpc_find_client_connection_rcu(struct rxrpc_local *,
                                                          struct sockaddr_rxrpc *,
                                                          struct sk_buff *);
void __rxrpc_disconnect_call(struct rxrpc_connection *, struct rxrpc_call *);
void rxrpc_disconnect_call(struct rxrpc_call *);
void rxrpc_kill_client_conn(struct rxrpc_connection *);
void rxrpc_queue_conn(struct rxrpc_connection *, enum rxrpc_conn_trace);
void rxrpc_see_connection(struct rxrpc_connection *, enum rxrpc_conn_trace);
struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *,
                                              enum rxrpc_conn_trace);
struct rxrpc_connection *rxrpc_get_connection_maybe(struct rxrpc_connection *,
                                                    enum rxrpc_conn_trace);
void rxrpc_put_connection(struct rxrpc_connection *, enum rxrpc_conn_trace);
void rxrpc_service_connection_reaper(struct work_struct *);
void rxrpc_destroy_all_connections(struct rxrpc_net *);

static inline bool rxrpc_conn_is_client(const struct rxrpc_connection *conn)
{
        return conn->out_clientflag;
}

static inline bool rxrpc_conn_is_service(const struct rxrpc_connection *conn)
{
        return !rxrpc_conn_is_client(conn);
}

static inline void rxrpc_reduce_conn_timer(struct rxrpc_connection *conn,
                                           unsigned long expire_at)
{
        timer_reduce(&conn->timer, expire_at);
}

/*
 * conn_service.c
 */
struct rxrpc_connection *rxrpc_find_service_conn_rcu(struct rxrpc_peer *,
                                                     struct sk_buff *);
struct rxrpc_connection *rxrpc_prealloc_service_connection(struct rxrpc_net *, gfp_t);
void rxrpc_new_incoming_connection(struct rxrpc_sock *, struct rxrpc_connection *,
                                   const struct rxrpc_security *, struct sk_buff *);
void rxrpc_unpublish_service_conn(struct rxrpc_connection *);

/*
 * input.c
 */
void rxrpc_congestion_degrade(struct rxrpc_call *);
void rxrpc_input_call_packet(struct rxrpc_call *, struct sk_buff *);
void rxrpc_implicit_end_call(struct rxrpc_call *, struct sk_buff *);

/*
 * io_thread.c
 */
int rxrpc_encap_rcv(struct sock *, struct sk_buff *);
void rxrpc_error_report(struct sock *);
bool rxrpc_direct_abort(struct sk_buff *skb, enum rxrpc_abort_reason why,
                        s32 abort_code, int err);
int rxrpc_io_thread(void *data);
static inline void rxrpc_wake_up_io_thread(struct rxrpc_local *local)
{
        wake_up_process(READ_ONCE(local->io_thread));
}

static inline bool rxrpc_protocol_error(struct sk_buff *skb, enum rxrpc_abort_reason why)
{
        return rxrpc_direct_abort(skb, why, RX_PROTOCOL_ERROR, -EPROTO);
}

/*
 * insecure.c
 */
extern const struct rxrpc_security rxrpc_no_security;

/*
 * key.c
 */
extern struct key_type key_type_rxrpc;

int rxrpc_request_key(struct rxrpc_sock *, sockptr_t , int);
int rxrpc_get_server_data_key(struct rxrpc_connection *, const void *, time64_t,
                              u32);

/*
 * local_event.c
 */
void rxrpc_gen_version_string(void);
void rxrpc_send_version_request(struct rxrpc_local *local,
                                struct rxrpc_host_header *hdr,
                                struct sk_buff *skb);

/*
 * local_object.c
 */
void rxrpc_local_dont_fragment(const struct rxrpc_local *local, bool set);
struct rxrpc_local *rxrpc_lookup_local(struct net *, const struct sockaddr_rxrpc *);
struct rxrpc_local *rxrpc_get_local(struct rxrpc_local *, enum rxrpc_local_trace);
struct rxrpc_local *rxrpc_get_local_maybe(struct rxrpc_local *, enum rxrpc_local_trace);
void rxrpc_put_local(struct rxrpc_local *, enum rxrpc_local_trace);
struct rxrpc_local *rxrpc_use_local(struct rxrpc_local *, enum rxrpc_local_trace);
void rxrpc_unuse_local(struct rxrpc_local *, enum rxrpc_local_trace);
void rxrpc_destroy_local(struct rxrpc_local *local);
void rxrpc_destroy_all_locals(struct rxrpc_net *);

static inline bool __rxrpc_use_local(struct rxrpc_local *local,
                                     enum rxrpc_local_trace why)
{
        int r, u;

        r = refcount_read(&local->ref);
        u = atomic_fetch_add_unless(&local->active_users, 1, 0);
        trace_rxrpc_local(local->debug_id, why, r, u);
        return u != 0;
}

static inline void rxrpc_see_local(struct rxrpc_local *local,
                                   enum rxrpc_local_trace why)
{
        int r, u;

        r = refcount_read(&local->ref);
        u = atomic_read(&local->active_users);
        trace_rxrpc_local(local->debug_id, why, r, u);
}

/*
 * misc.c
 */
extern unsigned int rxrpc_max_backlog __read_mostly;
extern unsigned long rxrpc_soft_ack_delay;
extern unsigned long rxrpc_idle_ack_delay;
extern unsigned int rxrpc_rx_window_size;
extern unsigned int rxrpc_rx_mtu;
extern unsigned int rxrpc_rx_jumbo_max;
#ifdef CONFIG_AF_RXRPC_INJECT_RX_DELAY
extern unsigned long rxrpc_inject_rx_delay;
#endif

/*
 * net_ns.c
 */
extern unsigned int rxrpc_net_id;
extern struct pernet_operations rxrpc_net_ops;

static inline struct rxrpc_net *rxrpc_net(struct net *net)
{
        return net_generic(net, rxrpc_net_id);
}

/*
 * output.c
 */
void rxrpc_send_ACK(struct rxrpc_call *call, u8 ack_reason,
                    rxrpc_serial_t serial, enum rxrpc_propose_ack_trace why);
int rxrpc_send_abort_packet(struct rxrpc_call *);
void rxrpc_send_conn_abort(struct rxrpc_connection *conn);
void rxrpc_reject_packet(struct rxrpc_local *local, struct sk_buff *skb);
void rxrpc_send_keepalive(struct rxrpc_peer *);
void rxrpc_transmit_one(struct rxrpc_call *call, struct rxrpc_txbuf *txb);

/*
 * peer_event.c
 */
void rxrpc_input_error(struct rxrpc_local *, struct sk_buff *);
void rxrpc_peer_keepalive_worker(struct work_struct *);

/*
 * peer_object.c
 */
struct rxrpc_peer *rxrpc_lookup_peer_rcu(struct rxrpc_local *,
                                         const struct sockaddr_rxrpc *);
struct rxrpc_peer *rxrpc_lookup_peer(struct rxrpc_local *local,
                                     struct sockaddr_rxrpc *srx, gfp_t gfp);
struct rxrpc_peer *rxrpc_alloc_peer(struct rxrpc_local *, gfp_t,
                                    enum rxrpc_peer_trace);
void rxrpc_new_incoming_peer(struct rxrpc_local *local, struct rxrpc_peer *peer);
void rxrpc_destroy_all_peers(struct rxrpc_net *);
struct rxrpc_peer *rxrpc_get_peer(struct rxrpc_peer *, enum rxrpc_peer_trace);
struct rxrpc_peer *rxrpc_get_peer_maybe(struct rxrpc_peer *, enum rxrpc_peer_trace);
void rxrpc_put_peer(struct rxrpc_peer *, enum rxrpc_peer_trace);

/*
 * proc.c
 */
extern const struct seq_operations rxrpc_call_seq_ops;
extern const struct seq_operations rxrpc_connection_seq_ops;
extern const struct seq_operations rxrpc_bundle_seq_ops;
extern const struct seq_operations rxrpc_peer_seq_ops;
extern const struct seq_operations rxrpc_local_seq_ops;

/*
 * recvmsg.c
 */
void rxrpc_notify_socket(struct rxrpc_call *);
int rxrpc_recvmsg(struct socket *, struct msghdr *, size_t, int);

/*
 * Abort a call due to a protocol error.
 */
static inline int rxrpc_abort_eproto(struct rxrpc_call *call,
                                     struct sk_buff *skb,
                                     s32 abort_code,
                                     enum rxrpc_abort_reason why)
{
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

        rxrpc_abort_call(call, sp->hdr.seq, abort_code, -EPROTO, why);
        return -EPROTO;
}

/*
 * rtt.c
 */
void rxrpc_peer_add_rtt(struct rxrpc_call *, enum rxrpc_rtt_rx_trace, int,
                        rxrpc_serial_t, rxrpc_serial_t, ktime_t, ktime_t);
ktime_t rxrpc_get_rto_backoff(struct rxrpc_peer *peer, bool retrans);
void rxrpc_peer_init_rtt(struct rxrpc_peer *);

/*
 * rxkad.c
 */
#ifdef CONFIG_RXKAD
extern const struct rxrpc_security rxkad;
#endif

/*
 * security.c
 */
int __init rxrpc_init_security(void);
const struct rxrpc_security *rxrpc_security_lookup(u8);
void rxrpc_exit_security(void);
int rxrpc_init_client_call_security(struct rxrpc_call *);
int rxrpc_init_client_conn_security(struct rxrpc_connection *);
const struct rxrpc_security *rxrpc_get_incoming_security(struct rxrpc_sock *,
                                                         struct sk_buff *);
struct key *rxrpc_look_up_server_security(struct rxrpc_connection *,
                                          struct sk_buff *, u32, u32);

/*
 * sendmsg.c
 */
bool rxrpc_propose_abort(struct rxrpc_call *call, s32 abort_code, int error,
                         enum rxrpc_abort_reason why);
int rxrpc_do_sendmsg(struct rxrpc_sock *, struct msghdr *, size_t);

/*
 * server_key.c
 */
extern struct key_type key_type_rxrpc_s;

int rxrpc_server_keyring(struct rxrpc_sock *, sockptr_t, int);

/*
 * skbuff.c
 */
void rxrpc_kernel_data_consumed(struct rxrpc_call *, struct sk_buff *);
void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace);
void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace);
void rxrpc_eaten_skb(struct sk_buff *, enum rxrpc_skb_trace);
void rxrpc_get_skb(struct sk_buff *, enum rxrpc_skb_trace);
void rxrpc_free_skb(struct sk_buff *, enum rxrpc_skb_trace);
void rxrpc_purge_queue(struct sk_buff_head *);

/*
 * stats.c
 */
int rxrpc_stats_show(struct seq_file *seq, void *v);
int rxrpc_stats_clear(struct file *file, char *buf, size_t size);

#define rxrpc_inc_stat(rxnet, s) atomic_inc(&(rxnet)->s)
#define rxrpc_dec_stat(rxnet, s) atomic_dec(&(rxnet)->s)

/*
 * sysctl.c
 */
#ifdef CONFIG_SYSCTL
extern int __init rxrpc_sysctl_init(void);
extern void rxrpc_sysctl_exit(void);
#else
static inline int __init rxrpc_sysctl_init(void) { return 0; }
static inline void rxrpc_sysctl_exit(void) {}
#endif

/*
 * txbuf.c
 */
extern atomic_t rxrpc_nr_txbuf;
struct rxrpc_txbuf *rxrpc_alloc_data_txbuf(struct rxrpc_call *call, size_t data_size,
                                           size_t data_align, gfp_t gfp);
struct rxrpc_txbuf *rxrpc_alloc_ack_txbuf(struct rxrpc_call *call, size_t sack_size);
void rxrpc_get_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what);
void rxrpc_see_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what);
void rxrpc_put_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what);

/*
 * utils.c
 */
int rxrpc_extract_addr_from_skb(struct sockaddr_rxrpc *, struct sk_buff *);

static inline bool before(u32 seq1, u32 seq2)
{
        return (s32)(seq1 - seq2) < 0;
}
static inline bool before_eq(u32 seq1, u32 seq2)
{
        return (s32)(seq1 - seq2) <= 0;
}
static inline bool after(u32 seq1, u32 seq2)
{
        return (s32)(seq1 - seq2) > 0;
}
static inline bool after_eq(u32 seq1, u32 seq2)
{
        return (s32)(seq1 - seq2) >= 0;
}

/*
 * debug tracing
 */
extern unsigned int rxrpc_debug;

#define dbgprintk(FMT,...) \
        printk("[%-6.6s] "FMT"\n", current->comm ,##__VA_ARGS__)

#define kenter(FMT,...)        dbgprintk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define kleave(FMT,...)        dbgprintk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define kdebug(FMT,...)        dbgprintk("    "FMT ,##__VA_ARGS__)


#if defined(__KDEBUG)
#define _enter(FMT,...)        kenter(FMT,##__VA_ARGS__)
#define _leave(FMT,...)        kleave(FMT,##__VA_ARGS__)
#define _debug(FMT,...)        kdebug(FMT,##__VA_ARGS__)

#elif defined(CONFIG_AF_RXRPC_DEBUG)
#define RXRPC_DEBUG_KENTER        0x01
#define RXRPC_DEBUG_KLEAVE        0x02
#define RXRPC_DEBUG_KDEBUG        0x04

#define _enter(FMT,...)                                        \
do {                                                        \
        if (unlikely(rxrpc_debug & RXRPC_DEBUG_KENTER))        \
                kenter(FMT,##__VA_ARGS__);                \
} while (0)

#define _leave(FMT,...)                                        \
do {                                                        \
        if (unlikely(rxrpc_debug & RXRPC_DEBUG_KLEAVE))        \
                kleave(FMT,##__VA_ARGS__);                \
} while (0)

#define _debug(FMT,...)                                        \
do {                                                        \
        if (unlikely(rxrpc_debug & RXRPC_DEBUG_KDEBUG))        \
                kdebug(FMT,##__VA_ARGS__);                \
} while (0)

#else
#define _enter(FMT,...)        no_printk("==> %s("FMT")",__func__ ,##__VA_ARGS__)
#define _leave(FMT,...)        no_printk("<== %s()"FMT"",__func__ ,##__VA_ARGS__)
#define _debug(FMT,...)        no_printk("    "FMT ,##__VA_ARGS__)
#endif

/*
 * debug assertion checking
 */
#if 1 // defined(__KDEBUGALL)

#define ASSERT(X)                                                \
do {                                                                \
        if (unlikely(!(X))) {                                        \
                pr_err("Assertion failed\n");                        \
                BUG();                                                \
        }                                                        \
} while (0)

#define ASSERTCMP(X, OP, Y)                                                \
do {                                                                        \
        __typeof__(X) _x = (X);                                                \
        __typeof__(Y) _y = (__typeof__(X))(Y);                                \
        if (unlikely(!(_x OP _y))) {                                        \
                pr_err("Assertion failed - %lu(0x%lx) %s %lu(0x%lx) is false\n", \
                       (unsigned long)_x, (unsigned long)_x, #OP,        \
                       (unsigned long)_y, (unsigned long)_y);                \
                BUG();                                                        \
        }                                                                \
} while (0)

#define ASSERTIF(C, X)                                                \
do {                                                                \
        if (unlikely((C) && !(X))) {                                \
                pr_err("Assertion failed\n");                        \
                BUG();                                                \
        }                                                        \
} while (0)

#define ASSERTIFCMP(C, X, OP, Y)                                        \
do {                                                                        \
        __typeof__(X) _x = (X);                                                \
        __typeof__(Y) _y = (__typeof__(X))(Y);                                \
        if (unlikely((C) && !(_x OP _y))) {                                \
                pr_err("Assertion failed - %lu(0x%lx) %s %lu(0x%lx) is false\n", \
                       (unsigned long)_x, (unsigned long)_x, #OP,        \
                       (unsigned long)_y, (unsigned long)_y);                \
                BUG();                                                        \
        }                                                                \
} while (0)

#else

#define ASSERT(X)                                \
do {                                                \
} while (0)

#define ASSERTCMP(X, OP, Y)                        \
do {                                                \
} while (0)

#define ASSERTIF(C, X)                                \
do {                                                \
} while (0)

#define ASSERTIFCMP(C, X, OP, Y)                \
do {                                                \
} while (0)

#endif /* __KDEBUGALL */
















































   17 







   17 



   17 











   23 




   26 
   26 


   25 
   22 



   21 
    1 
    1 


   20 





   19 


   18 


   18 



   17 






   17 


























































































































































































































































































































































































































































































































































    2 




    1 










    2 









    2 




    1 
    2 





    2 
    2 

    2 



    2 
































   11 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2016 Facebook
 */
#include <linux/bpf.h>
#include <linux/jhash.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <linux/stacktrace.h>
#include <linux/perf_event.h>
#include <linux/btf_ids.h>
#include <linux/buildid.h>
#include "percpu_freelist.h"
#include "mmap_unlock_work.h"

#define STACK_CREATE_FLAG_MASK                                        \
        (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY |        \
         BPF_F_STACK_BUILD_ID)

struct stack_map_bucket {
        struct pcpu_freelist_node fnode;
        u32 hash;
        u32 nr;
        u64 data[];
};

struct bpf_stack_map {
        struct bpf_map map;
        void *elems;
        struct pcpu_freelist freelist;
        u32 n_buckets;
        struct stack_map_bucket *buckets[] __counted_by(n_buckets);
};

static inline bool stack_map_use_build_id(struct bpf_map *map)
{
        return (map->map_flags & BPF_F_STACK_BUILD_ID);
}

static inline int stack_map_data_size(struct bpf_map *map)
{
        return stack_map_use_build_id(map) ?
                sizeof(struct bpf_stack_build_id) : sizeof(u64);
}

static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
{
        u64 elem_size = sizeof(struct stack_map_bucket) +
                        (u64)smap->map.value_size;
        int err;

        smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
                                         smap->map.numa_node);
        if (!smap->elems)
                return -ENOMEM;

        err = pcpu_freelist_init(&smap->freelist);
        if (err)
                goto free_elems;

        pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
                               smap->map.max_entries);
        return 0;

free_elems:
        bpf_map_area_free(smap->elems);
        return err;
}

/* Called from syscall */
static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
{
        u32 value_size = attr->value_size;
        struct bpf_stack_map *smap;
        u64 cost, n_buckets;
        int err;

        if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
                return ERR_PTR(-EINVAL);

        /* check sanity of attributes */
        if (attr->max_entries == 0 || attr->key_size != 4 ||
            value_size < 8 || value_size % 8)
                return ERR_PTR(-EINVAL);

        BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
        if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
                if (value_size % sizeof(struct bpf_stack_build_id) ||
                    value_size / sizeof(struct bpf_stack_build_id)
                    > sysctl_perf_event_max_stack)
                        return ERR_PTR(-EINVAL);
        } else if (value_size / 8 > sysctl_perf_event_max_stack)
                return ERR_PTR(-EINVAL);

        /* hash table size must be power of 2; roundup_pow_of_two() can overflow
         * into UB on 32-bit arches, so check that first
         */
        if (attr->max_entries > 1UL << 31)
                return ERR_PTR(-E2BIG);

        n_buckets = roundup_pow_of_two(attr->max_entries);

        cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
        smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
        if (!smap)
                return ERR_PTR(-ENOMEM);

        bpf_map_init_from_attr(&smap->map, attr);
        smap->n_buckets = n_buckets;

        err = get_callchain_buffers(sysctl_perf_event_max_stack);
        if (err)
                goto free_smap;

        err = prealloc_elems_and_freelist(smap);
        if (err)
                goto put_buffers;

        return &smap->map;

put_buffers:
        put_callchain_buffers();
free_smap:
        bpf_map_area_free(smap);
        return ERR_PTR(err);
}

static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, bool may_fault)
{
        return may_fault ? build_id_parse(vma, build_id, NULL)
                         : build_id_parse_nofault(vma, build_id, NULL);
}

/*
 * Expects all id_offs[i].ip values to be set to correct initial IPs.
 * They will be subsequently:
 *   - either adjusted in place to a file offset, if build ID fetching
 *     succeeds; in this case id_offs[i].build_id is set to correct build ID,
 *     and id_offs[i].status is set to BPF_STACK_BUILD_ID_VALID;
 *   - or IP will be kept intact, if build ID fetching failed; in this case
 *     id_offs[i].build_id is zeroed out and id_offs[i].status is set to
 *     BPF_STACK_BUILD_ID_IP.
 */
static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
                                          u32 trace_nr, bool user, bool may_fault)
{
        int i;
        struct mmap_unlock_irq_work *work = NULL;
        bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
        struct vm_area_struct *vma, *prev_vma = NULL;
        const char *prev_build_id;

        /* If the irq_work is in use, fall back to report ips. Same
         * fallback is used for kernel stack (!user) on a stackmap with
         * build_id.
         */
        if (!user || !current || !current->mm || irq_work_busy ||
            !mmap_read_trylock(current->mm)) {
                /* cannot access current->mm, fall back to ips */
                for (i = 0; i < trace_nr; i++) {
                        id_offs[i].status = BPF_STACK_BUILD_ID_IP;
                        memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
                }
                return;
        }

        for (i = 0; i < trace_nr; i++) {
                u64 ip = READ_ONCE(id_offs[i].ip);

                if (range_in_vma(prev_vma, ip, ip)) {
                        vma = prev_vma;
                        memcpy(id_offs[i].build_id, prev_build_id, BUILD_ID_SIZE_MAX);
                        goto build_id_valid;
                }
                vma = find_vma(current->mm, ip);
                if (!vma || fetch_build_id(vma, id_offs[i].build_id, may_fault)) {
                        /* per entry fall back to ips */
                        id_offs[i].status = BPF_STACK_BUILD_ID_IP;
                        memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
                        continue;
                }
build_id_valid:
                id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ip - vma->vm_start;
                id_offs[i].status = BPF_STACK_BUILD_ID_VALID;
                prev_vma = vma;
                prev_build_id = id_offs[i].build_id;
        }
        bpf_mmap_unlock_mm(work, current->mm);
}

static struct perf_callchain_entry *
get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
{
#ifdef CONFIG_STACKTRACE
        struct perf_callchain_entry *entry;
        int rctx;

        entry = get_callchain_entry(&rctx);

        if (!entry)
                return NULL;

        entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
                                         max_depth, 0);

        /* stack_trace_save_tsk() works on unsigned long array, while
         * perf_callchain_entry uses u64 array. For 32-bit systems, it is
         * necessary to fix this mismatch.
         */
        if (__BITS_PER_LONG != 64) {
                unsigned long *from = (unsigned long *) entry->ip;
                u64 *to = entry->ip;
                int i;

                /* copy data from the end to avoid using extra buffer */
                for (i = entry->nr - 1; i >= 0; i--)
                        to[i] = (u64)(from[i]);
        }

        put_callchain_entry(rctx);

        return entry;
#else /* CONFIG_STACKTRACE */
        return NULL;
#endif
}

static long __bpf_get_stackid(struct bpf_map *map,
                              struct perf_callchain_entry *trace, u64 flags)
{
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
        struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
        u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        u32 hash, id, trace_nr, trace_len, i;
        bool user = flags & BPF_F_USER_STACK;
        u64 *ips;
        bool hash_matches;

        if (trace->nr <= skip)
                /* skipping more than usable stack trace */
                return -EFAULT;

        trace_nr = trace->nr - skip;
        trace_len = trace_nr * sizeof(u64);
        ips = trace->ip + skip;
        hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
        id = hash & (smap->n_buckets - 1);
        bucket = READ_ONCE(smap->buckets[id]);

        hash_matches = bucket && bucket->hash == hash;
        /* fast cmp */
        if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
                return id;

        if (stack_map_use_build_id(map)) {
                struct bpf_stack_build_id *id_offs;

                /* for build_id+offset, pop a bucket before slow cmp */
                new_bucket = (struct stack_map_bucket *)
                        pcpu_freelist_pop(&smap->freelist);
                if (unlikely(!new_bucket))
                        return -ENOMEM;
                new_bucket->nr = trace_nr;
                id_offs = (struct bpf_stack_build_id *)new_bucket->data;
                for (i = 0; i < trace_nr; i++)
                        id_offs[i].ip = ips[i];
                stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */);
                trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
                if (hash_matches && bucket->nr == trace_nr &&
                    memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
                        pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
                        return id;
                }
                if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
                        pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
                        return -EEXIST;
                }
        } else {
                if (hash_matches && bucket->nr == trace_nr &&
                    memcmp(bucket->data, ips, trace_len) == 0)
                        return id;
                if (bucket && !(flags & BPF_F_REUSE_STACKID))
                        return -EEXIST;

                new_bucket = (struct stack_map_bucket *)
                        pcpu_freelist_pop(&smap->freelist);
                if (unlikely(!new_bucket))
                        return -ENOMEM;
                memcpy(new_bucket->data, ips, trace_len);
        }

        new_bucket->hash = hash;
        new_bucket->nr = trace_nr;

        old_bucket = xchg(&smap->buckets[id], new_bucket);
        if (old_bucket)
                pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
        return id;
}

BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
           u64, flags)
{
        u32 max_depth = map->value_size / stack_map_data_size(map);
        u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        bool user = flags & BPF_F_USER_STACK;
        struct perf_callchain_entry *trace;
        bool kernel = !user;

        if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
                               BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
                return -EINVAL;

        max_depth += skip;
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;

        trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
                                   false, false);

        if (unlikely(!trace))
                /* couldn't fetch the stack trace */
                return -EFAULT;

        return __bpf_get_stackid(map, trace, flags);
}

const struct bpf_func_proto bpf_get_stackid_proto = {
        .func                = bpf_get_stackid,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
};

static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
{
        __u64 nr_kernel = 0;

        while (nr_kernel < trace->nr) {
                if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
                        break;
                nr_kernel++;
        }
        return nr_kernel;
}

BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
           struct bpf_map *, map, u64, flags)
{
        struct perf_event *event = ctx->event;
        struct perf_callchain_entry *trace;
        bool kernel, user;
        __u64 nr_kernel;
        int ret;

        /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
        if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
                return bpf_get_stackid((unsigned long)(ctx->regs),
                                       (unsigned long) map, flags, 0, 0);

        if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
                               BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
                return -EINVAL;

        user = flags & BPF_F_USER_STACK;
        kernel = !user;

        trace = ctx->data->callchain;
        if (unlikely(!trace))
                return -EFAULT;

        nr_kernel = count_kernel_ip(trace);

        if (kernel) {
                __u64 nr = trace->nr;

                trace->nr = nr_kernel;
                ret = __bpf_get_stackid(map, trace, flags);

                /* restore nr */
                trace->nr = nr;
        } else { /* user */
                u64 skip = flags & BPF_F_SKIP_FIELD_MASK;

                skip += nr_kernel;
                if (skip > BPF_F_SKIP_FIELD_MASK)
                        return -EFAULT;

                flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
                ret = __bpf_get_stackid(map, trace, flags);
        }
        return ret;
}

const struct bpf_func_proto bpf_get_stackid_proto_pe = {
        .func                = bpf_get_stackid_pe,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
};

static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                            struct perf_callchain_entry *trace_in,
                            void *buf, u32 size, u64 flags, bool may_fault)
{
        u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
        bool user_build_id = flags & BPF_F_USER_BUILD_ID;
        bool crosstask = task && task != current;
        u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        bool user = flags & BPF_F_USER_STACK;
        struct perf_callchain_entry *trace;
        bool kernel = !user;
        int err = -EINVAL;
        u64 *ips;

        if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
                               BPF_F_USER_BUILD_ID)))
                goto clear;
        if (kernel && user_build_id)
                goto clear;

        elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
        if (unlikely(size % elem_size))
                goto clear;

        /* cannot get valid user stack for task without user_mode regs */
        if (task && user && !user_mode(regs))
                goto err_fault;

        /* get_perf_callchain does not support crosstask user stack walking
         * but returns an empty stack instead of NULL.
         */
        if (crosstask && user) {
                err = -EOPNOTSUPP;
                goto clear;
        }

        num_elem = size / elem_size;
        max_depth = num_elem + skip;
        if (sysctl_perf_event_max_stack < max_depth)
                max_depth = sysctl_perf_event_max_stack;

        if (may_fault)
                rcu_read_lock(); /* need RCU for perf's callchain below */

        if (trace_in)
                trace = trace_in;
        else if (kernel && task)
                trace = get_callchain_entry_for_task(task, max_depth);
        else
                trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
                                           crosstask, false);

        if (unlikely(!trace) || trace->nr < skip) {
                if (may_fault)
                        rcu_read_unlock();
                goto err_fault;
        }

        trace_nr = trace->nr - skip;
        trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
        copy_len = trace_nr * elem_size;

        ips = trace->ip + skip;
        if (user_build_id) {
                struct bpf_stack_build_id *id_offs = buf;
                u32 i;

                for (i = 0; i < trace_nr; i++)
                        id_offs[i].ip = ips[i];
        } else {
                memcpy(buf, ips, copy_len);
        }

        /* trace/ips should not be dereferenced after this point */
        if (may_fault)
                rcu_read_unlock();

        if (user_build_id)
                stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);

        if (size > copy_len)
                memset(buf + copy_len, 0, size - copy_len);
        return copy_len;

err_fault:
        err = -EFAULT;
clear:
        memset(buf, 0, size);
        return err;
}

BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
           u64, flags)
{
        return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
}

const struct bpf_func_proto bpf_get_stack_proto = {
        .func                = bpf_get_stack,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size,
           u64, flags)
{
        return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */);
}

const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
        .func                = bpf_get_stack_sleepable,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
                                 u64 flags, bool may_fault)
{
        struct pt_regs *regs;
        long res = -EINVAL;

        if (!try_get_task_stack(task))
                return -EFAULT;

        regs = task_pt_regs(task);
        if (regs)
                res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault);
        put_task_stack(task);

        return res;
}

BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
           u32, size, u64, flags)
{
        return __bpf_get_task_stack(task, buf, size, flags, false /* !may_fault */);
}

const struct bpf_func_proto bpf_get_task_stack_proto = {
        .func                = bpf_get_task_stack,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf,
           u32, size, u64, flags)
{
        return __bpf_get_task_stack(task, buf, size, flags, true /* !may_fault */);
}

const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
        .func                = bpf_get_task_stack_sleepable,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
           void *, buf, u32, size, u64, flags)
{
        struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
        struct perf_event *event = ctx->event;
        struct perf_callchain_entry *trace;
        bool kernel, user;
        int err = -EINVAL;
        __u64 nr_kernel;

        if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
                return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);

        if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
                               BPF_F_USER_BUILD_ID)))
                goto clear;

        user = flags & BPF_F_USER_STACK;
        kernel = !user;

        err = -EFAULT;
        trace = ctx->data->callchain;
        if (unlikely(!trace))
                goto clear;

        nr_kernel = count_kernel_ip(trace);

        if (kernel) {
                __u64 nr = trace->nr;

                trace->nr = nr_kernel;
                err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);

                /* restore nr */
                trace->nr = nr;
        } else { /* user */
                u64 skip = flags & BPF_F_SKIP_FIELD_MASK;

                skip += nr_kernel;
                if (skip > BPF_F_SKIP_FIELD_MASK)
                        goto clear;

                flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
                err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
        }
        return err;

clear:
        memset(buf, 0, size);
        return err;

}

const struct bpf_func_proto bpf_get_stack_proto_pe = {
        .func                = bpf_get_stack_pe,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

/* Called from eBPF program */
static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
{
        return ERR_PTR(-EOPNOTSUPP);
}

/* Called from syscall */
int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
{
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
        struct stack_map_bucket *bucket, *old_bucket;
        u32 id = *(u32 *)key, trace_len;

        if (unlikely(id >= smap->n_buckets))
                return -ENOENT;

        bucket = xchg(&smap->buckets[id], NULL);
        if (!bucket)
                return -ENOENT;

        trace_len = bucket->nr * stack_map_data_size(map);
        memcpy(value, bucket->data, trace_len);
        memset(value + trace_len, 0, map->value_size - trace_len);

        old_bucket = xchg(&smap->buckets[id], bucket);
        if (old_bucket)
                pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
        return 0;
}

static int stack_map_get_next_key(struct bpf_map *map, void *key,
                                  void *next_key)
{
        struct bpf_stack_map *smap = container_of(map,
                                                  struct bpf_stack_map, map);
        u32 id;

        WARN_ON_ONCE(!rcu_read_lock_held());

        if (!key) {
                id = 0;
        } else {
                id = *(u32 *)key;
                if (id >= smap->n_buckets || !smap->buckets[id])
                        id = 0;
                else
                        id++;
        }

        while (id < smap->n_buckets && !smap->buckets[id])
                id++;

        if (id >= smap->n_buckets)
                return -ENOENT;

        *(u32 *)next_key = id;
        return 0;
}

static long stack_map_update_elem(struct bpf_map *map, void *key, void *value,
                                  u64 map_flags)
{
        return -EINVAL;
}

/* Called from syscall or from eBPF program */
static long stack_map_delete_elem(struct bpf_map *map, void *key)
{
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
        struct stack_map_bucket *old_bucket;
        u32 id = *(u32 *)key;

        if (unlikely(id >= smap->n_buckets))
                return -E2BIG;

        old_bucket = xchg(&smap->buckets[id], NULL);
        if (old_bucket) {
                pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
                return 0;
        } else {
                return -ENOENT;
        }
}

/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
static void stack_map_free(struct bpf_map *map)
{
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);

        bpf_map_area_free(smap->elems);
        pcpu_freelist_destroy(&smap->freelist);
        bpf_map_area_free(smap);
        put_callchain_buffers();
}

static u64 stack_map_mem_usage(const struct bpf_map *map)
{
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
        u64 value_size = map->value_size;
        u64 n_buckets = smap->n_buckets;
        u64 enties = map->max_entries;
        u64 usage = sizeof(*smap);

        usage += n_buckets * sizeof(struct stack_map_bucket *);
        usage += enties * (sizeof(struct stack_map_bucket) + value_size);
        return usage;
}

BTF_ID_LIST_SINGLE(stack_trace_map_btf_ids, struct, bpf_stack_map)
const struct bpf_map_ops stack_trace_map_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = stack_map_alloc,
        .map_free = stack_map_free,
        .map_get_next_key = stack_map_get_next_key,
        .map_lookup_elem = stack_map_lookup_elem,
        .map_update_elem = stack_map_update_elem,
        .map_delete_elem = stack_map_delete_elem,
        .map_check_btf = map_check_no_btf,
        .map_mem_usage = stack_map_mem_usage,
        .map_btf_id = &stack_trace_map_btf_ids[0],
};









































































































































































































































































































































































































































































  362 













































   18 




















   16 













































































































































































































































































































































































































































































































































































































































































































    5 































































































































































































































































































































































































































































































 1193 


































































































































































































































































































































































































































































































































































































































































































































































 2555 

 1848 
























 2560 




















  163 























  358 

  357 

























   10 

    9 






























































































































































    6 





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  122 








































































    1 

































































































































































































































  188 

  189 
























  190 




















  152 















































































  121 


  122 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
// SPDX-License-Identifier: GPL-2.0

// Generated by scripts/atomic/gen-atomic-fallback.sh
// DO NOT MODIFY THIS FILE DIRECTLY

#ifndef _LINUX_ATOMIC_FALLBACK_H
#define _LINUX_ATOMIC_FALLBACK_H

#include <linux/compiler.h>

#if defined(arch_xchg)
#define raw_xchg arch_xchg
#elif defined(arch_xchg_relaxed)
#define raw_xchg(...) \
        __atomic_op_fence(arch_xchg, __VA_ARGS__)
#else
extern void raw_xchg_not_implemented(void);
#define raw_xchg(...) raw_xchg_not_implemented()
#endif

#if defined(arch_xchg_acquire)
#define raw_xchg_acquire arch_xchg_acquire
#elif defined(arch_xchg_relaxed)
#define raw_xchg_acquire(...) \
        __atomic_op_acquire(arch_xchg, __VA_ARGS__)
#elif defined(arch_xchg)
#define raw_xchg_acquire arch_xchg
#else
extern void raw_xchg_acquire_not_implemented(void);
#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
#endif

#if defined(arch_xchg_release)
#define raw_xchg_release arch_xchg_release
#elif defined(arch_xchg_relaxed)
#define raw_xchg_release(...) \
        __atomic_op_release(arch_xchg, __VA_ARGS__)
#elif defined(arch_xchg)
#define raw_xchg_release arch_xchg
#else
extern void raw_xchg_release_not_implemented(void);
#define raw_xchg_release(...) raw_xchg_release_not_implemented()
#endif

#if defined(arch_xchg_relaxed)
#define raw_xchg_relaxed arch_xchg_relaxed
#elif defined(arch_xchg)
#define raw_xchg_relaxed arch_xchg
#else
extern void raw_xchg_relaxed_not_implemented(void);
#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
#endif

#if defined(arch_cmpxchg)
#define raw_cmpxchg arch_cmpxchg
#elif defined(arch_cmpxchg_relaxed)
#define raw_cmpxchg(...) \
        __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
#else
extern void raw_cmpxchg_not_implemented(void);
#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
#endif

#if defined(arch_cmpxchg_acquire)
#define raw_cmpxchg_acquire arch_cmpxchg_acquire
#elif defined(arch_cmpxchg_relaxed)
#define raw_cmpxchg_acquire(...) \
        __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
#elif defined(arch_cmpxchg)
#define raw_cmpxchg_acquire arch_cmpxchg
#else
extern void raw_cmpxchg_acquire_not_implemented(void);
#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
#endif

#if defined(arch_cmpxchg_release)
#define raw_cmpxchg_release arch_cmpxchg_release
#elif defined(arch_cmpxchg_relaxed)
#define raw_cmpxchg_release(...) \
        __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
#elif defined(arch_cmpxchg)
#define raw_cmpxchg_release arch_cmpxchg
#else
extern void raw_cmpxchg_release_not_implemented(void);
#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
#endif

#if defined(arch_cmpxchg_relaxed)
#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
#elif defined(arch_cmpxchg)
#define raw_cmpxchg_relaxed arch_cmpxchg
#else
extern void raw_cmpxchg_relaxed_not_implemented(void);
#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
#endif

#if defined(arch_cmpxchg64)
#define raw_cmpxchg64 arch_cmpxchg64
#elif defined(arch_cmpxchg64_relaxed)
#define raw_cmpxchg64(...) \
        __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
#else
extern void raw_cmpxchg64_not_implemented(void);
#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
#endif

#if defined(arch_cmpxchg64_acquire)
#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
#elif defined(arch_cmpxchg64_relaxed)
#define raw_cmpxchg64_acquire(...) \
        __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
#elif defined(arch_cmpxchg64)
#define raw_cmpxchg64_acquire arch_cmpxchg64
#else
extern void raw_cmpxchg64_acquire_not_implemented(void);
#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
#endif

#if defined(arch_cmpxchg64_release)
#define raw_cmpxchg64_release arch_cmpxchg64_release
#elif defined(arch_cmpxchg64_relaxed)
#define raw_cmpxchg64_release(...) \
        __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
#elif defined(arch_cmpxchg64)
#define raw_cmpxchg64_release arch_cmpxchg64
#else
extern void raw_cmpxchg64_release_not_implemented(void);
#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
#endif

#if defined(arch_cmpxchg64_relaxed)
#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
#elif defined(arch_cmpxchg64)
#define raw_cmpxchg64_relaxed arch_cmpxchg64
#else
extern void raw_cmpxchg64_relaxed_not_implemented(void);
#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
#endif

#if defined(arch_cmpxchg128)
#define raw_cmpxchg128 arch_cmpxchg128
#elif defined(arch_cmpxchg128_relaxed)
#define raw_cmpxchg128(...) \
        __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
#else
extern void raw_cmpxchg128_not_implemented(void);
#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
#endif

#if defined(arch_cmpxchg128_acquire)
#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
#elif defined(arch_cmpxchg128_relaxed)
#define raw_cmpxchg128_acquire(...) \
        __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
#elif defined(arch_cmpxchg128)
#define raw_cmpxchg128_acquire arch_cmpxchg128
#else
extern void raw_cmpxchg128_acquire_not_implemented(void);
#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
#endif

#if defined(arch_cmpxchg128_release)
#define raw_cmpxchg128_release arch_cmpxchg128_release
#elif defined(arch_cmpxchg128_relaxed)
#define raw_cmpxchg128_release(...) \
        __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
#elif defined(arch_cmpxchg128)
#define raw_cmpxchg128_release arch_cmpxchg128
#else
extern void raw_cmpxchg128_release_not_implemented(void);
#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
#endif

#if defined(arch_cmpxchg128_relaxed)
#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
#elif defined(arch_cmpxchg128)
#define raw_cmpxchg128_relaxed arch_cmpxchg128
#else
extern void raw_cmpxchg128_relaxed_not_implemented(void);
#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
#endif

#if defined(arch_try_cmpxchg)
#define raw_try_cmpxchg arch_try_cmpxchg
#elif defined(arch_try_cmpxchg_relaxed)
#define raw_try_cmpxchg(...) \
        __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
#else
#define raw_try_cmpxchg(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg_acquire)
#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
#elif defined(arch_try_cmpxchg_relaxed)
#define raw_try_cmpxchg_acquire(...) \
        __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
#elif defined(arch_try_cmpxchg)
#define raw_try_cmpxchg_acquire arch_try_cmpxchg
#else
#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg_release)
#define raw_try_cmpxchg_release arch_try_cmpxchg_release
#elif defined(arch_try_cmpxchg_relaxed)
#define raw_try_cmpxchg_release(...) \
        __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
#elif defined(arch_try_cmpxchg)
#define raw_try_cmpxchg_release arch_try_cmpxchg
#else
#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg_relaxed)
#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
#elif defined(arch_try_cmpxchg)
#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
#else
#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg64)
#define raw_try_cmpxchg64 arch_try_cmpxchg64
#elif defined(arch_try_cmpxchg64_relaxed)
#define raw_try_cmpxchg64(...) \
        __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
#else
#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg64_acquire)
#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
#elif defined(arch_try_cmpxchg64_relaxed)
#define raw_try_cmpxchg64_acquire(...) \
        __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
#elif defined(arch_try_cmpxchg64)
#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
#else
#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg64_release)
#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
#elif defined(arch_try_cmpxchg64_relaxed)
#define raw_try_cmpxchg64_release(...) \
        __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
#elif defined(arch_try_cmpxchg64)
#define raw_try_cmpxchg64_release arch_try_cmpxchg64
#else
#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg64_relaxed)
#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
#elif defined(arch_try_cmpxchg64)
#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
#else
#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg128)
#define raw_try_cmpxchg128 arch_try_cmpxchg128
#elif defined(arch_try_cmpxchg128_relaxed)
#define raw_try_cmpxchg128(...) \
        __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
#else
#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg128_acquire)
#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
#elif defined(arch_try_cmpxchg128_relaxed)
#define raw_try_cmpxchg128_acquire(...) \
        __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
#elif defined(arch_try_cmpxchg128)
#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
#else
#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg128_release)
#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
#elif defined(arch_try_cmpxchg128_relaxed)
#define raw_try_cmpxchg128_release(...) \
        __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
#elif defined(arch_try_cmpxchg128)
#define raw_try_cmpxchg128_release arch_try_cmpxchg128
#else
#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#if defined(arch_try_cmpxchg128_relaxed)
#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
#elif defined(arch_try_cmpxchg128)
#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
#else
#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#define raw_cmpxchg_local arch_cmpxchg_local

#ifdef arch_try_cmpxchg_local
#define raw_try_cmpxchg_local arch_try_cmpxchg_local
#else
#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#define raw_cmpxchg64_local arch_cmpxchg64_local

#ifdef arch_try_cmpxchg64_local
#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
#else
#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#define raw_cmpxchg128_local arch_cmpxchg128_local

#ifdef arch_try_cmpxchg128_local
#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
#else
#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

#define raw_sync_cmpxchg arch_sync_cmpxchg

#ifdef arch_sync_try_cmpxchg
#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
#else
#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
({ \
        typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
        ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
        if (unlikely(___r != ___o)) \
                *___op = ___r; \
        likely(___r == ___o); \
})
#endif

/**
 * raw_atomic_read() - atomic load with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically loads the value of @v with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
 *
 * Return: The value loaded from @v.
 */
static __always_inline int
raw_atomic_read(const atomic_t *v)
{
        return arch_atomic_read(v);
}

/**
 * raw_atomic_read_acquire() - atomic load with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically loads the value of @v with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
 *
 * Return: The value loaded from @v.
 */
static __always_inline int
raw_atomic_read_acquire(const atomic_t *v)
{
#if defined(arch_atomic_read_acquire)
        return arch_atomic_read_acquire(v);
#else
        int ret;

        if (__native_word(atomic_t)) {
                ret = smp_load_acquire(&(v)->counter);
        } else {
                ret = raw_atomic_read(v);
                __atomic_acquire_fence();
        }

        return ret;
#endif
}

/**
 * raw_atomic_set() - atomic set with relaxed ordering
 * @v: pointer to atomic_t
 * @i: int value to assign
 *
 * Atomically sets @v to @i with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_set(atomic_t *v, int i)
{
        arch_atomic_set(v, i);
}

/**
 * raw_atomic_set_release() - atomic set with release ordering
 * @v: pointer to atomic_t
 * @i: int value to assign
 *
 * Atomically sets @v to @i with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_set_release(atomic_t *v, int i)
{
#if defined(arch_atomic_set_release)
        arch_atomic_set_release(v, i);
#else
        if (__native_word(atomic_t)) {
                smp_store_release(&(v)->counter, i);
        } else {
                __atomic_release_fence();
                raw_atomic_set(v, i);
        }
#endif
}

/**
 * raw_atomic_add() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_add(int i, atomic_t *v)
{
        arch_atomic_add(i, v);
}

/**
 * raw_atomic_add_return() - atomic add with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_add_return(int i, atomic_t *v)
{
#if defined(arch_atomic_add_return)
        return arch_atomic_add_return(i, v);
#elif defined(arch_atomic_add_return_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_add_return_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_add_return"
#endif
}

/**
 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_add_return_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_add_return_acquire)
        return arch_atomic_add_return_acquire(i, v);
#elif defined(arch_atomic_add_return_relaxed)
        int ret = arch_atomic_add_return_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_add_return)
        return arch_atomic_add_return(i, v);
#else
#error "Unable to define raw_atomic_add_return_acquire"
#endif
}

/**
 * raw_atomic_add_return_release() - atomic add with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_add_return_release(int i, atomic_t *v)
{
#if defined(arch_atomic_add_return_release)
        return arch_atomic_add_return_release(i, v);
#elif defined(arch_atomic_add_return_relaxed)
        __atomic_release_fence();
        return arch_atomic_add_return_relaxed(i, v);
#elif defined(arch_atomic_add_return)
        return arch_atomic_add_return(i, v);
#else
#error "Unable to define raw_atomic_add_return_release"
#endif
}

/**
 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_add_return_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_add_return_relaxed)
        return arch_atomic_add_return_relaxed(i, v);
#elif defined(arch_atomic_add_return)
        return arch_atomic_add_return(i, v);
#else
#error "Unable to define raw_atomic_add_return_relaxed"
#endif
}

/**
 * raw_atomic_fetch_add() - atomic add with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_add(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_add)
        return arch_atomic_fetch_add(i, v);
#elif defined(arch_atomic_fetch_add_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_add_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_fetch_add"
#endif
}

/**
 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_add_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_add_acquire)
        return arch_atomic_fetch_add_acquire(i, v);
#elif defined(arch_atomic_fetch_add_relaxed)
        int ret = arch_atomic_fetch_add_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_add)
        return arch_atomic_fetch_add(i, v);
#else
#error "Unable to define raw_atomic_fetch_add_acquire"
#endif
}

/**
 * raw_atomic_fetch_add_release() - atomic add with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_add_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_add_release)
        return arch_atomic_fetch_add_release(i, v);
#elif defined(arch_atomic_fetch_add_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_add_relaxed(i, v);
#elif defined(arch_atomic_fetch_add)
        return arch_atomic_fetch_add(i, v);
#else
#error "Unable to define raw_atomic_fetch_add_release"
#endif
}

/**
 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_add_relaxed)
        return arch_atomic_fetch_add_relaxed(i, v);
#elif defined(arch_atomic_fetch_add)
        return arch_atomic_fetch_add(i, v);
#else
#error "Unable to define raw_atomic_fetch_add_relaxed"
#endif
}

/**
 * raw_atomic_sub() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_sub(int i, atomic_t *v)
{
        arch_atomic_sub(i, v);
}

/**
 * raw_atomic_sub_return() - atomic subtract with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_sub_return(int i, atomic_t *v)
{
#if defined(arch_atomic_sub_return)
        return arch_atomic_sub_return(i, v);
#elif defined(arch_atomic_sub_return_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_sub_return_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_sub_return"
#endif
}

/**
 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_sub_return_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_sub_return_acquire)
        return arch_atomic_sub_return_acquire(i, v);
#elif defined(arch_atomic_sub_return_relaxed)
        int ret = arch_atomic_sub_return_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_sub_return)
        return arch_atomic_sub_return(i, v);
#else
#error "Unable to define raw_atomic_sub_return_acquire"
#endif
}

/**
 * raw_atomic_sub_return_release() - atomic subtract with release ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_sub_return_release(int i, atomic_t *v)
{
#if defined(arch_atomic_sub_return_release)
        return arch_atomic_sub_return_release(i, v);
#elif defined(arch_atomic_sub_return_relaxed)
        __atomic_release_fence();
        return arch_atomic_sub_return_relaxed(i, v);
#elif defined(arch_atomic_sub_return)
        return arch_atomic_sub_return(i, v);
#else
#error "Unable to define raw_atomic_sub_return_release"
#endif
}

/**
 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_sub_return_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_sub_return_relaxed)
        return arch_atomic_sub_return_relaxed(i, v);
#elif defined(arch_atomic_sub_return)
        return arch_atomic_sub_return(i, v);
#else
#error "Unable to define raw_atomic_sub_return_relaxed"
#endif
}

/**
 * raw_atomic_fetch_sub() - atomic subtract with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_sub(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_sub)
        return arch_atomic_fetch_sub(i, v);
#elif defined(arch_atomic_fetch_sub_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_sub_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_fetch_sub"
#endif
}

/**
 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_sub_acquire)
        return arch_atomic_fetch_sub_acquire(i, v);
#elif defined(arch_atomic_fetch_sub_relaxed)
        int ret = arch_atomic_fetch_sub_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_sub)
        return arch_atomic_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic_fetch_sub_acquire"
#endif
}

/**
 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_sub_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_sub_release)
        return arch_atomic_fetch_sub_release(i, v);
#elif defined(arch_atomic_fetch_sub_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_sub_relaxed(i, v);
#elif defined(arch_atomic_fetch_sub)
        return arch_atomic_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic_fetch_sub_release"
#endif
}

/**
 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_sub_relaxed)
        return arch_atomic_fetch_sub_relaxed(i, v);
#elif defined(arch_atomic_fetch_sub)
        return arch_atomic_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic_fetch_sub_relaxed"
#endif
}

/**
 * raw_atomic_inc() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_inc(atomic_t *v)
{
#if defined(arch_atomic_inc)
        arch_atomic_inc(v);
#else
        raw_atomic_add(1, v);
#endif
}

/**
 * raw_atomic_inc_return() - atomic increment with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_inc_return(atomic_t *v)
{
#if defined(arch_atomic_inc_return)
        return arch_atomic_inc_return(v);
#elif defined(arch_atomic_inc_return_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_inc_return_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_add_return(1, v);
#endif
}

/**
 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_inc_return_acquire(atomic_t *v)
{
#if defined(arch_atomic_inc_return_acquire)
        return arch_atomic_inc_return_acquire(v);
#elif defined(arch_atomic_inc_return_relaxed)
        int ret = arch_atomic_inc_return_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_inc_return)
        return arch_atomic_inc_return(v);
#else
        return raw_atomic_add_return_acquire(1, v);
#endif
}

/**
 * raw_atomic_inc_return_release() - atomic increment with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_inc_return_release(atomic_t *v)
{
#if defined(arch_atomic_inc_return_release)
        return arch_atomic_inc_return_release(v);
#elif defined(arch_atomic_inc_return_relaxed)
        __atomic_release_fence();
        return arch_atomic_inc_return_relaxed(v);
#elif defined(arch_atomic_inc_return)
        return arch_atomic_inc_return(v);
#else
        return raw_atomic_add_return_release(1, v);
#endif
}

/**
 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_inc_return_relaxed(atomic_t *v)
{
#if defined(arch_atomic_inc_return_relaxed)
        return arch_atomic_inc_return_relaxed(v);
#elif defined(arch_atomic_inc_return)
        return arch_atomic_inc_return(v);
#else
        return raw_atomic_add_return_relaxed(1, v);
#endif
}

/**
 * raw_atomic_fetch_inc() - atomic increment with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_inc(atomic_t *v)
{
#if defined(arch_atomic_fetch_inc)
        return arch_atomic_fetch_inc(v);
#elif defined(arch_atomic_fetch_inc_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_inc_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_fetch_add(1, v);
#endif
}

/**
 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_inc_acquire(atomic_t *v)
{
#if defined(arch_atomic_fetch_inc_acquire)
        return arch_atomic_fetch_inc_acquire(v);
#elif defined(arch_atomic_fetch_inc_relaxed)
        int ret = arch_atomic_fetch_inc_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_inc)
        return arch_atomic_fetch_inc(v);
#else
        return raw_atomic_fetch_add_acquire(1, v);
#endif
}

/**
 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_inc_release(atomic_t *v)
{
#if defined(arch_atomic_fetch_inc_release)
        return arch_atomic_fetch_inc_release(v);
#elif defined(arch_atomic_fetch_inc_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_inc_relaxed(v);
#elif defined(arch_atomic_fetch_inc)
        return arch_atomic_fetch_inc(v);
#else
        return raw_atomic_fetch_add_release(1, v);
#endif
}

/**
 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_inc_relaxed(atomic_t *v)
{
#if defined(arch_atomic_fetch_inc_relaxed)
        return arch_atomic_fetch_inc_relaxed(v);
#elif defined(arch_atomic_fetch_inc)
        return arch_atomic_fetch_inc(v);
#else
        return raw_atomic_fetch_add_relaxed(1, v);
#endif
}

/**
 * raw_atomic_dec() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_dec(atomic_t *v)
{
#if defined(arch_atomic_dec)
        arch_atomic_dec(v);
#else
        raw_atomic_sub(1, v);
#endif
}

/**
 * raw_atomic_dec_return() - atomic decrement with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_dec_return(atomic_t *v)
{
#if defined(arch_atomic_dec_return)
        return arch_atomic_dec_return(v);
#elif defined(arch_atomic_dec_return_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_dec_return_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_sub_return(1, v);
#endif
}

/**
 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_dec_return_acquire(atomic_t *v)
{
#if defined(arch_atomic_dec_return_acquire)
        return arch_atomic_dec_return_acquire(v);
#elif defined(arch_atomic_dec_return_relaxed)
        int ret = arch_atomic_dec_return_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_dec_return)
        return arch_atomic_dec_return(v);
#else
        return raw_atomic_sub_return_acquire(1, v);
#endif
}

/**
 * raw_atomic_dec_return_release() - atomic decrement with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_dec_return_release(atomic_t *v)
{
#if defined(arch_atomic_dec_return_release)
        return arch_atomic_dec_return_release(v);
#elif defined(arch_atomic_dec_return_relaxed)
        __atomic_release_fence();
        return arch_atomic_dec_return_relaxed(v);
#elif defined(arch_atomic_dec_return)
        return arch_atomic_dec_return(v);
#else
        return raw_atomic_sub_return_release(1, v);
#endif
}

/**
 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline int
raw_atomic_dec_return_relaxed(atomic_t *v)
{
#if defined(arch_atomic_dec_return_relaxed)
        return arch_atomic_dec_return_relaxed(v);
#elif defined(arch_atomic_dec_return)
        return arch_atomic_dec_return(v);
#else
        return raw_atomic_sub_return_relaxed(1, v);
#endif
}

/**
 * raw_atomic_fetch_dec() - atomic decrement with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_dec(atomic_t *v)
{
#if defined(arch_atomic_fetch_dec)
        return arch_atomic_fetch_dec(v);
#elif defined(arch_atomic_fetch_dec_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_dec_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_fetch_sub(1, v);
#endif
}

/**
 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_dec_acquire(atomic_t *v)
{
#if defined(arch_atomic_fetch_dec_acquire)
        return arch_atomic_fetch_dec_acquire(v);
#elif defined(arch_atomic_fetch_dec_relaxed)
        int ret = arch_atomic_fetch_dec_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_dec)
        return arch_atomic_fetch_dec(v);
#else
        return raw_atomic_fetch_sub_acquire(1, v);
#endif
}

/**
 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_dec_release(atomic_t *v)
{
#if defined(arch_atomic_fetch_dec_release)
        return arch_atomic_fetch_dec_release(v);
#elif defined(arch_atomic_fetch_dec_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_dec_relaxed(v);
#elif defined(arch_atomic_fetch_dec)
        return arch_atomic_fetch_dec(v);
#else
        return raw_atomic_fetch_sub_release(1, v);
#endif
}

/**
 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_dec_relaxed(atomic_t *v)
{
#if defined(arch_atomic_fetch_dec_relaxed)
        return arch_atomic_fetch_dec_relaxed(v);
#elif defined(arch_atomic_fetch_dec)
        return arch_atomic_fetch_dec(v);
#else
        return raw_atomic_fetch_sub_relaxed(1, v);
#endif
}

/**
 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_and(int i, atomic_t *v)
{
        arch_atomic_and(i, v);
}

/**
 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_and(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_and)
        return arch_atomic_fetch_and(i, v);
#elif defined(arch_atomic_fetch_and_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_and_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_fetch_and"
#endif
}

/**
 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_and_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_and_acquire)
        return arch_atomic_fetch_and_acquire(i, v);
#elif defined(arch_atomic_fetch_and_relaxed)
        int ret = arch_atomic_fetch_and_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_and)
        return arch_atomic_fetch_and(i, v);
#else
#error "Unable to define raw_atomic_fetch_and_acquire"
#endif
}

/**
 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_and_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_and_release)
        return arch_atomic_fetch_and_release(i, v);
#elif defined(arch_atomic_fetch_and_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_and_relaxed(i, v);
#elif defined(arch_atomic_fetch_and)
        return arch_atomic_fetch_and(i, v);
#else
#error "Unable to define raw_atomic_fetch_and_release"
#endif
}

/**
 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_and_relaxed)
        return arch_atomic_fetch_and_relaxed(i, v);
#elif defined(arch_atomic_fetch_and)
        return arch_atomic_fetch_and(i, v);
#else
#error "Unable to define raw_atomic_fetch_and_relaxed"
#endif
}

/**
 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_andnot(int i, atomic_t *v)
{
#if defined(arch_atomic_andnot)
        arch_atomic_andnot(i, v);
#else
        raw_atomic_and(~i, v);
#endif
}

/**
 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_andnot(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_andnot)
        return arch_atomic_fetch_andnot(i, v);
#elif defined(arch_atomic_fetch_andnot_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_andnot_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_fetch_and(~i, v);
#endif
}

/**
 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_andnot_acquire)
        return arch_atomic_fetch_andnot_acquire(i, v);
#elif defined(arch_atomic_fetch_andnot_relaxed)
        int ret = arch_atomic_fetch_andnot_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_andnot)
        return arch_atomic_fetch_andnot(i, v);
#else
        return raw_atomic_fetch_and_acquire(~i, v);
#endif
}

/**
 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_andnot_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_andnot_release)
        return arch_atomic_fetch_andnot_release(i, v);
#elif defined(arch_atomic_fetch_andnot_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_andnot_relaxed(i, v);
#elif defined(arch_atomic_fetch_andnot)
        return arch_atomic_fetch_andnot(i, v);
#else
        return raw_atomic_fetch_and_release(~i, v);
#endif
}

/**
 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_andnot_relaxed)
        return arch_atomic_fetch_andnot_relaxed(i, v);
#elif defined(arch_atomic_fetch_andnot)
        return arch_atomic_fetch_andnot(i, v);
#else
        return raw_atomic_fetch_and_relaxed(~i, v);
#endif
}

/**
 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_or(int i, atomic_t *v)
{
        arch_atomic_or(i, v);
}

/**
 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_or(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_or)
        return arch_atomic_fetch_or(i, v);
#elif defined(arch_atomic_fetch_or_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_or_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_fetch_or"
#endif
}

/**
 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_or_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_or_acquire)
        return arch_atomic_fetch_or_acquire(i, v);
#elif defined(arch_atomic_fetch_or_relaxed)
        int ret = arch_atomic_fetch_or_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_or)
        return arch_atomic_fetch_or(i, v);
#else
#error "Unable to define raw_atomic_fetch_or_acquire"
#endif
}

/**
 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_or_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_or_release)
        return arch_atomic_fetch_or_release(i, v);
#elif defined(arch_atomic_fetch_or_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_or_relaxed(i, v);
#elif defined(arch_atomic_fetch_or)
        return arch_atomic_fetch_or(i, v);
#else
#error "Unable to define raw_atomic_fetch_or_release"
#endif
}

/**
 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_or_relaxed)
        return arch_atomic_fetch_or_relaxed(i, v);
#elif defined(arch_atomic_fetch_or)
        return arch_atomic_fetch_or(i, v);
#else
#error "Unable to define raw_atomic_fetch_or_relaxed"
#endif
}

/**
 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic_xor(int i, atomic_t *v)
{
        arch_atomic_xor(i, v);
}

/**
 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_xor(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_xor)
        return arch_atomic_fetch_xor(i, v);
#elif defined(arch_atomic_fetch_xor_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_fetch_xor_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic_fetch_xor"
#endif
}

/**
 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_xor_acquire)
        return arch_atomic_fetch_xor_acquire(i, v);
#elif defined(arch_atomic_fetch_xor_relaxed)
        int ret = arch_atomic_fetch_xor_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_fetch_xor)
        return arch_atomic_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic_fetch_xor_acquire"
#endif
}

/**
 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_xor_release(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_xor_release)
        return arch_atomic_fetch_xor_release(i, v);
#elif defined(arch_atomic_fetch_xor_relaxed)
        __atomic_release_fence();
        return arch_atomic_fetch_xor_relaxed(i, v);
#elif defined(arch_atomic_fetch_xor)
        return arch_atomic_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic_fetch_xor_release"
#endif
}

/**
 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
 * @i: int value
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_fetch_xor_relaxed)
        return arch_atomic_fetch_xor_relaxed(i, v);
#elif defined(arch_atomic_fetch_xor)
        return arch_atomic_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic_fetch_xor_relaxed"
#endif
}

/**
 * raw_atomic_xchg() - atomic exchange with full ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_xchg(atomic_t *v, int new)
{
#if defined(arch_atomic_xchg)
        return arch_atomic_xchg(v, new);
#elif defined(arch_atomic_xchg_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_xchg_relaxed(v, new);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_xchg(&v->counter, new);
#endif
}

/**
 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_xchg_acquire(atomic_t *v, int new)
{
#if defined(arch_atomic_xchg_acquire)
        return arch_atomic_xchg_acquire(v, new);
#elif defined(arch_atomic_xchg_relaxed)
        int ret = arch_atomic_xchg_relaxed(v, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_xchg)
        return arch_atomic_xchg(v, new);
#else
        return raw_xchg_acquire(&v->counter, new);
#endif
}

/**
 * raw_atomic_xchg_release() - atomic exchange with release ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_xchg_release(atomic_t *v, int new)
{
#if defined(arch_atomic_xchg_release)
        return arch_atomic_xchg_release(v, new);
#elif defined(arch_atomic_xchg_relaxed)
        __atomic_release_fence();
        return arch_atomic_xchg_relaxed(v, new);
#elif defined(arch_atomic_xchg)
        return arch_atomic_xchg(v, new);
#else
        return raw_xchg_release(&v->counter, new);
#endif
}

/**
 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @new: int value to assign
 *
 * Atomically updates @v to @new with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_xchg_relaxed(atomic_t *v, int new)
{
#if defined(arch_atomic_xchg_relaxed)
        return arch_atomic_xchg_relaxed(v, new);
#elif defined(arch_atomic_xchg)
        return arch_atomic_xchg(v, new);
#else
        return raw_xchg_relaxed(&v->counter, new);
#endif
}

/**
 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_cmpxchg(atomic_t *v, int old, int new)
{
#if defined(arch_atomic_cmpxchg)
        return arch_atomic_cmpxchg(v, old, new);
#elif defined(arch_atomic_cmpxchg_relaxed)
        int ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_cmpxchg_relaxed(v, old, new);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_cmpxchg(&v->counter, old, new);
#endif
}

/**
 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
{
#if defined(arch_atomic_cmpxchg_acquire)
        return arch_atomic_cmpxchg_acquire(v, old, new);
#elif defined(arch_atomic_cmpxchg_relaxed)
        int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_cmpxchg)
        return arch_atomic_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_acquire(&v->counter, old, new);
#endif
}

/**
 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
{
#if defined(arch_atomic_cmpxchg_release)
        return arch_atomic_cmpxchg_release(v, old, new);
#elif defined(arch_atomic_cmpxchg_relaxed)
        __atomic_release_fence();
        return arch_atomic_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic_cmpxchg)
        return arch_atomic_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_release(&v->counter, old, new);
#endif
}

/**
 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @old: int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
{
#if defined(arch_atomic_cmpxchg_relaxed)
        return arch_atomic_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic_cmpxchg)
        return arch_atomic_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_relaxed(&v->counter, old, new);
#endif
}

/**
 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
#if defined(arch_atomic_try_cmpxchg)
        return arch_atomic_try_cmpxchg(v, old, new);
#elif defined(arch_atomic_try_cmpxchg_relaxed)
        bool ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
        __atomic_post_full_fence();
        return ret;
#else
        int r, o = *old;
        r = raw_atomic_cmpxchg(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
{
#if defined(arch_atomic_try_cmpxchg_acquire)
        return arch_atomic_try_cmpxchg_acquire(v, old, new);
#elif defined(arch_atomic_try_cmpxchg_relaxed)
        bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_try_cmpxchg)
        return arch_atomic_try_cmpxchg(v, old, new);
#else
        int r, o = *old;
        r = raw_atomic_cmpxchg_acquire(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
{
#if defined(arch_atomic_try_cmpxchg_release)
        return arch_atomic_try_cmpxchg_release(v, old, new);
#elif defined(arch_atomic_try_cmpxchg_relaxed)
        __atomic_release_fence();
        return arch_atomic_try_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic_try_cmpxchg)
        return arch_atomic_try_cmpxchg(v, old, new);
#else
        int r, o = *old;
        r = raw_atomic_cmpxchg_release(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic_t
 * @old: pointer to int value to compare with
 * @new: int value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
{
#if defined(arch_atomic_try_cmpxchg_relaxed)
        return arch_atomic_try_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic_try_cmpxchg)
        return arch_atomic_try_cmpxchg(v, old, new);
#else
        int r, o = *old;
        r = raw_atomic_cmpxchg_relaxed(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
 * @i: int value to subtract
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic_sub_and_test(int i, atomic_t *v)
{
#if defined(arch_atomic_sub_and_test)
        return arch_atomic_sub_and_test(i, v);
#else
        return raw_atomic_sub_return(i, v) == 0;
#endif
}

/**
 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic_dec_and_test(atomic_t *v)
{
#if defined(arch_atomic_dec_and_test)
        return arch_atomic_dec_and_test(v);
#else
        return raw_atomic_dec_return(v) == 0;
#endif
}

/**
 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic_inc_and_test(atomic_t *v)
{
#if defined(arch_atomic_inc_and_test)
        return arch_atomic_inc_and_test(v);
#else
        return raw_atomic_inc_return(v) == 0;
#endif
}

/**
 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic_add_negative(int i, atomic_t *v)
{
#if defined(arch_atomic_add_negative)
        return arch_atomic_add_negative(i, v);
#elif defined(arch_atomic_add_negative_relaxed)
        bool ret;
        __atomic_pre_full_fence();
        ret = arch_atomic_add_negative_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic_add_return(i, v) < 0;
#endif
}

/**
 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic_add_negative_acquire(int i, atomic_t *v)
{
#if defined(arch_atomic_add_negative_acquire)
        return arch_atomic_add_negative_acquire(i, v);
#elif defined(arch_atomic_add_negative_relaxed)
        bool ret = arch_atomic_add_negative_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic_add_negative)
        return arch_atomic_add_negative(i, v);
#else
        return raw_atomic_add_return_acquire(i, v) < 0;
#endif
}

/**
 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic_add_negative_release(int i, atomic_t *v)
{
#if defined(arch_atomic_add_negative_release)
        return arch_atomic_add_negative_release(i, v);
#elif defined(arch_atomic_add_negative_relaxed)
        __atomic_release_fence();
        return arch_atomic_add_negative_relaxed(i, v);
#elif defined(arch_atomic_add_negative)
        return arch_atomic_add_negative(i, v);
#else
        return raw_atomic_add_return_release(i, v) < 0;
#endif
}

/**
 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
 * @i: int value to add
 * @v: pointer to atomic_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic_add_negative_relaxed(int i, atomic_t *v)
{
#if defined(arch_atomic_add_negative_relaxed)
        return arch_atomic_add_negative_relaxed(i, v);
#elif defined(arch_atomic_add_negative)
        return arch_atomic_add_negative(i, v);
#else
        return raw_atomic_add_return_relaxed(i, v) < 0;
#endif
}

/**
 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_t
 * @a: int value to add
 * @u: int value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline int
raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
{
#if defined(arch_atomic_fetch_add_unless)
        return arch_atomic_fetch_add_unless(v, a, u);
#else
        int c = raw_atomic_read(v);

        do {
                if (unlikely(c == u))
                        break;
        } while (!raw_atomic_try_cmpxchg(v, &c, c + a));

        return c;
#endif
}

/**
 * raw_atomic_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic_t
 * @a: int value to add
 * @u: int value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic_add_unless(atomic_t *v, int a, int u)
{
#if defined(arch_atomic_add_unless)
        return arch_atomic_add_unless(v, a, u);
#else
        return raw_atomic_fetch_add_unless(v, a, u) != u;
#endif
}

/**
 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic_inc_not_zero(atomic_t *v)
{
#if defined(arch_atomic_inc_not_zero)
        return arch_atomic_inc_not_zero(v);
#else
        return raw_atomic_add_unless(v, 1, 0);
#endif
}

/**
 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic_inc_unless_negative(atomic_t *v)
{
#if defined(arch_atomic_inc_unless_negative)
        return arch_atomic_inc_unless_negative(v);
#else
        int c = raw_atomic_read(v);

        do {
                if (unlikely(c < 0))
                        return false;
        } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));

        return true;
#endif
}

/**
 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic_dec_unless_positive(atomic_t *v)
{
#if defined(arch_atomic_dec_unless_positive)
        return arch_atomic_dec_unless_positive(v);
#else
        int c = raw_atomic_read(v);

        do {
                if (unlikely(c > 0))
                        return false;
        } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));

        return true;
#endif
}

/**
 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
 * @v: pointer to atomic_t
 *
 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
 *
 * Return: The old value of (@v - 1), regardless of whether @v was updated.
 */
static __always_inline int
raw_atomic_dec_if_positive(atomic_t *v)
{
#if defined(arch_atomic_dec_if_positive)
        return arch_atomic_dec_if_positive(v);
#else
        int dec, c = raw_atomic_read(v);

        do {
                dec = c - 1;
                if (unlikely(dec < 0))
                        break;
        } while (!raw_atomic_try_cmpxchg(v, &c, dec));

        return dec;
#endif
}

#ifdef CONFIG_GENERIC_ATOMIC64
#include <asm-generic/atomic64.h>
#endif

/**
 * raw_atomic64_read() - atomic load with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically loads the value of @v with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
 *
 * Return: The value loaded from @v.
 */
static __always_inline s64
raw_atomic64_read(const atomic64_t *v)
{
        return arch_atomic64_read(v);
}

/**
 * raw_atomic64_read_acquire() - atomic load with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically loads the value of @v with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
 *
 * Return: The value loaded from @v.
 */
static __always_inline s64
raw_atomic64_read_acquire(const atomic64_t *v)
{
#if defined(arch_atomic64_read_acquire)
        return arch_atomic64_read_acquire(v);
#else
        s64 ret;

        if (__native_word(atomic64_t)) {
                ret = smp_load_acquire(&(v)->counter);
        } else {
                ret = raw_atomic64_read(v);
                __atomic_acquire_fence();
        }

        return ret;
#endif
}

/**
 * raw_atomic64_set() - atomic set with relaxed ordering
 * @v: pointer to atomic64_t
 * @i: s64 value to assign
 *
 * Atomically sets @v to @i with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_set(atomic64_t *v, s64 i)
{
        arch_atomic64_set(v, i);
}

/**
 * raw_atomic64_set_release() - atomic set with release ordering
 * @v: pointer to atomic64_t
 * @i: s64 value to assign
 *
 * Atomically sets @v to @i with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_set_release(atomic64_t *v, s64 i)
{
#if defined(arch_atomic64_set_release)
        arch_atomic64_set_release(v, i);
#else
        if (__native_word(atomic64_t)) {
                smp_store_release(&(v)->counter, i);
        } else {
                __atomic_release_fence();
                raw_atomic64_set(v, i);
        }
#endif
}

/**
 * raw_atomic64_add() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_add(s64 i, atomic64_t *v)
{
        arch_atomic64_add(i, v);
}

/**
 * raw_atomic64_add_return() - atomic add with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_add_return(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_return)
        return arch_atomic64_add_return(i, v);
#elif defined(arch_atomic64_add_return_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_add_return_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_add_return"
#endif
}

/**
 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_return_acquire)
        return arch_atomic64_add_return_acquire(i, v);
#elif defined(arch_atomic64_add_return_relaxed)
        s64 ret = arch_atomic64_add_return_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_add_return)
        return arch_atomic64_add_return(i, v);
#else
#error "Unable to define raw_atomic64_add_return_acquire"
#endif
}

/**
 * raw_atomic64_add_return_release() - atomic add with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_add_return_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_return_release)
        return arch_atomic64_add_return_release(i, v);
#elif defined(arch_atomic64_add_return_relaxed)
        __atomic_release_fence();
        return arch_atomic64_add_return_relaxed(i, v);
#elif defined(arch_atomic64_add_return)
        return arch_atomic64_add_return(i, v);
#else
#error "Unable to define raw_atomic64_add_return_release"
#endif
}

/**
 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_return_relaxed)
        return arch_atomic64_add_return_relaxed(i, v);
#elif defined(arch_atomic64_add_return)
        return arch_atomic64_add_return(i, v);
#else
#error "Unable to define raw_atomic64_add_return_relaxed"
#endif
}

/**
 * raw_atomic64_fetch_add() - atomic add with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_add(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_add)
        return arch_atomic64_fetch_add(i, v);
#elif defined(arch_atomic64_fetch_add_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_add_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_fetch_add"
#endif
}

/**
 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_add_acquire)
        return arch_atomic64_fetch_add_acquire(i, v);
#elif defined(arch_atomic64_fetch_add_relaxed)
        s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_add)
        return arch_atomic64_fetch_add(i, v);
#else
#error "Unable to define raw_atomic64_fetch_add_acquire"
#endif
}

/**
 * raw_atomic64_fetch_add_release() - atomic add with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_add_release)
        return arch_atomic64_fetch_add_release(i, v);
#elif defined(arch_atomic64_fetch_add_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_add_relaxed(i, v);
#elif defined(arch_atomic64_fetch_add)
        return arch_atomic64_fetch_add(i, v);
#else
#error "Unable to define raw_atomic64_fetch_add_release"
#endif
}

/**
 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_add_relaxed)
        return arch_atomic64_fetch_add_relaxed(i, v);
#elif defined(arch_atomic64_fetch_add)
        return arch_atomic64_fetch_add(i, v);
#else
#error "Unable to define raw_atomic64_fetch_add_relaxed"
#endif
}

/**
 * raw_atomic64_sub() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_sub(s64 i, atomic64_t *v)
{
        arch_atomic64_sub(i, v);
}

/**
 * raw_atomic64_sub_return() - atomic subtract with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_sub_return(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_sub_return)
        return arch_atomic64_sub_return(i, v);
#elif defined(arch_atomic64_sub_return_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_sub_return_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_sub_return"
#endif
}

/**
 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_sub_return_acquire)
        return arch_atomic64_sub_return_acquire(i, v);
#elif defined(arch_atomic64_sub_return_relaxed)
        s64 ret = arch_atomic64_sub_return_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_sub_return)
        return arch_atomic64_sub_return(i, v);
#else
#error "Unable to define raw_atomic64_sub_return_acquire"
#endif
}

/**
 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_sub_return_release)
        return arch_atomic64_sub_return_release(i, v);
#elif defined(arch_atomic64_sub_return_relaxed)
        __atomic_release_fence();
        return arch_atomic64_sub_return_relaxed(i, v);
#elif defined(arch_atomic64_sub_return)
        return arch_atomic64_sub_return(i, v);
#else
#error "Unable to define raw_atomic64_sub_return_release"
#endif
}

/**
 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_sub_return_relaxed)
        return arch_atomic64_sub_return_relaxed(i, v);
#elif defined(arch_atomic64_sub_return)
        return arch_atomic64_sub_return(i, v);
#else
#error "Unable to define raw_atomic64_sub_return_relaxed"
#endif
}

/**
 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_sub)
        return arch_atomic64_fetch_sub(i, v);
#elif defined(arch_atomic64_fetch_sub_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_sub_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_fetch_sub"
#endif
}

/**
 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_sub_acquire)
        return arch_atomic64_fetch_sub_acquire(i, v);
#elif defined(arch_atomic64_fetch_sub_relaxed)
        s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_sub)
        return arch_atomic64_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic64_fetch_sub_acquire"
#endif
}

/**
 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_sub_release)
        return arch_atomic64_fetch_sub_release(i, v);
#elif defined(arch_atomic64_fetch_sub_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_sub_relaxed(i, v);
#elif defined(arch_atomic64_fetch_sub)
        return arch_atomic64_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic64_fetch_sub_release"
#endif
}

/**
 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_sub_relaxed)
        return arch_atomic64_fetch_sub_relaxed(i, v);
#elif defined(arch_atomic64_fetch_sub)
        return arch_atomic64_fetch_sub(i, v);
#else
#error "Unable to define raw_atomic64_fetch_sub_relaxed"
#endif
}

/**
 * raw_atomic64_inc() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_inc(atomic64_t *v)
{
#if defined(arch_atomic64_inc)
        arch_atomic64_inc(v);
#else
        raw_atomic64_add(1, v);
#endif
}

/**
 * raw_atomic64_inc_return() - atomic increment with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_inc_return(atomic64_t *v)
{
#if defined(arch_atomic64_inc_return)
        return arch_atomic64_inc_return(v);
#elif defined(arch_atomic64_inc_return_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_inc_return_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_add_return(1, v);
#endif
}

/**
 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_inc_return_acquire(atomic64_t *v)
{
#if defined(arch_atomic64_inc_return_acquire)
        return arch_atomic64_inc_return_acquire(v);
#elif defined(arch_atomic64_inc_return_relaxed)
        s64 ret = arch_atomic64_inc_return_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_inc_return)
        return arch_atomic64_inc_return(v);
#else
        return raw_atomic64_add_return_acquire(1, v);
#endif
}

/**
 * raw_atomic64_inc_return_release() - atomic increment with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_inc_return_release(atomic64_t *v)
{
#if defined(arch_atomic64_inc_return_release)
        return arch_atomic64_inc_return_release(v);
#elif defined(arch_atomic64_inc_return_relaxed)
        __atomic_release_fence();
        return arch_atomic64_inc_return_relaxed(v);
#elif defined(arch_atomic64_inc_return)
        return arch_atomic64_inc_return(v);
#else
        return raw_atomic64_add_return_release(1, v);
#endif
}

/**
 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_inc_return_relaxed(atomic64_t *v)
{
#if defined(arch_atomic64_inc_return_relaxed)
        return arch_atomic64_inc_return_relaxed(v);
#elif defined(arch_atomic64_inc_return)
        return arch_atomic64_inc_return(v);
#else
        return raw_atomic64_add_return_relaxed(1, v);
#endif
}

/**
 * raw_atomic64_fetch_inc() - atomic increment with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_inc(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_inc)
        return arch_atomic64_fetch_inc(v);
#elif defined(arch_atomic64_fetch_inc_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_inc_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_fetch_add(1, v);
#endif
}

/**
 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_inc_acquire(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_inc_acquire)
        return arch_atomic64_fetch_inc_acquire(v);
#elif defined(arch_atomic64_fetch_inc_relaxed)
        s64 ret = arch_atomic64_fetch_inc_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_inc)
        return arch_atomic64_fetch_inc(v);
#else
        return raw_atomic64_fetch_add_acquire(1, v);
#endif
}

/**
 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_inc_release(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_inc_release)
        return arch_atomic64_fetch_inc_release(v);
#elif defined(arch_atomic64_fetch_inc_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_inc_relaxed(v);
#elif defined(arch_atomic64_fetch_inc)
        return arch_atomic64_fetch_inc(v);
#else
        return raw_atomic64_fetch_add_release(1, v);
#endif
}

/**
 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_inc_relaxed)
        return arch_atomic64_fetch_inc_relaxed(v);
#elif defined(arch_atomic64_fetch_inc)
        return arch_atomic64_fetch_inc(v);
#else
        return raw_atomic64_fetch_add_relaxed(1, v);
#endif
}

/**
 * raw_atomic64_dec() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_dec(atomic64_t *v)
{
#if defined(arch_atomic64_dec)
        arch_atomic64_dec(v);
#else
        raw_atomic64_sub(1, v);
#endif
}

/**
 * raw_atomic64_dec_return() - atomic decrement with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_dec_return(atomic64_t *v)
{
#if defined(arch_atomic64_dec_return)
        return arch_atomic64_dec_return(v);
#elif defined(arch_atomic64_dec_return_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_dec_return_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_sub_return(1, v);
#endif
}

/**
 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_dec_return_acquire(atomic64_t *v)
{
#if defined(arch_atomic64_dec_return_acquire)
        return arch_atomic64_dec_return_acquire(v);
#elif defined(arch_atomic64_dec_return_relaxed)
        s64 ret = arch_atomic64_dec_return_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_dec_return)
        return arch_atomic64_dec_return(v);
#else
        return raw_atomic64_sub_return_acquire(1, v);
#endif
}

/**
 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_dec_return_release(atomic64_t *v)
{
#if defined(arch_atomic64_dec_return_release)
        return arch_atomic64_dec_return_release(v);
#elif defined(arch_atomic64_dec_return_relaxed)
        __atomic_release_fence();
        return arch_atomic64_dec_return_relaxed(v);
#elif defined(arch_atomic64_dec_return)
        return arch_atomic64_dec_return(v);
#else
        return raw_atomic64_sub_return_release(1, v);
#endif
}

/**
 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
 *
 * Return: The updated value of @v.
 */
static __always_inline s64
raw_atomic64_dec_return_relaxed(atomic64_t *v)
{
#if defined(arch_atomic64_dec_return_relaxed)
        return arch_atomic64_dec_return_relaxed(v);
#elif defined(arch_atomic64_dec_return)
        return arch_atomic64_dec_return(v);
#else
        return raw_atomic64_sub_return_relaxed(1, v);
#endif
}

/**
 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_dec(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_dec)
        return arch_atomic64_fetch_dec(v);
#elif defined(arch_atomic64_fetch_dec_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_dec_relaxed(v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_fetch_sub(1, v);
#endif
}

/**
 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_dec_acquire(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_dec_acquire)
        return arch_atomic64_fetch_dec_acquire(v);
#elif defined(arch_atomic64_fetch_dec_relaxed)
        s64 ret = arch_atomic64_fetch_dec_relaxed(v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_dec)
        return arch_atomic64_fetch_dec(v);
#else
        return raw_atomic64_fetch_sub_acquire(1, v);
#endif
}

/**
 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_dec_release(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_dec_release)
        return arch_atomic64_fetch_dec_release(v);
#elif defined(arch_atomic64_fetch_dec_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_dec_relaxed(v);
#elif defined(arch_atomic64_fetch_dec)
        return arch_atomic64_fetch_dec(v);
#else
        return raw_atomic64_fetch_sub_release(1, v);
#endif
}

/**
 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
{
#if defined(arch_atomic64_fetch_dec_relaxed)
        return arch_atomic64_fetch_dec_relaxed(v);
#elif defined(arch_atomic64_fetch_dec)
        return arch_atomic64_fetch_dec(v);
#else
        return raw_atomic64_fetch_sub_relaxed(1, v);
#endif
}

/**
 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_and(s64 i, atomic64_t *v)
{
        arch_atomic64_and(i, v);
}

/**
 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_and(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_and)
        return arch_atomic64_fetch_and(i, v);
#elif defined(arch_atomic64_fetch_and_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_and_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_fetch_and"
#endif
}

/**
 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_and_acquire)
        return arch_atomic64_fetch_and_acquire(i, v);
#elif defined(arch_atomic64_fetch_and_relaxed)
        s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_and)
        return arch_atomic64_fetch_and(i, v);
#else
#error "Unable to define raw_atomic64_fetch_and_acquire"
#endif
}

/**
 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_and_release)
        return arch_atomic64_fetch_and_release(i, v);
#elif defined(arch_atomic64_fetch_and_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_and_relaxed(i, v);
#elif defined(arch_atomic64_fetch_and)
        return arch_atomic64_fetch_and(i, v);
#else
#error "Unable to define raw_atomic64_fetch_and_release"
#endif
}

/**
 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_and_relaxed)
        return arch_atomic64_fetch_and_relaxed(i, v);
#elif defined(arch_atomic64_fetch_and)
        return arch_atomic64_fetch_and(i, v);
#else
#error "Unable to define raw_atomic64_fetch_and_relaxed"
#endif
}

/**
 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_andnot(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_andnot)
        arch_atomic64_andnot(i, v);
#else
        raw_atomic64_and(~i, v);
#endif
}

/**
 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_andnot)
        return arch_atomic64_fetch_andnot(i, v);
#elif defined(arch_atomic64_fetch_andnot_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_andnot_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_fetch_and(~i, v);
#endif
}

/**
 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_andnot_acquire)
        return arch_atomic64_fetch_andnot_acquire(i, v);
#elif defined(arch_atomic64_fetch_andnot_relaxed)
        s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_andnot)
        return arch_atomic64_fetch_andnot(i, v);
#else
        return raw_atomic64_fetch_and_acquire(~i, v);
#endif
}

/**
 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_andnot_release)
        return arch_atomic64_fetch_andnot_release(i, v);
#elif defined(arch_atomic64_fetch_andnot_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_andnot_relaxed(i, v);
#elif defined(arch_atomic64_fetch_andnot)
        return arch_atomic64_fetch_andnot(i, v);
#else
        return raw_atomic64_fetch_and_release(~i, v);
#endif
}

/**
 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_andnot_relaxed)
        return arch_atomic64_fetch_andnot_relaxed(i, v);
#elif defined(arch_atomic64_fetch_andnot)
        return arch_atomic64_fetch_andnot(i, v);
#else
        return raw_atomic64_fetch_and_relaxed(~i, v);
#endif
}

/**
 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_or(s64 i, atomic64_t *v)
{
        arch_atomic64_or(i, v);
}

/**
 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_or(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_or)
        return arch_atomic64_fetch_or(i, v);
#elif defined(arch_atomic64_fetch_or_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_or_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_fetch_or"
#endif
}

/**
 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_or_acquire)
        return arch_atomic64_fetch_or_acquire(i, v);
#elif defined(arch_atomic64_fetch_or_relaxed)
        s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_or)
        return arch_atomic64_fetch_or(i, v);
#else
#error "Unable to define raw_atomic64_fetch_or_acquire"
#endif
}

/**
 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_or_release)
        return arch_atomic64_fetch_or_release(i, v);
#elif defined(arch_atomic64_fetch_or_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_or_relaxed(i, v);
#elif defined(arch_atomic64_fetch_or)
        return arch_atomic64_fetch_or(i, v);
#else
#error "Unable to define raw_atomic64_fetch_or_release"
#endif
}

/**
 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v | @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_or_relaxed)
        return arch_atomic64_fetch_or_relaxed(i, v);
#elif defined(arch_atomic64_fetch_or)
        return arch_atomic64_fetch_or(i, v);
#else
#error "Unable to define raw_atomic64_fetch_or_relaxed"
#endif
}

/**
 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
 *
 * Return: Nothing.
 */
static __always_inline void
raw_atomic64_xor(s64 i, atomic64_t *v)
{
        arch_atomic64_xor(i, v);
}

/**
 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_xor)
        return arch_atomic64_fetch_xor(i, v);
#elif defined(arch_atomic64_fetch_xor_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_fetch_xor_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
#error "Unable to define raw_atomic64_fetch_xor"
#endif
}

/**
 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_xor_acquire)
        return arch_atomic64_fetch_xor_acquire(i, v);
#elif defined(arch_atomic64_fetch_xor_relaxed)
        s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_fetch_xor)
        return arch_atomic64_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic64_fetch_xor_acquire"
#endif
}

/**
 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_xor_release)
        return arch_atomic64_fetch_xor_release(i, v);
#elif defined(arch_atomic64_fetch_xor_relaxed)
        __atomic_release_fence();
        return arch_atomic64_fetch_xor_relaxed(i, v);
#elif defined(arch_atomic64_fetch_xor)
        return arch_atomic64_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic64_fetch_xor_release"
#endif
}

/**
 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
 * @i: s64 value
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_fetch_xor_relaxed)
        return arch_atomic64_fetch_xor_relaxed(i, v);
#elif defined(arch_atomic64_fetch_xor)
        return arch_atomic64_fetch_xor(i, v);
#else
#error "Unable to define raw_atomic64_fetch_xor_relaxed"
#endif
}

/**
 * raw_atomic64_xchg() - atomic exchange with full ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_xchg(atomic64_t *v, s64 new)
{
#if defined(arch_atomic64_xchg)
        return arch_atomic64_xchg(v, new);
#elif defined(arch_atomic64_xchg_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_xchg_relaxed(v, new);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_xchg(&v->counter, new);
#endif
}

/**
 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
{
#if defined(arch_atomic64_xchg_acquire)
        return arch_atomic64_xchg_acquire(v, new);
#elif defined(arch_atomic64_xchg_relaxed)
        s64 ret = arch_atomic64_xchg_relaxed(v, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_xchg)
        return arch_atomic64_xchg(v, new);
#else
        return raw_xchg_acquire(&v->counter, new);
#endif
}

/**
 * raw_atomic64_xchg_release() - atomic exchange with release ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_xchg_release(atomic64_t *v, s64 new)
{
#if defined(arch_atomic64_xchg_release)
        return arch_atomic64_xchg_release(v, new);
#elif defined(arch_atomic64_xchg_relaxed)
        __atomic_release_fence();
        return arch_atomic64_xchg_relaxed(v, new);
#elif defined(arch_atomic64_xchg)
        return arch_atomic64_xchg(v, new);
#else
        return raw_xchg_release(&v->counter, new);
#endif
}

/**
 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @new: s64 value to assign
 *
 * Atomically updates @v to @new with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
{
#if defined(arch_atomic64_xchg_relaxed)
        return arch_atomic64_xchg_relaxed(v, new);
#elif defined(arch_atomic64_xchg)
        return arch_atomic64_xchg(v, new);
#else
        return raw_xchg_relaxed(&v->counter, new);
#endif
}

/**
 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
#if defined(arch_atomic64_cmpxchg)
        return arch_atomic64_cmpxchg(v, old, new);
#elif defined(arch_atomic64_cmpxchg_relaxed)
        s64 ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_cmpxchg(&v->counter, old, new);
#endif
}

/**
 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
{
#if defined(arch_atomic64_cmpxchg_acquire)
        return arch_atomic64_cmpxchg_acquire(v, old, new);
#elif defined(arch_atomic64_cmpxchg_relaxed)
        s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_cmpxchg)
        return arch_atomic64_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_acquire(&v->counter, old, new);
#endif
}

/**
 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
{
#if defined(arch_atomic64_cmpxchg_release)
        return arch_atomic64_cmpxchg_release(v, old, new);
#elif defined(arch_atomic64_cmpxchg_relaxed)
        __atomic_release_fence();
        return arch_atomic64_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic64_cmpxchg)
        return arch_atomic64_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_release(&v->counter, old, new);
#endif
}

/**
 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @old: s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
{
#if defined(arch_atomic64_cmpxchg_relaxed)
        return arch_atomic64_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic64_cmpxchg)
        return arch_atomic64_cmpxchg(v, old, new);
#else
        return raw_cmpxchg_relaxed(&v->counter, old, new);
#endif
}

/**
 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with full ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
#if defined(arch_atomic64_try_cmpxchg)
        return arch_atomic64_try_cmpxchg(v, old, new);
#elif defined(arch_atomic64_try_cmpxchg_relaxed)
        bool ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
        __atomic_post_full_fence();
        return ret;
#else
        s64 r, o = *old;
        r = raw_atomic64_cmpxchg(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with acquire ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
{
#if defined(arch_atomic64_try_cmpxchg_acquire)
        return arch_atomic64_try_cmpxchg_acquire(v, old, new);
#elif defined(arch_atomic64_try_cmpxchg_relaxed)
        bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_try_cmpxchg)
        return arch_atomic64_try_cmpxchg(v, old, new);
#else
        s64 r, o = *old;
        r = raw_atomic64_cmpxchg_acquire(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with release ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
{
#if defined(arch_atomic64_try_cmpxchg_release)
        return arch_atomic64_try_cmpxchg_release(v, old, new);
#elif defined(arch_atomic64_try_cmpxchg_relaxed)
        __atomic_release_fence();
        return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic64_try_cmpxchg)
        return arch_atomic64_try_cmpxchg(v, old, new);
#else
        s64 r, o = *old;
        r = raw_atomic64_cmpxchg_release(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
 * @v: pointer to atomic64_t
 * @old: pointer to s64 value to compare with
 * @new: s64 value to assign
 *
 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
 * Otherwise, @v is not modified, @old is updated to the current value of @v,
 * and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
 *
 * Return: @true if the exchange occured, @false otherwise.
 */
static __always_inline bool
raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
{
#if defined(arch_atomic64_try_cmpxchg_relaxed)
        return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
#elif defined(arch_atomic64_try_cmpxchg)
        return arch_atomic64_try_cmpxchg(v, old, new);
#else
        s64 r, o = *old;
        r = raw_atomic64_cmpxchg_relaxed(v, o, new);
        if (unlikely(r != o))
                *old = r;
        return likely(r == o);
#endif
}

/**
 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
 * @i: s64 value to subtract
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_sub_and_test)
        return arch_atomic64_sub_and_test(i, v);
#else
        return raw_atomic64_sub_return(i, v) == 0;
#endif
}

/**
 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v - 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic64_dec_and_test(atomic64_t *v)
{
#if defined(arch_atomic64_dec_and_test)
        return arch_atomic64_dec_and_test(v);
#else
        return raw_atomic64_dec_return(v) == 0;
#endif
}

/**
 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + 1) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
 *
 * Return: @true if the resulting value of @v is zero, @false otherwise.
 */
static __always_inline bool
raw_atomic64_inc_and_test(atomic64_t *v)
{
#if defined(arch_atomic64_inc_and_test)
        return arch_atomic64_inc_and_test(v);
#else
        return raw_atomic64_inc_return(v) == 0;
#endif
}

/**
 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with full ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic64_add_negative(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_negative)
        return arch_atomic64_add_negative(i, v);
#elif defined(arch_atomic64_add_negative_relaxed)
        bool ret;
        __atomic_pre_full_fence();
        ret = arch_atomic64_add_negative_relaxed(i, v);
        __atomic_post_full_fence();
        return ret;
#else
        return raw_atomic64_add_return(i, v) < 0;
#endif
}

/**
 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with acquire ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_negative_acquire)
        return arch_atomic64_add_negative_acquire(i, v);
#elif defined(arch_atomic64_add_negative_relaxed)
        bool ret = arch_atomic64_add_negative_relaxed(i, v);
        __atomic_acquire_fence();
        return ret;
#elif defined(arch_atomic64_add_negative)
        return arch_atomic64_add_negative(i, v);
#else
        return raw_atomic64_add_return_acquire(i, v) < 0;
#endif
}

/**
 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with release ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_negative_release)
        return arch_atomic64_add_negative_release(i, v);
#elif defined(arch_atomic64_add_negative_relaxed)
        __atomic_release_fence();
        return arch_atomic64_add_negative_relaxed(i, v);
#elif defined(arch_atomic64_add_negative)
        return arch_atomic64_add_negative(i, v);
#else
        return raw_atomic64_add_return_release(i, v) < 0;
#endif
}

/**
 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
 * @i: s64 value to add
 * @v: pointer to atomic64_t
 *
 * Atomically updates @v to (@v + @i) with relaxed ordering.
 *
 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
 *
 * Return: @true if the resulting value of @v is negative, @false otherwise.
 */
static __always_inline bool
raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
{
#if defined(arch_atomic64_add_negative_relaxed)
        return arch_atomic64_add_negative_relaxed(i, v);
#elif defined(arch_atomic64_add_negative)
        return arch_atomic64_add_negative(i, v);
#else
        return raw_atomic64_add_return_relaxed(i, v) < 0;
#endif
}

/**
 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic64_t
 * @a: s64 value to add
 * @u: s64 value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
 *
 * Return: The original value of @v.
 */
static __always_inline s64
raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
{
#if defined(arch_atomic64_fetch_add_unless)
        return arch_atomic64_fetch_add_unless(v, a, u);
#else
        s64 c = raw_atomic64_read(v);

        do {
                if (unlikely(c == u))
                        break;
        } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));

        return c;
#endif
}

/**
 * raw_atomic64_add_unless() - atomic add unless value with full ordering
 * @v: pointer to atomic64_t
 * @a: s64 value to add
 * @u: s64 value to compare with
 *
 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
{
#if defined(arch_atomic64_add_unless)
        return arch_atomic64_add_unless(v, a, u);
#else
        return raw_atomic64_fetch_add_unless(v, a, u) != u;
#endif
}

/**
 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic64_inc_not_zero(atomic64_t *v)
{
#if defined(arch_atomic64_inc_not_zero)
        return arch_atomic64_inc_not_zero(v);
#else
        return raw_atomic64_add_unless(v, 1, 0);
#endif
}

/**
 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic64_inc_unless_negative(atomic64_t *v)
{
#if defined(arch_atomic64_inc_unless_negative)
        return arch_atomic64_inc_unless_negative(v);
#else
        s64 c = raw_atomic64_read(v);

        do {
                if (unlikely(c < 0))
                        return false;
        } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));

        return true;
#endif
}

/**
 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
 *
 * Return: @true if @v was updated, @false otherwise.
 */
static __always_inline bool
raw_atomic64_dec_unless_positive(atomic64_t *v)
{
#if defined(arch_atomic64_dec_unless_positive)
        return arch_atomic64_dec_unless_positive(v);
#else
        s64 c = raw_atomic64_read(v);

        do {
                if (unlikely(c > 0))
                        return false;
        } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));

        return true;
#endif
}

/**
 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
 * @v: pointer to atomic64_t
 *
 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
 * Otherwise, @v is not modified and relaxed ordering is provided.
 *
 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
 *
 * Return: The old value of (@v - 1), regardless of whether @v was updated.
 */
static __always_inline s64
raw_atomic64_dec_if_positive(atomic64_t *v)
{
#if defined(arch_atomic64_dec_if_positive)
        return arch_atomic64_dec_if_positive(v);
#else
        s64 dec, c = raw_atomic64_read(v);

        do {
                dec = c - 1;
                if (unlikely(dec < 0))
                        break;
        } while (!raw_atomic64_try_cmpxchg(v, &c, dec));

        return dec;
#endif
}

#endif /* _LINUX_ATOMIC_FALLBACK_H */
// b565db590afeeff0d7c9485ccbca5bb6e155749f

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  203 






  205 


  204 























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
// SPDX-License-Identifier: GPL-2.0

/* net/sched/sch_taprio.c         Time Aware Priority Scheduler
 *
 * Authors:        Vinicius Costa Gomes <vinicius.gomes@intel.com>
 *
 */

#include <linux/ethtool.h>
#include <linux/ethtool_netlink.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/math64.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/time.h>
#include <net/gso.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/sch_generic.h>
#include <net/sock.h>
#include <net/tcp.h>

#define TAPRIO_STAT_NOT_SET        (~0ULL)

#include "sch_mqprio_lib.h"

static LIST_HEAD(taprio_list);
static struct static_key_false taprio_have_broken_mqprio;
static struct static_key_false taprio_have_working_mqprio;

#define TAPRIO_ALL_GATES_OPEN -1

#define TXTIME_ASSIST_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST)
#define FULL_OFFLOAD_IS_ENABLED(flags) ((flags) & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)
#define TAPRIO_SUPPORTED_FLAGS \
        (TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST | TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)
#define TAPRIO_FLAGS_INVALID U32_MAX

struct sched_entry {
        /* Durations between this GCL entry and the GCL entry where the
         * respective traffic class gate closes
         */
        u64 gate_duration[TC_MAX_QUEUE];
        atomic_t budget[TC_MAX_QUEUE];
        /* The qdisc makes some effort so that no packet leaves
         * after this time
         */
        ktime_t gate_close_time[TC_MAX_QUEUE];
        struct list_head list;
        /* Used to calculate when to advance the schedule */
        ktime_t end_time;
        ktime_t next_txtime;
        int index;
        u32 gate_mask;
        u32 interval;
        u8 command;
};

struct sched_gate_list {
        /* Longest non-zero contiguous gate durations per traffic class,
         * or 0 if a traffic class gate never opens during the schedule.
         */
        u64 max_open_gate_duration[TC_MAX_QUEUE];
        u32 max_frm_len[TC_MAX_QUEUE]; /* for the fast path */
        u32 max_sdu[TC_MAX_QUEUE]; /* for dump */
        struct rcu_head rcu;
        struct list_head entries;
        size_t num_entries;
        ktime_t cycle_end_time;
        s64 cycle_time;
        s64 cycle_time_extension;
        s64 base_time;
};

struct taprio_sched {
        struct Qdisc **qdiscs;
        struct Qdisc *root;
        u32 flags;
        enum tk_offsets tk_offset;
        int clockid;
        bool offloaded;
        bool detected_mqprio;
        bool broken_mqprio;
        atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+
                                    * speeds it's sub-nanoseconds per byte
                                    */

        /* Protects the update side of the RCU protected current_entry */
        spinlock_t current_entry_lock;
        struct sched_entry __rcu *current_entry;
        struct sched_gate_list __rcu *oper_sched;
        struct sched_gate_list __rcu *admin_sched;
        struct hrtimer advance_timer;
        struct list_head taprio_list;
        int cur_txq[TC_MAX_QUEUE];
        u32 max_sdu[TC_MAX_QUEUE]; /* save info from the user */
        u32 fp[TC_QOPT_MAX_QUEUE]; /* only for dump and offloading */
        u32 txtime_delay;
};

struct __tc_taprio_qopt_offload {
        refcount_t users;
        struct tc_taprio_qopt_offload offload;
};

static void taprio_calculate_gate_durations(struct taprio_sched *q,
                                            struct sched_gate_list *sched)
{
        struct net_device *dev = qdisc_dev(q->root);
        int num_tc = netdev_get_num_tc(dev);
        struct sched_entry *entry, *cur;
        int tc;

        list_for_each_entry(entry, &sched->entries, list) {
                u32 gates_still_open = entry->gate_mask;

                /* For each traffic class, calculate each open gate duration,
                 * starting at this schedule entry and ending at the schedule
                 * entry containing a gate close event for that TC.
                 */
                cur = entry;

                do {
                        if (!gates_still_open)
                                break;

                        for (tc = 0; tc < num_tc; tc++) {
                                if (!(gates_still_open & BIT(tc)))
                                        continue;

                                if (cur->gate_mask & BIT(tc))
                                        entry->gate_duration[tc] += cur->interval;
                                else
                                        gates_still_open &= ~BIT(tc);
                        }

                        cur = list_next_entry_circular(cur, &sched->entries, list);
                } while (cur != entry);

                /* Keep track of the maximum gate duration for each traffic
                 * class, taking care to not confuse a traffic class which is
                 * temporarily closed with one that is always closed.
                 */
                for (tc = 0; tc < num_tc; tc++)
                        if (entry->gate_duration[tc] &&
                            sched->max_open_gate_duration[tc] < entry->gate_duration[tc])
                                sched->max_open_gate_duration[tc] = entry->gate_duration[tc];
        }
}

static bool taprio_entry_allows_tx(ktime_t skb_end_time,
                                   struct sched_entry *entry, int tc)
{
        return ktime_before(skb_end_time, entry->gate_close_time[tc]);
}

static ktime_t sched_base_time(const struct sched_gate_list *sched)
{
        if (!sched)
                return KTIME_MAX;

        return ns_to_ktime(sched->base_time);
}

static ktime_t taprio_mono_to_any(const struct taprio_sched *q, ktime_t mono)
{
        /* This pairs with WRITE_ONCE() in taprio_parse_clockid() */
        enum tk_offsets tk_offset = READ_ONCE(q->tk_offset);

        switch (tk_offset) {
        case TK_OFFS_MAX:
                return mono;
        default:
                return ktime_mono_to_any(mono, tk_offset);
        }
}

static ktime_t taprio_get_time(const struct taprio_sched *q)
{
        return taprio_mono_to_any(q, ktime_get());
}

static void taprio_free_sched_cb(struct rcu_head *head)
{
        struct sched_gate_list *sched = container_of(head, struct sched_gate_list, rcu);
        struct sched_entry *entry, *n;

        list_for_each_entry_safe(entry, n, &sched->entries, list) {
                list_del(&entry->list);
                kfree(entry);
        }

        kfree(sched);
}

static void switch_schedules(struct taprio_sched *q,
                             struct sched_gate_list **admin,
                             struct sched_gate_list **oper)
{
        rcu_assign_pointer(q->oper_sched, *admin);
        rcu_assign_pointer(q->admin_sched, NULL);

        if (*oper)
                call_rcu(&(*oper)->rcu, taprio_free_sched_cb);

        *oper = *admin;
        *admin = NULL;
}

/* Get how much time has been already elapsed in the current cycle. */
static s32 get_cycle_time_elapsed(struct sched_gate_list *sched, ktime_t time)
{
        ktime_t time_since_sched_start;
        s32 time_elapsed;

        time_since_sched_start = ktime_sub(time, sched->base_time);
        div_s64_rem(time_since_sched_start, sched->cycle_time, &time_elapsed);

        return time_elapsed;
}

static ktime_t get_interval_end_time(struct sched_gate_list *sched,
                                     struct sched_gate_list *admin,
                                     struct sched_entry *entry,
                                     ktime_t intv_start)
{
        s32 cycle_elapsed = get_cycle_time_elapsed(sched, intv_start);
        ktime_t intv_end, cycle_ext_end, cycle_end;

        cycle_end = ktime_add_ns(intv_start, sched->cycle_time - cycle_elapsed);
        intv_end = ktime_add_ns(intv_start, entry->interval);
        cycle_ext_end = ktime_add(cycle_end, sched->cycle_time_extension);

        if (ktime_before(intv_end, cycle_end))
                return intv_end;
        else if (admin && admin != sched &&
                 ktime_after(admin->base_time, cycle_end) &&
                 ktime_before(admin->base_time, cycle_ext_end))
                return admin->base_time;
        else
                return cycle_end;
}

static int length_to_duration(struct taprio_sched *q, int len)
{
        return div_u64(len * atomic64_read(&q->picos_per_byte), PSEC_PER_NSEC);
}

static int duration_to_length(struct taprio_sched *q, u64 duration)
{
        return div_u64(duration * PSEC_PER_NSEC, atomic64_read(&q->picos_per_byte));
}

/* Sets sched->max_sdu[] and sched->max_frm_len[] to the minimum between the
 * q->max_sdu[] requested by the user and the max_sdu dynamically determined by
 * the maximum open gate durations at the given link speed.
 */
static void taprio_update_queue_max_sdu(struct taprio_sched *q,
                                        struct sched_gate_list *sched,
                                        struct qdisc_size_table *stab)
{
        struct net_device *dev = qdisc_dev(q->root);
        int num_tc = netdev_get_num_tc(dev);
        u32 max_sdu_from_user;
        u32 max_sdu_dynamic;
        u32 max_sdu;
        int tc;

        for (tc = 0; tc < num_tc; tc++) {
                max_sdu_from_user = q->max_sdu[tc] ?: U32_MAX;

                /* TC gate never closes => keep the queueMaxSDU
                 * selected by the user
                 */
                if (sched->max_open_gate_duration[tc] == sched->cycle_time) {
                        max_sdu_dynamic = U32_MAX;
                } else {
                        u32 max_frm_len;

                        max_frm_len = duration_to_length(q, sched->max_open_gate_duration[tc]);
                        /* Compensate for L1 overhead from size table,
                         * but don't let the frame size go negative
                         */
                        if (stab) {
                                max_frm_len -= stab->szopts.overhead;
                                max_frm_len = max_t(int, max_frm_len,
                                                    dev->hard_header_len + 1);
                        }
                        max_sdu_dynamic = max_frm_len - dev->hard_header_len;
                        if (max_sdu_dynamic > dev->max_mtu)
                                max_sdu_dynamic = U32_MAX;
                }

                max_sdu = min(max_sdu_dynamic, max_sdu_from_user);

                if (max_sdu != U32_MAX) {
                        sched->max_frm_len[tc] = max_sdu + dev->hard_header_len;
                        sched->max_sdu[tc] = max_sdu;
                } else {
                        sched->max_frm_len[tc] = U32_MAX; /* never oversized */
                        sched->max_sdu[tc] = 0;
                }
        }
}

/* Returns the entry corresponding to next available interval. If
 * validate_interval is set, it only validates whether the timestamp occurs
 * when the gate corresponding to the skb's traffic class is open.
 */
static struct sched_entry *find_entry_to_transmit(struct sk_buff *skb,
                                                  struct Qdisc *sch,
                                                  struct sched_gate_list *sched,
                                                  struct sched_gate_list *admin,
                                                  ktime_t time,
                                                  ktime_t *interval_start,
                                                  ktime_t *interval_end,
                                                  bool validate_interval)
{
        ktime_t curr_intv_start, curr_intv_end, cycle_end, packet_transmit_time;
        ktime_t earliest_txtime = KTIME_MAX, txtime, cycle, transmit_end_time;
        struct sched_entry *entry = NULL, *entry_found = NULL;
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        bool entry_available = false;
        s32 cycle_elapsed;
        int tc, n;

        tc = netdev_get_prio_tc_map(dev, skb->priority);
        packet_transmit_time = length_to_duration(q, qdisc_pkt_len(skb));

        *interval_start = 0;
        *interval_end = 0;

        if (!sched)
                return NULL;

        cycle = sched->cycle_time;
        cycle_elapsed = get_cycle_time_elapsed(sched, time);
        curr_intv_end = ktime_sub_ns(time, cycle_elapsed);
        cycle_end = ktime_add_ns(curr_intv_end, cycle);

        list_for_each_entry(entry, &sched->entries, list) {
                curr_intv_start = curr_intv_end;
                curr_intv_end = get_interval_end_time(sched, admin, entry,
                                                      curr_intv_start);

                if (ktime_after(curr_intv_start, cycle_end))
                        break;

                if (!(entry->gate_mask & BIT(tc)) ||
                    packet_transmit_time > entry->interval)
                        continue;

                txtime = entry->next_txtime;

                if (ktime_before(txtime, time) || validate_interval) {
                        transmit_end_time = ktime_add_ns(time, packet_transmit_time);
                        if ((ktime_before(curr_intv_start, time) &&
                             ktime_before(transmit_end_time, curr_intv_end)) ||
                            (ktime_after(curr_intv_start, time) && !validate_interval)) {
                                entry_found = entry;
                                *interval_start = curr_intv_start;
                                *interval_end = curr_intv_end;
                                break;
                        } else if (!entry_available && !validate_interval) {
                                /* Here, we are just trying to find out the
                                 * first available interval in the next cycle.
                                 */
                                entry_available = true;
                                entry_found = entry;
                                *interval_start = ktime_add_ns(curr_intv_start, cycle);
                                *interval_end = ktime_add_ns(curr_intv_end, cycle);
                        }
                } else if (ktime_before(txtime, earliest_txtime) &&
                           !entry_available) {
                        earliest_txtime = txtime;
                        entry_found = entry;
                        n = div_s64(ktime_sub(txtime, curr_intv_start), cycle);
                        *interval_start = ktime_add(curr_intv_start, n * cycle);
                        *interval_end = ktime_add(curr_intv_end, n * cycle);
                }
        }

        return entry_found;
}

static bool is_valid_interval(struct sk_buff *skb, struct Qdisc *sch)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct sched_gate_list *sched, *admin;
        ktime_t interval_start, interval_end;
        struct sched_entry *entry;

        rcu_read_lock();
        sched = rcu_dereference(q->oper_sched);
        admin = rcu_dereference(q->admin_sched);

        entry = find_entry_to_transmit(skb, sch, sched, admin, skb->tstamp,
                                       &interval_start, &interval_end, true);
        rcu_read_unlock();

        return entry;
}

/* This returns the tstamp value set by TCP in terms of the set clock. */
static ktime_t get_tcp_tstamp(struct taprio_sched *q, struct sk_buff *skb)
{
        unsigned int offset = skb_network_offset(skb);
        const struct ipv6hdr *ipv6h;
        const struct iphdr *iph;
        struct ipv6hdr _ipv6h;

        ipv6h = skb_header_pointer(skb, offset, sizeof(_ipv6h), &_ipv6h);
        if (!ipv6h)
                return 0;

        if (ipv6h->version == 4) {
                iph = (struct iphdr *)ipv6h;
                offset += iph->ihl * 4;

                /* special-case 6in4 tunnelling, as that is a common way to get
                 * v6 connectivity in the home
                 */
                if (iph->protocol == IPPROTO_IPV6) {
                        ipv6h = skb_header_pointer(skb, offset,
                                                   sizeof(_ipv6h), &_ipv6h);

                        if (!ipv6h || ipv6h->nexthdr != IPPROTO_TCP)
                                return 0;
                } else if (iph->protocol != IPPROTO_TCP) {
                        return 0;
                }
        } else if (ipv6h->version == 6 && ipv6h->nexthdr != IPPROTO_TCP) {
                return 0;
        }

        return taprio_mono_to_any(q, skb->skb_mstamp_ns);
}

/* There are a few scenarios where we will have to modify the txtime from
 * what is read from next_txtime in sched_entry. They are:
 * 1. If txtime is in the past,
 *    a. The gate for the traffic class is currently open and packet can be
 *       transmitted before it closes, schedule the packet right away.
 *    b. If the gate corresponding to the traffic class is going to open later
 *       in the cycle, set the txtime of packet to the interval start.
 * 2. If txtime is in the future, there are packets corresponding to the
 *    current traffic class waiting to be transmitted. So, the following
 *    possibilities exist:
 *    a. We can transmit the packet before the window containing the txtime
 *       closes.
 *    b. The window might close before the transmission can be completed
 *       successfully. So, schedule the packet in the next open window.
 */
static long get_packet_txtime(struct sk_buff *skb, struct Qdisc *sch)
{
        ktime_t transmit_end_time, interval_end, interval_start, tcp_tstamp;
        struct taprio_sched *q = qdisc_priv(sch);
        struct sched_gate_list *sched, *admin;
        ktime_t minimum_time, now, txtime;
        int len, packet_transmit_time;
        struct sched_entry *entry;
        bool sched_changed;

        now = taprio_get_time(q);
        minimum_time = ktime_add_ns(now, q->txtime_delay);

        tcp_tstamp = get_tcp_tstamp(q, skb);
        minimum_time = max_t(ktime_t, minimum_time, tcp_tstamp);

        rcu_read_lock();
        admin = rcu_dereference(q->admin_sched);
        sched = rcu_dereference(q->oper_sched);
        if (admin && ktime_after(minimum_time, admin->base_time))
                switch_schedules(q, &admin, &sched);

        /* Until the schedule starts, all the queues are open */
        if (!sched || ktime_before(minimum_time, sched->base_time)) {
                txtime = minimum_time;
                goto done;
        }

        len = qdisc_pkt_len(skb);
        packet_transmit_time = length_to_duration(q, len);

        do {
                sched_changed = false;

                entry = find_entry_to_transmit(skb, sch, sched, admin,
                                               minimum_time,
                                               &interval_start, &interval_end,
                                               false);
                if (!entry) {
                        txtime = 0;
                        goto done;
                }

                txtime = entry->next_txtime;
                txtime = max_t(ktime_t, txtime, minimum_time);
                txtime = max_t(ktime_t, txtime, interval_start);

                if (admin && admin != sched &&
                    ktime_after(txtime, admin->base_time)) {
                        sched = admin;
                        sched_changed = true;
                        continue;
                }

                transmit_end_time = ktime_add(txtime, packet_transmit_time);
                minimum_time = transmit_end_time;

                /* Update the txtime of current entry to the next time it's
                 * interval starts.
                 */
                if (ktime_after(transmit_end_time, interval_end))
                        entry->next_txtime = ktime_add(interval_start, sched->cycle_time);
        } while (sched_changed || ktime_after(transmit_end_time, interval_end));

        entry->next_txtime = transmit_end_time;

done:
        rcu_read_unlock();
        return txtime;
}

/* Devices with full offload are expected to honor this in hardware */
static bool taprio_skb_exceeds_queue_max_sdu(struct Qdisc *sch,
                                             struct sk_buff *skb)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct sched_gate_list *sched;
        int prio = skb->priority;
        bool exceeds = false;
        u8 tc;

        tc = netdev_get_prio_tc_map(dev, prio);

        rcu_read_lock();
        sched = rcu_dereference(q->oper_sched);
        if (sched && skb->len > sched->max_frm_len[tc])
                exceeds = true;
        rcu_read_unlock();

        return exceeds;
}

static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
                              struct Qdisc *child, struct sk_buff **to_free)
{
        struct taprio_sched *q = qdisc_priv(sch);

        /* sk_flags are only safe to use on full sockets. */
        if (skb->sk && sk_fullsock(skb->sk) && sock_flag(skb->sk, SOCK_TXTIME)) {
                if (!is_valid_interval(skb, sch))
                        return qdisc_drop(skb, sch, to_free);
        } else if (TXTIME_ASSIST_IS_ENABLED(q->flags)) {
                skb->tstamp = get_packet_txtime(skb, sch);
                if (!skb->tstamp)
                        return qdisc_drop(skb, sch, to_free);
        }

        qdisc_qstats_backlog_inc(sch, skb);
        sch->q.qlen++;

        return qdisc_enqueue(skb, child, to_free);
}

static int taprio_enqueue_segmented(struct sk_buff *skb, struct Qdisc *sch,
                                    struct Qdisc *child,
                                    struct sk_buff **to_free)
{
        unsigned int slen = 0, numsegs = 0, len = qdisc_pkt_len(skb);
        netdev_features_t features = netif_skb_features(skb);
        struct sk_buff *segs, *nskb;
        int ret;

        segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
        if (IS_ERR_OR_NULL(segs))
                return qdisc_drop(skb, sch, to_free);

        skb_list_walk_safe(segs, segs, nskb) {
                skb_mark_not_on_list(segs);
                qdisc_skb_cb(segs)->pkt_len = segs->len;
                slen += segs->len;

                /* FIXME: we should be segmenting to a smaller size
                 * rather than dropping these
                 */
                if (taprio_skb_exceeds_queue_max_sdu(sch, segs))
                        ret = qdisc_drop(segs, sch, to_free);
                else
                        ret = taprio_enqueue_one(segs, sch, child, to_free);

                if (ret != NET_XMIT_SUCCESS) {
                        if (net_xmit_drop_count(ret))
                                qdisc_qstats_drop(sch);
                } else {
                        numsegs++;
                }
        }

        if (numsegs > 1)
                qdisc_tree_reduce_backlog(sch, 1 - numsegs, len - slen);
        consume_skb(skb);

        return numsegs > 0 ? NET_XMIT_SUCCESS : NET_XMIT_DROP;
}

/* Will not be called in the full offload case, since the TX queues are
 * attached to the Qdisc created using qdisc_create_dflt()
 */
static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                          struct sk_buff **to_free)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct Qdisc *child;
        int queue;

        queue = skb_get_queue_mapping(skb);

        child = q->qdiscs[queue];
        if (unlikely(!child))
                return qdisc_drop(skb, sch, to_free);

        if (taprio_skb_exceeds_queue_max_sdu(sch, skb)) {
                /* Large packets might not be transmitted when the transmission
                 * duration exceeds any configured interval. Therefore, segment
                 * the skb into smaller chunks. Drivers with full offload are
                 * expected to handle this in hardware.
                 */
                if (skb_is_gso(skb))
                        return taprio_enqueue_segmented(skb, sch, child,
                                                        to_free);

                return qdisc_drop(skb, sch, to_free);
        }

        return taprio_enqueue_one(skb, sch, child, to_free);
}

static struct sk_buff *taprio_peek(struct Qdisc *sch)
{
        WARN_ONCE(1, "taprio only supports operating as root qdisc, peek() not implemented");
        return NULL;
}

static void taprio_set_budgets(struct taprio_sched *q,
                               struct sched_gate_list *sched,
                               struct sched_entry *entry)
{
        struct net_device *dev = qdisc_dev(q->root);
        int num_tc = netdev_get_num_tc(dev);
        int tc, budget;

        for (tc = 0; tc < num_tc; tc++) {
                /* Traffic classes which never close have infinite budget */
                if (entry->gate_duration[tc] == sched->cycle_time)
                        budget = INT_MAX;
                else
                        budget = div64_u64((u64)entry->gate_duration[tc] * PSEC_PER_NSEC,
                                           atomic64_read(&q->picos_per_byte));

                atomic_set(&entry->budget[tc], budget);
        }
}

/* When an skb is sent, it consumes from the budget of all traffic classes */
static int taprio_update_budgets(struct sched_entry *entry, size_t len,
                                 int tc_consumed, int num_tc)
{
        int tc, budget, new_budget = 0;

        for (tc = 0; tc < num_tc; tc++) {
                budget = atomic_read(&entry->budget[tc]);
                /* Don't consume from infinite budget */
                if (budget == INT_MAX) {
                        if (tc == tc_consumed)
                                new_budget = budget;
                        continue;
                }

                if (tc == tc_consumed)
                        new_budget = atomic_sub_return(len, &entry->budget[tc]);
                else
                        atomic_sub(len, &entry->budget[tc]);
        }

        return new_budget;
}

static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,
                                               struct sched_entry *entry,
                                               u32 gate_mask)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct Qdisc *child = q->qdiscs[txq];
        int num_tc = netdev_get_num_tc(dev);
        struct sk_buff *skb;
        ktime_t guard;
        int prio;
        int len;
        u8 tc;

        if (unlikely(!child))
                return NULL;

        if (TXTIME_ASSIST_IS_ENABLED(q->flags))
                goto skip_peek_checks;

        skb = child->ops->peek(child);
        if (!skb)
                return NULL;

        prio = skb->priority;
        tc = netdev_get_prio_tc_map(dev, prio);

        if (!(gate_mask & BIT(tc)))
                return NULL;

        len = qdisc_pkt_len(skb);
        guard = ktime_add_ns(taprio_get_time(q), length_to_duration(q, len));

        /* In the case that there's no gate entry, there's no
         * guard band ...
         */
        if (gate_mask != TAPRIO_ALL_GATES_OPEN &&
            !taprio_entry_allows_tx(guard, entry, tc))
                return NULL;

        /* ... and no budget. */
        if (gate_mask != TAPRIO_ALL_GATES_OPEN &&
            taprio_update_budgets(entry, len, tc, num_tc) < 0)
                return NULL;

skip_peek_checks:
        skb = child->ops->dequeue(child);
        if (unlikely(!skb))
                return NULL;

        qdisc_bstats_update(sch, skb);
        qdisc_qstats_backlog_dec(sch, skb);
        sch->q.qlen--;

        return skb;
}

static void taprio_next_tc_txq(struct net_device *dev, int tc, int *txq)
{
        int offset = dev->tc_to_txq[tc].offset;
        int count = dev->tc_to_txq[tc].count;

        (*txq)++;
        if (*txq == offset + count)
                *txq = offset;
}

/* Prioritize higher traffic classes, and select among TXQs belonging to the
 * same TC using round robin
 */
static struct sk_buff *taprio_dequeue_tc_priority(struct Qdisc *sch,
                                                  struct sched_entry *entry,
                                                  u32 gate_mask)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        int num_tc = netdev_get_num_tc(dev);
        struct sk_buff *skb;
        int tc;

        for (tc = num_tc - 1; tc >= 0; tc--) {
                int first_txq = q->cur_txq[tc];

                if (!(gate_mask & BIT(tc)))
                        continue;

                do {
                        skb = taprio_dequeue_from_txq(sch, q->cur_txq[tc],
                                                      entry, gate_mask);

                        taprio_next_tc_txq(dev, tc, &q->cur_txq[tc]);

                        if (q->cur_txq[tc] >= dev->num_tx_queues)
                                q->cur_txq[tc] = first_txq;

                        if (skb)
                                return skb;
                } while (q->cur_txq[tc] != first_txq);
        }

        return NULL;
}

/* Broken way of prioritizing smaller TXQ indices and ignoring the traffic
 * class other than to determine whether the gate is open or not
 */
static struct sk_buff *taprio_dequeue_txq_priority(struct Qdisc *sch,
                                                   struct sched_entry *entry,
                                                   u32 gate_mask)
{
        struct net_device *dev = qdisc_dev(sch);
        struct sk_buff *skb;
        int i;

        for (i = 0; i < dev->num_tx_queues; i++) {
                skb = taprio_dequeue_from_txq(sch, i, entry, gate_mask);
                if (skb)
                        return skb;
        }

        return NULL;
}

/* Will not be called in the full offload case, since the TX queues are
 * attached to the Qdisc created using qdisc_create_dflt()
 */
static struct sk_buff *taprio_dequeue(struct Qdisc *sch)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct sk_buff *skb = NULL;
        struct sched_entry *entry;
        u32 gate_mask;

        rcu_read_lock();
        entry = rcu_dereference(q->current_entry);
        /* if there's no entry, it means that the schedule didn't
         * start yet, so force all gates to be open, this is in
         * accordance to IEEE 802.1Qbv-2015 Section 8.6.9.4.5
         * "AdminGateStates"
         */
        gate_mask = entry ? entry->gate_mask : TAPRIO_ALL_GATES_OPEN;
        if (!gate_mask)
                goto done;

        if (static_branch_unlikely(&taprio_have_broken_mqprio) &&
            !static_branch_likely(&taprio_have_working_mqprio)) {
                /* Single NIC kind which is broken */
                skb = taprio_dequeue_txq_priority(sch, entry, gate_mask);
        } else if (static_branch_likely(&taprio_have_working_mqprio) &&
                   !static_branch_unlikely(&taprio_have_broken_mqprio)) {
                /* Single NIC kind which prioritizes properly */
                skb = taprio_dequeue_tc_priority(sch, entry, gate_mask);
        } else {
                /* Mixed NIC kinds present in system, need dynamic testing */
                if (q->broken_mqprio)
                        skb = taprio_dequeue_txq_priority(sch, entry, gate_mask);
                else
                        skb = taprio_dequeue_tc_priority(sch, entry, gate_mask);
        }

done:
        rcu_read_unlock();

        return skb;
}

static bool should_restart_cycle(const struct sched_gate_list *oper,
                                 const struct sched_entry *entry)
{
        if (list_is_last(&entry->list, &oper->entries))
                return true;

        if (ktime_compare(entry->end_time, oper->cycle_end_time) == 0)
                return true;

        return false;
}

static bool should_change_schedules(const struct sched_gate_list *admin,
                                    const struct sched_gate_list *oper,
                                    ktime_t end_time)
{
        ktime_t next_base_time, extension_time;

        if (!admin)
                return false;

        next_base_time = sched_base_time(admin);

        /* This is the simple case, the end_time would fall after
         * the next schedule base_time.
         */
        if (ktime_compare(next_base_time, end_time) <= 0)
                return true;

        /* This is the cycle_time_extension case, if the end_time
         * plus the amount that can be extended would fall after the
         * next schedule base_time, we can extend the current schedule
         * for that amount.
         */
        extension_time = ktime_add_ns(end_time, oper->cycle_time_extension);

        /* FIXME: the IEEE 802.1Q-2018 Specification isn't clear about
         * how precisely the extension should be made. So after
         * conformance testing, this logic may change.
         */
        if (ktime_compare(next_base_time, extension_time) <= 0)
                return true;

        return false;
}

static enum hrtimer_restart advance_sched(struct hrtimer *timer)
{
        struct taprio_sched *q = container_of(timer, struct taprio_sched,
                                              advance_timer);
        struct net_device *dev = qdisc_dev(q->root);
        struct sched_gate_list *oper, *admin;
        int num_tc = netdev_get_num_tc(dev);
        struct sched_entry *entry, *next;
        struct Qdisc *sch = q->root;
        ktime_t end_time;
        int tc;

        spin_lock(&q->current_entry_lock);
        entry = rcu_dereference_protected(q->current_entry,
                                          lockdep_is_held(&q->current_entry_lock));
        oper = rcu_dereference_protected(q->oper_sched,
                                         lockdep_is_held(&q->current_entry_lock));
        admin = rcu_dereference_protected(q->admin_sched,
                                          lockdep_is_held(&q->current_entry_lock));

        if (!oper)
                switch_schedules(q, &admin, &oper);

        /* This can happen in two cases: 1. this is the very first run
         * of this function (i.e. we weren't running any schedule
         * previously); 2. The previous schedule just ended. The first
         * entry of all schedules are pre-calculated during the
         * schedule initialization.
         */
        if (unlikely(!entry || entry->end_time == oper->base_time)) {
                next = list_first_entry(&oper->entries, struct sched_entry,
                                        list);
                end_time = next->end_time;
                goto first_run;
        }

        if (should_restart_cycle(oper, entry)) {
                next = list_first_entry(&oper->entries, struct sched_entry,
                                        list);
                oper->cycle_end_time = ktime_add_ns(oper->cycle_end_time,
                                                    oper->cycle_time);
        } else {
                next = list_next_entry(entry, list);
        }

        end_time = ktime_add_ns(entry->end_time, next->interval);
        end_time = min_t(ktime_t, end_time, oper->cycle_end_time);

        for (tc = 0; tc < num_tc; tc++) {
                if (next->gate_duration[tc] == oper->cycle_time)
                        next->gate_close_time[tc] = KTIME_MAX;
                else
                        next->gate_close_time[tc] = ktime_add_ns(entry->end_time,
                                                                 next->gate_duration[tc]);
        }

        if (should_change_schedules(admin, oper, end_time)) {
                /* Set things so the next time this runs, the new
                 * schedule runs.
                 */
                end_time = sched_base_time(admin);
                switch_schedules(q, &admin, &oper);
        }

        next->end_time = end_time;
        taprio_set_budgets(q, oper, next);

first_run:
        rcu_assign_pointer(q->current_entry, next);
        spin_unlock(&q->current_entry_lock);

        hrtimer_set_expires(&q->advance_timer, end_time);

        rcu_read_lock();
        __netif_schedule(sch);
        rcu_read_unlock();

        return HRTIMER_RESTART;
}

static const struct nla_policy entry_policy[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = {
        [TCA_TAPRIO_SCHED_ENTRY_INDEX]           = { .type = NLA_U32 },
        [TCA_TAPRIO_SCHED_ENTRY_CMD]           = { .type = NLA_U8 },
        [TCA_TAPRIO_SCHED_ENTRY_GATE_MASK] = { .type = NLA_U32 },
        [TCA_TAPRIO_SCHED_ENTRY_INTERVAL]  = { .type = NLA_U32 },
};

static const struct nla_policy taprio_tc_policy[TCA_TAPRIO_TC_ENTRY_MAX + 1] = {
        [TCA_TAPRIO_TC_ENTRY_INDEX]           = NLA_POLICY_MAX(NLA_U32,
                                                            TC_QOPT_MAX_QUEUE),
        [TCA_TAPRIO_TC_ENTRY_MAX_SDU]           = { .type = NLA_U32 },
        [TCA_TAPRIO_TC_ENTRY_FP]           = NLA_POLICY_RANGE(NLA_U32,
                                                              TC_FP_EXPRESS,
                                                              TC_FP_PREEMPTIBLE),
};

static const struct netlink_range_validation_signed taprio_cycle_time_range = {
        .min = 0,
        .max = INT_MAX,
};

static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = {
        [TCA_TAPRIO_ATTR_PRIOMAP]               = {
                .len = sizeof(struct tc_mqprio_qopt)
        },
        [TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST]           = { .type = NLA_NESTED },
        [TCA_TAPRIO_ATTR_SCHED_BASE_TIME]            = { .type = NLA_S64 },
        [TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY]         = { .type = NLA_NESTED },
        [TCA_TAPRIO_ATTR_SCHED_CLOCKID]              = { .type = NLA_S32 },
        [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]           =
                NLA_POLICY_FULL_RANGE_SIGNED(NLA_S64, &taprio_cycle_time_range),
        [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION] = { .type = NLA_S64 },
        [TCA_TAPRIO_ATTR_FLAGS]                      =
                NLA_POLICY_MASK(NLA_U32, TAPRIO_SUPPORTED_FLAGS),
        [TCA_TAPRIO_ATTR_TXTIME_DELAY]                     = { .type = NLA_U32 },
        [TCA_TAPRIO_ATTR_TC_ENTRY]                     = { .type = NLA_NESTED },
};

static int fill_sched_entry(struct taprio_sched *q, struct nlattr **tb,
                            struct sched_entry *entry,
                            struct netlink_ext_ack *extack)
{
        int min_duration = length_to_duration(q, ETH_ZLEN);
        u32 interval = 0;

        if (tb[TCA_TAPRIO_SCHED_ENTRY_CMD])
                entry->command = nla_get_u8(
                        tb[TCA_TAPRIO_SCHED_ENTRY_CMD]);

        if (tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK])
                entry->gate_mask = nla_get_u32(
                        tb[TCA_TAPRIO_SCHED_ENTRY_GATE_MASK]);

        if (tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL])
                interval = nla_get_u32(
                        tb[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]);

        /* The interval should allow at least the minimum ethernet
         * frame to go out.
         */
        if (interval < min_duration) {
                NL_SET_ERR_MSG(extack, "Invalid interval for schedule entry");
                return -EINVAL;
        }

        entry->interval = interval;

        return 0;
}

static int parse_sched_entry(struct taprio_sched *q, struct nlattr *n,
                             struct sched_entry *entry, int index,
                             struct netlink_ext_ack *extack)
{
        struct nlattr *tb[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { };
        int err;

        err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_SCHED_ENTRY_MAX, n,
                                          entry_policy, NULL);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Could not parse nested entry");
                return -EINVAL;
        }

        entry->index = index;

        return fill_sched_entry(q, tb, entry, extack);
}

static int parse_sched_list(struct taprio_sched *q, struct nlattr *list,
                            struct sched_gate_list *sched,
                            struct netlink_ext_ack *extack)
{
        struct nlattr *n;
        int err, rem;
        int i = 0;

        if (!list)
                return -EINVAL;

        nla_for_each_nested(n, list, rem) {
                struct sched_entry *entry;

                if (nla_type(n) != TCA_TAPRIO_SCHED_ENTRY) {
                        NL_SET_ERR_MSG(extack, "Attribute is not of type 'entry'");
                        continue;
                }

                entry = kzalloc(sizeof(*entry), GFP_KERNEL);
                if (!entry) {
                        NL_SET_ERR_MSG(extack, "Not enough memory for entry");
                        return -ENOMEM;
                }

                err = parse_sched_entry(q, n, entry, i, extack);
                if (err < 0) {
                        kfree(entry);
                        return err;
                }

                list_add_tail(&entry->list, &sched->entries);
                i++;
        }

        sched->num_entries = i;

        return i;
}

static int parse_taprio_schedule(struct taprio_sched *q, struct nlattr **tb,
                                 struct sched_gate_list *new,
                                 struct netlink_ext_ack *extack)
{
        int err = 0;

        if (tb[TCA_TAPRIO_ATTR_SCHED_SINGLE_ENTRY]) {
                NL_SET_ERR_MSG(extack, "Adding a single entry is not supported");
                return -ENOTSUPP;
        }

        if (tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME])
                new->base_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_BASE_TIME]);

        if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION])
                new->cycle_time_extension = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION]);

        if (tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME])
                new->cycle_time = nla_get_s64(tb[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME]);

        if (tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST])
                err = parse_sched_list(q, tb[TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST],
                                       new, extack);
        if (err < 0)
                return err;

        if (!new->cycle_time) {
                struct sched_entry *entry;
                ktime_t cycle = 0;

                list_for_each_entry(entry, &new->entries, list)
                        cycle = ktime_add_ns(cycle, entry->interval);

                if (cycle < 0 || cycle > INT_MAX) {
                        NL_SET_ERR_MSG(extack, "'cycle_time' is too big");
                        return -EINVAL;
                }

                new->cycle_time = cycle;
        }

        if (new->cycle_time < new->num_entries * length_to_duration(q, ETH_ZLEN)) {
                NL_SET_ERR_MSG(extack, "'cycle_time' is too small");
                return -EINVAL;
        }

        taprio_calculate_gate_durations(q, new);

        return 0;
}

static int taprio_parse_mqprio_opt(struct net_device *dev,
                                   struct tc_mqprio_qopt *qopt,
                                   struct netlink_ext_ack *extack,
                                   u32 taprio_flags)
{
        bool allow_overlapping_txqs = TXTIME_ASSIST_IS_ENABLED(taprio_flags);

        if (!qopt) {
                if (!dev->num_tc) {
                        NL_SET_ERR_MSG(extack, "'mqprio' configuration is necessary");
                        return -EINVAL;
                }
                return 0;
        }

        /* taprio imposes that traffic classes map 1:n to tx queues */
        if (qopt->num_tc > dev->num_tx_queues) {
                NL_SET_ERR_MSG(extack, "Number of traffic classes is greater than number of HW queues");
                return -EINVAL;
        }

        /* For some reason, in txtime-assist mode, we allow TXQ ranges for
         * different TCs to overlap, and just validate the TXQ ranges.
         */
        return mqprio_validate_qopt(dev, qopt, true, allow_overlapping_txqs,
                                    extack);
}

static int taprio_get_start_time(struct Qdisc *sch,
                                 struct sched_gate_list *sched,
                                 ktime_t *start)
{
        struct taprio_sched *q = qdisc_priv(sch);
        ktime_t now, base, cycle;
        s64 n;

        base = sched_base_time(sched);
        now = taprio_get_time(q);

        if (ktime_after(base, now)) {
                *start = base;
                return 0;
        }

        cycle = sched->cycle_time;

        /* The qdisc is expected to have at least one sched_entry.  Moreover,
         * any entry must have 'interval' > 0. Thus if the cycle time is zero,
         * something went really wrong. In that case, we should warn about this
         * inconsistent state and return error.
         */
        if (WARN_ON(!cycle))
                return -EFAULT;

        /* Schedule the start time for the beginning of the next
         * cycle.
         */
        n = div64_s64(ktime_sub_ns(now, base), cycle);
        *start = ktime_add_ns(base, (n + 1) * cycle);
        return 0;
}

static void setup_first_end_time(struct taprio_sched *q,
                                 struct sched_gate_list *sched, ktime_t base)
{
        struct net_device *dev = qdisc_dev(q->root);
        int num_tc = netdev_get_num_tc(dev);
        struct sched_entry *first;
        ktime_t cycle;
        int tc;

        first = list_first_entry(&sched->entries,
                                 struct sched_entry, list);

        cycle = sched->cycle_time;

        /* FIXME: find a better place to do this */
        sched->cycle_end_time = ktime_add_ns(base, cycle);

        first->end_time = ktime_add_ns(base, first->interval);
        taprio_set_budgets(q, sched, first);

        for (tc = 0; tc < num_tc; tc++) {
                if (first->gate_duration[tc] == sched->cycle_time)
                        first->gate_close_time[tc] = KTIME_MAX;
                else
                        first->gate_close_time[tc] = ktime_add_ns(base, first->gate_duration[tc]);
        }

        rcu_assign_pointer(q->current_entry, NULL);
}

static void taprio_start_sched(struct Qdisc *sch,
                               ktime_t start, struct sched_gate_list *new)
{
        struct taprio_sched *q = qdisc_priv(sch);
        ktime_t expires;

        if (FULL_OFFLOAD_IS_ENABLED(q->flags))
                return;

        expires = hrtimer_get_expires(&q->advance_timer);
        if (expires == 0)
                expires = KTIME_MAX;

        /* If the new schedule starts before the next expiration, we
         * reprogram it to the earliest one, so we change the admin
         * schedule to the operational one at the right time.
         */
        start = min_t(ktime_t, start, expires);

        hrtimer_start(&q->advance_timer, start, HRTIMER_MODE_ABS);
}

static void taprio_set_picos_per_byte(struct net_device *dev,
                                      struct taprio_sched *q)
{
        struct ethtool_link_ksettings ecmd;
        int speed = SPEED_10;
        int picos_per_byte;
        int err;

        err = __ethtool_get_link_ksettings(dev, &ecmd);
        if (err < 0)
                goto skip;

        if (ecmd.base.speed && ecmd.base.speed != SPEED_UNKNOWN)
                speed = ecmd.base.speed;

skip:
        picos_per_byte = (USEC_PER_SEC * 8) / speed;

        atomic64_set(&q->picos_per_byte, picos_per_byte);
        netdev_dbg(dev, "taprio: set %s's picos_per_byte to: %lld, linkspeed: %d\n",
                   dev->name, (long long)atomic64_read(&q->picos_per_byte),
                   ecmd.base.speed);
}

static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event,
                               void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct sched_gate_list *oper, *admin;
        struct qdisc_size_table *stab;
        struct taprio_sched *q;

        ASSERT_RTNL();

        if (event != NETDEV_UP && event != NETDEV_CHANGE)
                return NOTIFY_DONE;

        list_for_each_entry(q, &taprio_list, taprio_list) {
                if (dev != qdisc_dev(q->root))
                        continue;

                taprio_set_picos_per_byte(dev, q);

                stab = rtnl_dereference(q->root->stab);

                oper = rtnl_dereference(q->oper_sched);
                if (oper)
                        taprio_update_queue_max_sdu(q, oper, stab);

                admin = rtnl_dereference(q->admin_sched);
                if (admin)
                        taprio_update_queue_max_sdu(q, admin, stab);

                break;
        }

        return NOTIFY_DONE;
}

static void setup_txtime(struct taprio_sched *q,
                         struct sched_gate_list *sched, ktime_t base)
{
        struct sched_entry *entry;
        u64 interval = 0;

        list_for_each_entry(entry, &sched->entries, list) {
                entry->next_txtime = ktime_add_ns(base, interval);
                interval += entry->interval;
        }
}

static struct tc_taprio_qopt_offload *taprio_offload_alloc(int num_entries)
{
        struct __tc_taprio_qopt_offload *__offload;

        __offload = kzalloc(struct_size(__offload, offload.entries, num_entries),
                            GFP_KERNEL);
        if (!__offload)
                return NULL;

        refcount_set(&__offload->users, 1);

        return &__offload->offload;
}

struct tc_taprio_qopt_offload *taprio_offload_get(struct tc_taprio_qopt_offload
                                                  *offload)
{
        struct __tc_taprio_qopt_offload *__offload;

        __offload = container_of(offload, struct __tc_taprio_qopt_offload,
                                 offload);

        refcount_inc(&__offload->users);

        return offload;
}
EXPORT_SYMBOL_GPL(taprio_offload_get);

void taprio_offload_free(struct tc_taprio_qopt_offload *offload)
{
        struct __tc_taprio_qopt_offload *__offload;

        __offload = container_of(offload, struct __tc_taprio_qopt_offload,
                                 offload);

        if (!refcount_dec_and_test(&__offload->users))
                return;

        kfree(__offload);
}
EXPORT_SYMBOL_GPL(taprio_offload_free);

/* The function will only serve to keep the pointers to the "oper" and "admin"
 * schedules valid in relation to their base times, so when calling dump() the
 * users looks at the right schedules.
 * When using full offload, the admin configuration is promoted to oper at the
 * base_time in the PHC time domain.  But because the system time is not
 * necessarily in sync with that, we can't just trigger a hrtimer to call
 * switch_schedules at the right hardware time.
 * At the moment we call this by hand right away from taprio, but in the future
 * it will be useful to create a mechanism for drivers to notify taprio of the
 * offload state (PENDING, ACTIVE, INACTIVE) so it can be visible in dump().
 * This is left as TODO.
 */
static void taprio_offload_config_changed(struct taprio_sched *q)
{
        struct sched_gate_list *oper, *admin;

        oper = rtnl_dereference(q->oper_sched);
        admin = rtnl_dereference(q->admin_sched);

        switch_schedules(q, &admin, &oper);
}

static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask)
{
        u32 i, queue_mask = 0;

        for (i = 0; i < dev->num_tc; i++) {
                u32 offset, count;

                if (!(tc_mask & BIT(i)))
                        continue;

                offset = dev->tc_to_txq[i].offset;
                count = dev->tc_to_txq[i].count;

                queue_mask |= GENMASK(offset + count - 1, offset);
        }

        return queue_mask;
}

static void taprio_sched_to_offload(struct net_device *dev,
                                    struct sched_gate_list *sched,
                                    struct tc_taprio_qopt_offload *offload,
                                    const struct tc_taprio_caps *caps)
{
        struct sched_entry *entry;
        int i = 0;

        offload->base_time = sched->base_time;
        offload->cycle_time = sched->cycle_time;
        offload->cycle_time_extension = sched->cycle_time_extension;

        list_for_each_entry(entry, &sched->entries, list) {
                struct tc_taprio_sched_entry *e = &offload->entries[i];

                e->command = entry->command;
                e->interval = entry->interval;
                if (caps->gate_mask_per_txq)
                        e->gate_mask = tc_map_to_queue_mask(dev,
                                                            entry->gate_mask);
                else
                        e->gate_mask = entry->gate_mask;

                i++;
        }

        offload->num_entries = i;
}

static void taprio_detect_broken_mqprio(struct taprio_sched *q)
{
        struct net_device *dev = qdisc_dev(q->root);
        struct tc_taprio_caps caps;

        qdisc_offload_query_caps(dev, TC_SETUP_QDISC_TAPRIO,
                                 &caps, sizeof(caps));

        q->broken_mqprio = caps.broken_mqprio;
        if (q->broken_mqprio)
                static_branch_inc(&taprio_have_broken_mqprio);
        else
                static_branch_inc(&taprio_have_working_mqprio);

        q->detected_mqprio = true;
}

static void taprio_cleanup_broken_mqprio(struct taprio_sched *q)
{
        if (!q->detected_mqprio)
                return;

        if (q->broken_mqprio)
                static_branch_dec(&taprio_have_broken_mqprio);
        else
                static_branch_dec(&taprio_have_working_mqprio);
}

static int taprio_enable_offload(struct net_device *dev,
                                 struct taprio_sched *q,
                                 struct sched_gate_list *sched,
                                 struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct tc_taprio_qopt_offload *offload;
        struct tc_taprio_caps caps;
        int tc, err = 0;

        if (!ops->ndo_setup_tc) {
                NL_SET_ERR_MSG(extack,
                               "Device does not support taprio offload");
                return -EOPNOTSUPP;
        }

        qdisc_offload_query_caps(dev, TC_SETUP_QDISC_TAPRIO,
                                 &caps, sizeof(caps));

        if (!caps.supports_queue_max_sdu) {
                for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
                        if (q->max_sdu[tc]) {
                                NL_SET_ERR_MSG_MOD(extack,
                                                   "Device does not handle queueMaxSDU");
                                return -EOPNOTSUPP;
                        }
                }
        }

        offload = taprio_offload_alloc(sched->num_entries);
        if (!offload) {
                NL_SET_ERR_MSG(extack,
                               "Not enough memory for enabling offload mode");
                return -ENOMEM;
        }
        offload->cmd = TAPRIO_CMD_REPLACE;
        offload->extack = extack;
        mqprio_qopt_reconstruct(dev, &offload->mqprio.qopt);
        offload->mqprio.extack = extack;
        taprio_sched_to_offload(dev, sched, offload, &caps);
        mqprio_fp_to_offload(q->fp, &offload->mqprio);

        for (tc = 0; tc < TC_MAX_QUEUE; tc++)
                offload->max_sdu[tc] = q->max_sdu[tc];

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload);
        if (err < 0) {
                NL_SET_ERR_MSG_WEAK(extack,
                                    "Device failed to setup taprio offload");
                goto done;
        }

        q->offloaded = true;

done:
        /* The offload structure may linger around via a reference taken by the
         * device driver, so clear up the netlink extack pointer so that the
         * driver isn't tempted to dereference data which stopped being valid
         */
        offload->extack = NULL;
        offload->mqprio.extack = NULL;
        taprio_offload_free(offload);

        return err;
}

static int taprio_disable_offload(struct net_device *dev,
                                  struct taprio_sched *q,
                                  struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct tc_taprio_qopt_offload *offload;
        int err;

        if (!q->offloaded)
                return 0;

        offload = taprio_offload_alloc(0);
        if (!offload) {
                NL_SET_ERR_MSG(extack,
                               "Not enough memory to disable offload mode");
                return -ENOMEM;
        }
        offload->cmd = TAPRIO_CMD_DESTROY;

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload);
        if (err < 0) {
                NL_SET_ERR_MSG(extack,
                               "Device failed to disable offload");
                goto out;
        }

        q->offloaded = false;

out:
        taprio_offload_free(offload);

        return err;
}

/* If full offload is enabled, the only possible clockid is the net device's
 * PHC. For that reason, specifying a clockid through netlink is incorrect.
 * For txtime-assist, it is implicitly assumed that the device's PHC is kept
 * in sync with the specified clockid via a user space daemon such as phc2sys.
 * For both software taprio and txtime-assist, the clockid is used for the
 * hrtimer that advances the schedule and hence mandatory.
 */
static int taprio_parse_clockid(struct Qdisc *sch, struct nlattr **tb,
                                struct netlink_ext_ack *extack)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        int err = -EINVAL;

        if (FULL_OFFLOAD_IS_ENABLED(q->flags)) {
                const struct ethtool_ops *ops = dev->ethtool_ops;
                struct kernel_ethtool_ts_info info = {
                        .cmd = ETHTOOL_GET_TS_INFO,
                        .phc_index = -1,
                };

                if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) {
                        NL_SET_ERR_MSG(extack,
                                       "The 'clockid' cannot be specified for full offload");
                        goto out;
                }

                if (ops && ops->get_ts_info)
                        err = ops->get_ts_info(dev, &info);

                if (err || info.phc_index < 0) {
                        NL_SET_ERR_MSG(extack,
                                       "Device does not have a PTP clock");
                        err = -ENOTSUPP;
                        goto out;
                }
        } else if (tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]) {
                int clockid = nla_get_s32(tb[TCA_TAPRIO_ATTR_SCHED_CLOCKID]);
                enum tk_offsets tk_offset;

                /* We only support static clockids and we don't allow
                 * for it to be modified after the first init.
                 */
                if (clockid < 0 ||
                    (q->clockid != -1 && q->clockid != clockid)) {
                        NL_SET_ERR_MSG(extack,
                                       "Changing the 'clockid' of a running schedule is not supported");
                        err = -ENOTSUPP;
                        goto out;
                }

                switch (clockid) {
                case CLOCK_REALTIME:
                        tk_offset = TK_OFFS_REAL;
                        break;
                case CLOCK_MONOTONIC:
                        tk_offset = TK_OFFS_MAX;
                        break;
                case CLOCK_BOOTTIME:
                        tk_offset = TK_OFFS_BOOT;
                        break;
                case CLOCK_TAI:
                        tk_offset = TK_OFFS_TAI;
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Invalid 'clockid'");
                        err = -EINVAL;
                        goto out;
                }
                /* This pairs with READ_ONCE() in taprio_mono_to_any */
                WRITE_ONCE(q->tk_offset, tk_offset);

                q->clockid = clockid;
        } else {
                NL_SET_ERR_MSG(extack, "Specifying a 'clockid' is mandatory");
                goto out;
        }

        /* Everything went ok, return success. */
        err = 0;

out:
        return err;
}

static int taprio_parse_tc_entry(struct Qdisc *sch,
                                 struct nlattr *opt,
                                 u32 max_sdu[TC_QOPT_MAX_QUEUE],
                                 u32 fp[TC_QOPT_MAX_QUEUE],
                                 unsigned long *seen_tcs,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *tb[TCA_TAPRIO_TC_ENTRY_MAX + 1] = { };
        struct net_device *dev = qdisc_dev(sch);
        int err, tc;
        u32 val;

        err = nla_parse_nested(tb, TCA_TAPRIO_TC_ENTRY_MAX, opt,
                               taprio_tc_policy, extack);
        if (err < 0)
                return err;

        if (!tb[TCA_TAPRIO_TC_ENTRY_INDEX]) {
                NL_SET_ERR_MSG_MOD(extack, "TC entry index missing");
                return -EINVAL;
        }

        tc = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_INDEX]);
        if (tc >= TC_QOPT_MAX_QUEUE) {
                NL_SET_ERR_MSG_MOD(extack, "TC entry index out of range");
                return -ERANGE;
        }

        if (*seen_tcs & BIT(tc)) {
                NL_SET_ERR_MSG_MOD(extack, "Duplicate TC entry");
                return -EINVAL;
        }

        *seen_tcs |= BIT(tc);

        if (tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU]) {
                val = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU]);
                if (val > dev->max_mtu) {
                        NL_SET_ERR_MSG_MOD(extack, "TC max SDU exceeds device max MTU");
                        return -ERANGE;
                }

                max_sdu[tc] = val;
        }

        if (tb[TCA_TAPRIO_TC_ENTRY_FP])
                fp[tc] = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_FP]);

        return 0;
}

static int taprio_parse_tc_entries(struct Qdisc *sch,
                                   struct nlattr *opt,
                                   struct netlink_ext_ack *extack)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        u32 max_sdu[TC_QOPT_MAX_QUEUE];
        bool have_preemption = false;
        unsigned long seen_tcs = 0;
        u32 fp[TC_QOPT_MAX_QUEUE];
        struct nlattr *n;
        int tc, rem;
        int err = 0;

        for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) {
                max_sdu[tc] = q->max_sdu[tc];
                fp[tc] = q->fp[tc];
        }

        nla_for_each_nested_type(n, TCA_TAPRIO_ATTR_TC_ENTRY, opt, rem) {
                err = taprio_parse_tc_entry(sch, n, max_sdu, fp, &seen_tcs,
                                            extack);
                if (err)
                        return err;
        }

        for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) {
                q->max_sdu[tc] = max_sdu[tc];
                q->fp[tc] = fp[tc];
                if (fp[tc] != TC_FP_EXPRESS)
                        have_preemption = true;
        }

        if (have_preemption) {
                if (!FULL_OFFLOAD_IS_ENABLED(q->flags)) {
                        NL_SET_ERR_MSG(extack,
                                       "Preemption only supported with full offload");
                        return -EOPNOTSUPP;
                }

                if (!ethtool_dev_mm_supported(dev)) {
                        NL_SET_ERR_MSG(extack,
                                       "Device does not support preemption");
                        return -EOPNOTSUPP;
                }
        }

        return err;
}

static int taprio_mqprio_cmp(const struct net_device *dev,
                             const struct tc_mqprio_qopt *mqprio)
{
        int i;

        if (!mqprio || mqprio->num_tc != dev->num_tc)
                return -1;

        for (i = 0; i < mqprio->num_tc; i++)
                if (dev->tc_to_txq[i].count != mqprio->count[i] ||
                    dev->tc_to_txq[i].offset != mqprio->offset[i])
                        return -1;

        for (i = 0; i <= TC_BITMASK; i++)
                if (dev->prio_tc_map[i] != mqprio->prio_tc_map[i])
                        return -1;

        return 0;
}

static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
                         struct netlink_ext_ack *extack)
{
        struct qdisc_size_table *stab = rtnl_dereference(sch->stab);
        struct nlattr *tb[TCA_TAPRIO_ATTR_MAX + 1] = { };
        struct sched_gate_list *oper, *admin, *new_admin;
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct tc_mqprio_qopt *mqprio = NULL;
        unsigned long flags;
        u32 taprio_flags;
        ktime_t start;
        int i, err;

        err = nla_parse_nested_deprecated(tb, TCA_TAPRIO_ATTR_MAX, opt,
                                          taprio_policy, extack);
        if (err < 0)
                return err;

        if (tb[TCA_TAPRIO_ATTR_PRIOMAP])
                mqprio = nla_data(tb[TCA_TAPRIO_ATTR_PRIOMAP]);

        /* The semantics of the 'flags' argument in relation to 'change()'
         * requests, are interpreted following two rules (which are applied in
         * this order): (1) an omitted 'flags' argument is interpreted as
         * zero; (2) the 'flags' of a "running" taprio instance cannot be
         * changed.
         */
        taprio_flags = nla_get_u32_default(tb[TCA_TAPRIO_ATTR_FLAGS], 0);

        /* txtime-assist and full offload are mutually exclusive */
        if ((taprio_flags & TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST) &&
            (taprio_flags & TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD)) {
                NL_SET_ERR_MSG_ATTR(extack, tb[TCA_TAPRIO_ATTR_FLAGS],
                                    "TXTIME_ASSIST and FULL_OFFLOAD are mutually exclusive");
                return -EINVAL;
        }

        if (q->flags != TAPRIO_FLAGS_INVALID && q->flags != taprio_flags) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Changing 'flags' of a running schedule is not supported");
                return -EOPNOTSUPP;
        }
        q->flags = taprio_flags;

        /* Needed for length_to_duration() during netlink attribute parsing */
        taprio_set_picos_per_byte(dev, q);

        err = taprio_parse_mqprio_opt(dev, mqprio, extack, q->flags);
        if (err < 0)
                return err;

        err = taprio_parse_tc_entries(sch, opt, extack);
        if (err)
                return err;

        new_admin = kzalloc(sizeof(*new_admin), GFP_KERNEL);
        if (!new_admin) {
                NL_SET_ERR_MSG(extack, "Not enough memory for a new schedule");
                return -ENOMEM;
        }
        INIT_LIST_HEAD(&new_admin->entries);

        oper = rtnl_dereference(q->oper_sched);
        admin = rtnl_dereference(q->admin_sched);

        /* no changes - no new mqprio settings */
        if (!taprio_mqprio_cmp(dev, mqprio))
                mqprio = NULL;

        if (mqprio && (oper || admin)) {
                NL_SET_ERR_MSG(extack, "Changing the traffic mapping of a running schedule is not supported");
                err = -ENOTSUPP;
                goto free_sched;
        }

        if (mqprio) {
                err = netdev_set_num_tc(dev, mqprio->num_tc);
                if (err)
                        goto free_sched;
                for (i = 0; i < mqprio->num_tc; i++) {
                        netdev_set_tc_queue(dev, i,
                                            mqprio->count[i],
                                            mqprio->offset[i]);
                        q->cur_txq[i] = mqprio->offset[i];
                }

                /* Always use supplied priority mappings */
                for (i = 0; i <= TC_BITMASK; i++)
                        netdev_set_prio_tc_map(dev, i,
                                               mqprio->prio_tc_map[i]);
        }

        err = parse_taprio_schedule(q, tb, new_admin, extack);
        if (err < 0)
                goto free_sched;

        if (new_admin->num_entries == 0) {
                NL_SET_ERR_MSG(extack, "There should be at least one entry in the schedule");
                err = -EINVAL;
                goto free_sched;
        }

        err = taprio_parse_clockid(sch, tb, extack);
        if (err < 0)
                goto free_sched;

        taprio_update_queue_max_sdu(q, new_admin, stab);

        if (FULL_OFFLOAD_IS_ENABLED(q->flags))
                err = taprio_enable_offload(dev, q, new_admin, extack);
        else
                err = taprio_disable_offload(dev, q, extack);
        if (err)
                goto free_sched;

        /* Protects against enqueue()/dequeue() */
        spin_lock_bh(qdisc_lock(sch));

        if (tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]) {
                if (!TXTIME_ASSIST_IS_ENABLED(q->flags)) {
                        NL_SET_ERR_MSG_MOD(extack, "txtime-delay can only be set when txtime-assist mode is enabled");
                        err = -EINVAL;
                        goto unlock;
                }

                q->txtime_delay = nla_get_u32(tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]);
        }

        if (!TXTIME_ASSIST_IS_ENABLED(q->flags) &&
            !FULL_OFFLOAD_IS_ENABLED(q->flags) &&
            !hrtimer_active(&q->advance_timer)) {
                hrtimer_init(&q->advance_timer, q->clockid, HRTIMER_MODE_ABS);
                q->advance_timer.function = advance_sched;
        }

        err = taprio_get_start_time(sch, new_admin, &start);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Internal error: failed get start time");
                goto unlock;
        }

        setup_txtime(q, new_admin, start);

        if (TXTIME_ASSIST_IS_ENABLED(q->flags)) {
                if (!oper) {
                        rcu_assign_pointer(q->oper_sched, new_admin);
                        err = 0;
                        new_admin = NULL;
                        goto unlock;
                }

                /* Not going to race against advance_sched(), but still */
                admin = rcu_replace_pointer(q->admin_sched, new_admin,
                                            lockdep_rtnl_is_held());
                if (admin)
                        call_rcu(&admin->rcu, taprio_free_sched_cb);
        } else {
                setup_first_end_time(q, new_admin, start);

                /* Protects against advance_sched() */
                spin_lock_irqsave(&q->current_entry_lock, flags);

                taprio_start_sched(sch, start, new_admin);

                admin = rcu_replace_pointer(q->admin_sched, new_admin,
                                            lockdep_rtnl_is_held());
                if (admin)
                        call_rcu(&admin->rcu, taprio_free_sched_cb);

                spin_unlock_irqrestore(&q->current_entry_lock, flags);

                if (FULL_OFFLOAD_IS_ENABLED(q->flags))
                        taprio_offload_config_changed(q);
        }

        new_admin = NULL;
        err = 0;

        if (!stab)
                NL_SET_ERR_MSG_MOD(extack,
                                   "Size table not specified, frame length estimations may be inaccurate");

unlock:
        spin_unlock_bh(qdisc_lock(sch));

free_sched:
        if (new_admin)
                call_rcu(&new_admin->rcu, taprio_free_sched_cb);

        return err;
}

static void taprio_reset(struct Qdisc *sch)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        int i;

        hrtimer_cancel(&q->advance_timer);

        if (q->qdiscs) {
                for (i = 0; i < dev->num_tx_queues; i++)
                        if (q->qdiscs[i])
                                qdisc_reset(q->qdiscs[i]);
        }
}

static void taprio_destroy(struct Qdisc *sch)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct sched_gate_list *oper, *admin;
        unsigned int i;

        list_del(&q->taprio_list);

        /* Note that taprio_reset() might not be called if an error
         * happens in qdisc_create(), after taprio_init() has been called.
         */
        hrtimer_cancel(&q->advance_timer);
        qdisc_synchronize(sch);

        taprio_disable_offload(dev, q, NULL);

        if (q->qdiscs) {
                for (i = 0; i < dev->num_tx_queues; i++)
                        qdisc_put(q->qdiscs[i]);

                kfree(q->qdiscs);
        }
        q->qdiscs = NULL;

        netdev_reset_tc(dev);

        oper = rtnl_dereference(q->oper_sched);
        admin = rtnl_dereference(q->admin_sched);

        if (oper)
                call_rcu(&oper->rcu, taprio_free_sched_cb);

        if (admin)
                call_rcu(&admin->rcu, taprio_free_sched_cb);

        taprio_cleanup_broken_mqprio(q);
}

static int taprio_init(struct Qdisc *sch, struct nlattr *opt,
                       struct netlink_ext_ack *extack)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        int i, tc;

        spin_lock_init(&q->current_entry_lock);

        hrtimer_init(&q->advance_timer, CLOCK_TAI, HRTIMER_MODE_ABS);
        q->advance_timer.function = advance_sched;

        q->root = sch;

        /* We only support static clockids. Use an invalid value as default
         * and get the valid one on taprio_change().
         */
        q->clockid = -1;
        q->flags = TAPRIO_FLAGS_INVALID;

        list_add(&q->taprio_list, &taprio_list);

        if (sch->parent != TC_H_ROOT) {
                NL_SET_ERR_MSG_MOD(extack, "Can only be attached as root qdisc");
                return -EOPNOTSUPP;
        }

        if (!netif_is_multiqueue(dev)) {
                NL_SET_ERR_MSG_MOD(extack, "Multi-queue device is required");
                return -EOPNOTSUPP;
        }

        q->qdiscs = kcalloc(dev->num_tx_queues, sizeof(q->qdiscs[0]),
                            GFP_KERNEL);
        if (!q->qdiscs)
                return -ENOMEM;

        if (!opt)
                return -EINVAL;

        for (i = 0; i < dev->num_tx_queues; i++) {
                struct netdev_queue *dev_queue;
                struct Qdisc *qdisc;

                dev_queue = netdev_get_tx_queue(dev, i);
                qdisc = qdisc_create_dflt(dev_queue,
                                          &pfifo_qdisc_ops,
                                          TC_H_MAKE(TC_H_MAJ(sch->handle),
                                                    TC_H_MIN(i + 1)),
                                          extack);
                if (!qdisc)
                        return -ENOMEM;

                if (i < dev->real_num_tx_queues)
                        qdisc_hash_add(qdisc, false);

                q->qdiscs[i] = qdisc;
        }

        for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++)
                q->fp[tc] = TC_FP_EXPRESS;

        taprio_detect_broken_mqprio(q);

        return taprio_change(sch, opt, extack);
}

static void taprio_attach(struct Qdisc *sch)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        unsigned int ntx;

        /* Attach underlying qdisc */
        for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
                struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, ntx);
                struct Qdisc *old, *dev_queue_qdisc;

                if (FULL_OFFLOAD_IS_ENABLED(q->flags)) {
                        struct Qdisc *qdisc = q->qdiscs[ntx];

                        /* In offload mode, the root taprio qdisc is bypassed
                         * and the netdev TX queues see the children directly
                         */
                        qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
                        dev_queue_qdisc = qdisc;
                } else {
                        /* In software mode, attach the root taprio qdisc
                         * to all netdev TX queues, so that dev_qdisc_enqueue()
                         * goes through taprio_enqueue().
                         */
                        dev_queue_qdisc = sch;
                }
                old = dev_graft_qdisc(dev_queue, dev_queue_qdisc);
                /* The qdisc's refcount requires to be elevated once
                 * for each netdev TX queue it is grafted onto
                 */
                qdisc_refcount_inc(dev_queue_qdisc);
                if (old)
                        qdisc_put(old);
        }
}

static struct netdev_queue *taprio_queue_get(struct Qdisc *sch,
                                             unsigned long cl)
{
        struct net_device *dev = qdisc_dev(sch);
        unsigned long ntx = cl - 1;

        if (ntx >= dev->num_tx_queues)
                return NULL;

        return netdev_get_tx_queue(dev, ntx);
}

static int taprio_graft(struct Qdisc *sch, unsigned long cl,
                        struct Qdisc *new, struct Qdisc **old,
                        struct netlink_ext_ack *extack)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct netdev_queue *dev_queue = taprio_queue_get(sch, cl);

        if (!dev_queue)
                return -EINVAL;

        if (dev->flags & IFF_UP)
                dev_deactivate(dev);

        /* In offload mode, the child Qdisc is directly attached to the netdev
         * TX queue, and thus, we need to keep its refcount elevated in order
         * to counteract qdisc_graft()'s call to qdisc_put() once per TX queue.
         * However, save the reference to the new qdisc in the private array in
         * both software and offload cases, to have an up-to-date reference to
         * our children.
         */
        *old = q->qdiscs[cl - 1];
        if (FULL_OFFLOAD_IS_ENABLED(q->flags)) {
                WARN_ON_ONCE(dev_graft_qdisc(dev_queue, new) != *old);
                if (new)
                        qdisc_refcount_inc(new);
                if (*old)
                        qdisc_put(*old);
        }

        q->qdiscs[cl - 1] = new;
        if (new)
                new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;

        if (dev->flags & IFF_UP)
                dev_activate(dev);

        return 0;
}

static int dump_entry(struct sk_buff *msg,
                      const struct sched_entry *entry)
{
        struct nlattr *item;

        item = nla_nest_start_noflag(msg, TCA_TAPRIO_SCHED_ENTRY);
        if (!item)
                return -ENOSPC;

        if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INDEX, entry->index))
                goto nla_put_failure;

        if (nla_put_u8(msg, TCA_TAPRIO_SCHED_ENTRY_CMD, entry->command))
                goto nla_put_failure;

        if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_GATE_MASK,
                        entry->gate_mask))
                goto nla_put_failure;

        if (nla_put_u32(msg, TCA_TAPRIO_SCHED_ENTRY_INTERVAL,
                        entry->interval))
                goto nla_put_failure;

        return nla_nest_end(msg, item);

nla_put_failure:
        nla_nest_cancel(msg, item);
        return -1;
}

static int dump_schedule(struct sk_buff *msg,
                         const struct sched_gate_list *root)
{
        struct nlattr *entry_list;
        struct sched_entry *entry;

        if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_BASE_TIME,
                        root->base_time, TCA_TAPRIO_PAD))
                return -1;

        if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME,
                        root->cycle_time, TCA_TAPRIO_PAD))
                return -1;

        if (nla_put_s64(msg, TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION,
                        root->cycle_time_extension, TCA_TAPRIO_PAD))
                return -1;

        entry_list = nla_nest_start_noflag(msg,
                                           TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST);
        if (!entry_list)
                goto error_nest;

        list_for_each_entry(entry, &root->entries, list) {
                if (dump_entry(msg, entry) < 0)
                        goto error_nest;
        }

        nla_nest_end(msg, entry_list);
        return 0;

error_nest:
        nla_nest_cancel(msg, entry_list);
        return -1;
}

static int taprio_dump_tc_entries(struct sk_buff *skb,
                                  struct taprio_sched *q,
                                  struct sched_gate_list *sched)
{
        struct nlattr *n;
        int tc;

        for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
                n = nla_nest_start(skb, TCA_TAPRIO_ATTR_TC_ENTRY);
                if (!n)
                        return -EMSGSIZE;

                if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_INDEX, tc))
                        goto nla_put_failure;

                if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_MAX_SDU,
                                sched->max_sdu[tc]))
                        goto nla_put_failure;

                if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_FP, q->fp[tc]))
                        goto nla_put_failure;

                nla_nest_end(skb, n);
        }

        return 0;

nla_put_failure:
        nla_nest_cancel(skb, n);
        return -EMSGSIZE;
}

static int taprio_put_stat(struct sk_buff *skb, u64 val, u16 attrtype)
{
        if (val == TAPRIO_STAT_NOT_SET)
                return 0;
        if (nla_put_u64_64bit(skb, attrtype, val, TCA_TAPRIO_OFFLOAD_STATS_PAD))
                return -EMSGSIZE;
        return 0;
}

static int taprio_dump_xstats(struct Qdisc *sch, struct gnet_dump *d,
                              struct tc_taprio_qopt_offload *offload,
                              struct tc_taprio_qopt_stats *stats)
{
        struct net_device *dev = qdisc_dev(sch);
        const struct net_device_ops *ops;
        struct sk_buff *skb = d->skb;
        struct nlattr *xstats;
        int err;

        ops = qdisc_dev(sch)->netdev_ops;

        /* FIXME I could use qdisc_offload_dump_helper(), but that messes
         * with sch->flags depending on whether the device reports taprio
         * stats, and I'm not sure whether that's a good idea, considering
         * that stats are optional to the offload itself
         */
        if (!ops->ndo_setup_tc)
                return 0;

        memset(stats, 0xff, sizeof(*stats));

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload);
        if (err == -EOPNOTSUPP)
                return 0;
        if (err)
                return err;

        xstats = nla_nest_start(skb, TCA_STATS_APP);
        if (!xstats)
                goto err;

        if (taprio_put_stat(skb, stats->window_drops,
                            TCA_TAPRIO_OFFLOAD_STATS_WINDOW_DROPS) ||
            taprio_put_stat(skb, stats->tx_overruns,
                            TCA_TAPRIO_OFFLOAD_STATS_TX_OVERRUNS))
                goto err_cancel;

        nla_nest_end(skb, xstats);

        return 0;

err_cancel:
        nla_nest_cancel(skb, xstats);
err:
        return -EMSGSIZE;
}

static int taprio_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
        struct tc_taprio_qopt_offload offload = {
                .cmd = TAPRIO_CMD_STATS,
        };

        return taprio_dump_xstats(sch, d, &offload, &offload.stats);
}

static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct sched_gate_list *oper, *admin;
        struct tc_mqprio_qopt opt = { 0 };
        struct nlattr *nest, *sched_nest;

        mqprio_qopt_reconstruct(dev, &opt);

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (!nest)
                goto start_error;

        if (nla_put(skb, TCA_TAPRIO_ATTR_PRIOMAP, sizeof(opt), &opt))
                goto options_error;

        if (!FULL_OFFLOAD_IS_ENABLED(q->flags) &&
            nla_put_s32(skb, TCA_TAPRIO_ATTR_SCHED_CLOCKID, q->clockid))
                goto options_error;

        if (q->flags && nla_put_u32(skb, TCA_TAPRIO_ATTR_FLAGS, q->flags))
                goto options_error;

        if (q->txtime_delay &&
            nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay))
                goto options_error;

        rcu_read_lock();

        oper = rtnl_dereference(q->oper_sched);
        admin = rtnl_dereference(q->admin_sched);

        if (oper && taprio_dump_tc_entries(skb, q, oper))
                goto options_error_rcu;

        if (oper && dump_schedule(skb, oper))
                goto options_error_rcu;

        if (!admin)
                goto done;

        sched_nest = nla_nest_start_noflag(skb, TCA_TAPRIO_ATTR_ADMIN_SCHED);
        if (!sched_nest)
                goto options_error_rcu;

        if (dump_schedule(skb, admin))
                goto admin_error;

        nla_nest_end(skb, sched_nest);

done:
        rcu_read_unlock();
        return nla_nest_end(skb, nest);

admin_error:
        nla_nest_cancel(skb, sched_nest);

options_error_rcu:
        rcu_read_unlock();

options_error:
        nla_nest_cancel(skb, nest);

start_error:
        return -ENOSPC;
}

static struct Qdisc *taprio_leaf(struct Qdisc *sch, unsigned long cl)
{
        struct taprio_sched *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        unsigned int ntx = cl - 1;

        if (ntx >= dev->num_tx_queues)
                return NULL;

        return q->qdiscs[ntx];
}

static unsigned long taprio_find(struct Qdisc *sch, u32 classid)
{
        unsigned int ntx = TC_H_MIN(classid);

        if (!taprio_queue_get(sch, ntx))
                return 0;
        return ntx;
}

static int taprio_dump_class(struct Qdisc *sch, unsigned long cl,
                             struct sk_buff *skb, struct tcmsg *tcm)
{
        struct Qdisc *child = taprio_leaf(sch, cl);

        tcm->tcm_parent = TC_H_ROOT;
        tcm->tcm_handle |= TC_H_MIN(cl);
        tcm->tcm_info = child->handle;

        return 0;
}

static int taprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                   struct gnet_dump *d)
        __releases(d->lock)
        __acquires(d->lock)
{
        struct Qdisc *child = taprio_leaf(sch, cl);
        struct tc_taprio_qopt_offload offload = {
                .cmd = TAPRIO_CMD_QUEUE_STATS,
                .queue_stats = {
                        .queue = cl - 1,
                },
        };

        if (gnet_stats_copy_basic(d, NULL, &child->bstats, true) < 0 ||
            qdisc_qstats_copy(d, child) < 0)
                return -1;

        return taprio_dump_xstats(sch, d, &offload, &offload.queue_stats.stats);
}

static void taprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
        struct net_device *dev = qdisc_dev(sch);
        unsigned long ntx;

        if (arg->stop)
                return;

        arg->count = arg->skip;
        for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) {
                if (!tc_qdisc_stats_dump(sch, ntx + 1, arg))
                        break;
        }
}

static struct netdev_queue *taprio_select_queue(struct Qdisc *sch,
                                                struct tcmsg *tcm)
{
        return taprio_queue_get(sch, TC_H_MIN(tcm->tcm_parent));
}

static const struct Qdisc_class_ops taprio_class_ops = {
        .graft                = taprio_graft,
        .leaf                = taprio_leaf,
        .find                = taprio_find,
        .walk                = taprio_walk,
        .dump                = taprio_dump_class,
        .dump_stats        = taprio_dump_class_stats,
        .select_queue        = taprio_select_queue,
};

static struct Qdisc_ops taprio_qdisc_ops __read_mostly = {
        .cl_ops                = &taprio_class_ops,
        .id                = "taprio",
        .priv_size        = sizeof(struct taprio_sched),
        .init                = taprio_init,
        .change                = taprio_change,
        .destroy        = taprio_destroy,
        .reset                = taprio_reset,
        .attach                = taprio_attach,
        .peek                = taprio_peek,
        .dequeue        = taprio_dequeue,
        .enqueue        = taprio_enqueue,
        .dump                = taprio_dump,
        .dump_stats        = taprio_dump_stats,
        .owner                = THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("taprio");

static struct notifier_block taprio_device_notifier = {
        .notifier_call = taprio_dev_notifier,
};

static int __init taprio_module_init(void)
{
        int err = register_netdevice_notifier(&taprio_device_notifier);

        if (err)
                return err;

        return register_qdisc(&taprio_qdisc_ops);
}

static void __exit taprio_module_exit(void)
{
        unregister_qdisc(&taprio_qdisc_ops);
        unregister_netdevice_notifier(&taprio_device_notifier);
}

module_init(taprio_module_init);
module_exit(taprio_module_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Time Aware Priority qdisc");






















































































































































































































































































































































































  131 





























    9 














































































































































































































































































































































































































































































































































































































































































































































 2729 




















  724 


  711 












  277 
    5 














  192 
  197 









  735 



























    1 













 3335 



 3298 













 2541 

 2533 
 2235 
 2437 











  748 








  643 










    1 







































 2328 


































    3 
    3 
    2 












  141 













   20 



























































































   14 




   10 


   10 





    1 








    1 












   18 





















































  157 





































   60 
   38 

   69 




  255 
  185 





  104 





















   27 








 2676 









   16 






 3503 




 3131 




















































   57 




 3396 











  513 























































   37 










    4 













  513 






  258 




  271 





  130 






















  165 











   42 













   99 












   31 















































  191 

















  154 
  155 






    4 
    1 













    2 

   15 















   18 


   34 
   29 







    1 













    2 













  663 


















   63 
    7 


    7 































    4 
    4 



    4 






















  715 

  229 



























   10 

    6 



















  151 















  164 











   10 














  210 













   36 























   28 



   23 

























    3 















   20 













    5 
    5 















   66 
   84 






























  627 
































  627 












  995 

















  616 
  651 
  550 














   79 
   27 
   27 







 2838 




 2633 







    3 





   61 










































   69 

















   61 











  186 





  186 
  186 






   50 





  136 
  135 



















  136 










































 3243 




   11 




   11 










































 1174 

 1184 




























   19 



    3 





    5 














  517 

  599 

   11 





 2071 

 2783 








 2786 




 2619 









   25 


 2310 


   25 


    3 







 2293 




   46 


   46 













  649 










 2124 



























  751 

































   11 






   33 





  126 






  253 




  159 



  176 






    1 











    5 






    5 
    5 




   34 














   34 




























  598 




  537 
  640 




   26 

  242 
  594 





  264 
  327 




  791 




   34 

  492 
 2757 




   45 
  410 




  761 
  772 










    9 









  433 

   19 
 2799 

















   64 
   65 

  159 

   54 





























  242 




   13 




  169 




    4 
  277 









  363 




   34 





  190 




   25 






















































  313 

  319 





  319 






   17 
   16 
    2 
   18 





   33 









































 1162 
  562 

   34 

 1172 














 2491 









  387 


















  184 





  169 







    3 
















































































  117 


































































































   50 









   79 









   12 


























   29 


























   29 


  178 





















































































   26 




















    5 
    5 














   54 



   16 

































   80 






































    2 


    2 
















    2 
























   19 


   91 

  129 
   72 

   72 
   16 






    2 











   14 











   11 
    2 











































   46 
    2 





























































   16 

   17 







































































  185 


































   54 



















































 2101 




   19 















  193 


   66 
  301 

   46 


  193 





   88 
   81 























 2675 

   84 



































    1 














































   19 











  504 

   20 
    1 

  365 





















  472 




















  511 


  389 




























    7 




































































































































  555 
   14 
















  125 
    2 



























































































































































































































 3336 








  691 



























































 3313 






 2561 


   27 

   27 
   27 






  104 




   18 




 2684 










 2684 
   15 









   60 
   28 




















  160 
  158 






















 2505 




  104 








  102 
  104 





   68 


































 2679 




   38 














   13 




  116 




   86 




  681 





 2675 







 2699 











    3 





































    2 












    2 


































   12 












































   94 














  242 










































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *        Definitions for the 'struct sk_buff' memory handlers.
 *
 *        Authors:
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *                Florian La Roche, <rzsfl@rz.uni-sb.de>
 */

#ifndef _LINUX_SKBUFF_H
#define _LINUX_SKBUFF_H

#include <linux/kernel.h>
#include <linux/compiler.h>
#include <linux/time.h>
#include <linux/bug.h>
#include <linux/bvec.h>
#include <linux/cache.h>
#include <linux/rbtree.h>
#include <linux/socket.h>
#include <linux/refcount.h>

#include <linux/atomic.h>
#include <asm/types.h>
#include <linux/spinlock.h>
#include <net/checksum.h>
#include <linux/rcupdate.h>
#include <linux/dma-mapping.h>
#include <linux/netdev_features.h>
#include <net/flow_dissector.h>
#include <linux/in6.h>
#include <linux/if_packet.h>
#include <linux/llist.h>
#include <linux/page_frag_cache.h>
#include <net/flow.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <linux/netfilter/nf_conntrack_common.h>
#endif
#include <net/net_debug.h>
#include <net/dropreason-core.h>
#include <net/netmem.h>

/**
 * DOC: skb checksums
 *
 * The interface for checksum offload between the stack and networking drivers
 * is as follows...
 *
 * IP checksum related features
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * Drivers advertise checksum offload capabilities in the features of a device.
 * From the stack's point of view these are capabilities offered by the driver.
 * A driver typically only advertises features that it is capable of offloading
 * to its device.
 *
 * .. flat-table:: Checksum related device features
 *   :widths: 1 10
 *
 *   * - %NETIF_F_HW_CSUM
 *     - The driver (or its device) is able to compute one
 *         IP (one's complement) checksum for any combination
 *         of protocols or protocol layering. The checksum is
 *         computed and set in a packet per the CHECKSUM_PARTIAL
 *         interface (see below).
 *
 *   * - %NETIF_F_IP_CSUM
 *     - Driver (device) is only able to checksum plain
 *         TCP or UDP packets over IPv4. These are specifically
 *         unencapsulated packets of the form IPv4|TCP or
 *         IPv4|UDP where the Protocol field in the IPv4 header
 *         is TCP or UDP. The IPv4 header may contain IP options.
 *         This feature cannot be set in features for a device
 *         with NETIF_F_HW_CSUM also set. This feature is being
 *         DEPRECATED (see below).
 *
 *   * - %NETIF_F_IPV6_CSUM
 *     - Driver (device) is only able to checksum plain
 *         TCP or UDP packets over IPv6. These are specifically
 *         unencapsulated packets of the form IPv6|TCP or
 *         IPv6|UDP where the Next Header field in the IPv6
 *         header is either TCP or UDP. IPv6 extension headers
 *         are not supported with this feature. This feature
 *         cannot be set in features for a device with
 *         NETIF_F_HW_CSUM also set. This feature is being
 *         DEPRECATED (see below).
 *
 *   * - %NETIF_F_RXCSUM
 *     - Driver (device) performs receive checksum offload.
 *         This flag is only used to disable the RX checksum
 *         feature for a device. The stack will accept receive
 *         checksum indication in packets received on a device
 *         regardless of whether NETIF_F_RXCSUM is set.
 *
 * Checksumming of received packets by device
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * Indication of checksum verification is set in &sk_buff.ip_summed.
 * Possible values are:
 *
 * - %CHECKSUM_NONE
 *
 *   Device did not checksum this packet e.g. due to lack of capabilities.
 *   The packet contains full (though not verified) checksum in packet but
 *   not in skb->csum. Thus, skb->csum is undefined in this case.
 *
 * - %CHECKSUM_UNNECESSARY
 *
 *   The hardware you're dealing with doesn't calculate the full checksum
 *   (as in %CHECKSUM_COMPLETE), but it does parse headers and verify checksums
 *   for specific protocols. For such packets it will set %CHECKSUM_UNNECESSARY
 *   if their checksums are okay. &sk_buff.csum is still undefined in this case
 *   though. A driver or device must never modify the checksum field in the
 *   packet even if checksum is verified.
 *
 *   %CHECKSUM_UNNECESSARY is applicable to following protocols:
 *
 *     - TCP: IPv6 and IPv4.
 *     - UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
 *       zero UDP checksum for either IPv4 or IPv6, the networking stack
 *       may perform further validation in this case.
 *     - GRE: only if the checksum is present in the header.
 *     - SCTP: indicates the CRC in SCTP header has been validated.
 *     - FCOE: indicates the CRC in FC frame has been validated.
 *
 *   &sk_buff.csum_level indicates the number of consecutive checksums found in
 *   the packet minus one that have been verified as %CHECKSUM_UNNECESSARY.
 *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
 *   and a device is able to verify the checksums for UDP (possibly zero),
 *   GRE (checksum flag is set) and TCP, &sk_buff.csum_level would be set to
 *   two. If the device were only able to verify the UDP checksum and not
 *   GRE, either because it doesn't support GRE checksum or because GRE
 *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
 *   not considered in this case).
 *
 * - %CHECKSUM_COMPLETE
 *
 *   This is the most generic way. The device supplied checksum of the _whole_
 *   packet as seen by netif_rx() and fills in &sk_buff.csum. This means the
 *   hardware doesn't need to parse L3/L4 headers to implement this.
 *
 *   Notes:
 *
 *   - Even if device supports only some protocols, but is able to produce
 *     skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
 *   - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
 *
 * - %CHECKSUM_PARTIAL
 *
 *   A checksum is set up to be offloaded to a device as described in the
 *   output description for CHECKSUM_PARTIAL. This may occur on a packet
 *   received directly from another Linux OS, e.g., a virtualized Linux kernel
 *   on the same host, or it may be set in the input path in GRO or remote
 *   checksum offload. For the purposes of checksum verification, the checksum
 *   referred to by skb->csum_start + skb->csum_offset and any preceding
 *   checksums in the packet are considered verified. Any checksums in the
 *   packet that are after the checksum being offloaded are not considered to
 *   be verified.
 *
 * Checksumming on transmit for non-GSO
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * The stack requests checksum offload in the &sk_buff.ip_summed for a packet.
 * Values are:
 *
 * - %CHECKSUM_PARTIAL
 *
 *   The driver is required to checksum the packet as seen by hard_start_xmit()
 *   from &sk_buff.csum_start up to the end, and to record/write the checksum at
 *   offset &sk_buff.csum_start + &sk_buff.csum_offset.
 *   A driver may verify that the
 *   csum_start and csum_offset values are valid values given the length and
 *   offset of the packet, but it should not attempt to validate that the
 *   checksum refers to a legitimate transport layer checksum -- it is the
 *   purview of the stack to validate that csum_start and csum_offset are set
 *   correctly.
 *
 *   When the stack requests checksum offload for a packet, the driver MUST
 *   ensure that the checksum is set correctly. A driver can either offload the
 *   checksum calculation to the device, or call skb_checksum_help (in the case
 *   that the device does not support offload for a particular checksum).
 *
 *   %NETIF_F_IP_CSUM and %NETIF_F_IPV6_CSUM are being deprecated in favor of
 *   %NETIF_F_HW_CSUM. New devices should use %NETIF_F_HW_CSUM to indicate
 *   checksum offload capability.
 *   skb_csum_hwoffload_help() can be called to resolve %CHECKSUM_PARTIAL based
 *   on network device checksumming capabilities: if a packet does not match
 *   them, skb_checksum_help() or skb_crc32c_help() (depending on the value of
 *   &sk_buff.csum_not_inet, see :ref:`crc`)
 *   is called to resolve the checksum.
 *
 * - %CHECKSUM_NONE
 *
 *   The skb was already checksummed by the protocol, or a checksum is not
 *   required.
 *
 * - %CHECKSUM_UNNECESSARY
 *
 *   This has the same meaning as CHECKSUM_NONE for checksum offload on
 *   output.
 *
 * - %CHECKSUM_COMPLETE
 *
 *   Not used in checksum output. If a driver observes a packet with this value
 *   set in skbuff, it should treat the packet as if %CHECKSUM_NONE were set.
 *
 * .. _crc:
 *
 * Non-IP checksum (CRC) offloads
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * .. flat-table::
 *   :widths: 1 10
 *
 *   * - %NETIF_F_SCTP_CRC
 *     - This feature indicates that a device is capable of
 *         offloading the SCTP CRC in a packet. To perform this offload the stack
 *         will set csum_start and csum_offset accordingly, set ip_summed to
 *         %CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication
 *         in the skbuff that the %CHECKSUM_PARTIAL refers to CRC32c.
 *         A driver that supports both IP checksum offload and SCTP CRC32c offload
 *         must verify which offload is configured for a packet by testing the
 *         value of &sk_buff.csum_not_inet; skb_crc32c_csum_help() is provided to
 *         resolve %CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
 *
 *   * - %NETIF_F_FCOE_CRC
 *     - This feature indicates that a device is capable of offloading the FCOE
 *         CRC in a packet. To perform this offload the stack will set ip_summed
 *         to %CHECKSUM_PARTIAL and set csum_start and csum_offset
 *         accordingly. Note that there is no indication in the skbuff that the
 *         %CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
 *         both IP checksum offload and FCOE CRC offload must verify which offload
 *         is configured for a packet, presumably by inspecting packet headers.
 *
 * Checksumming on output with GSO
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 * In the case of a GSO packet (skb_is_gso() is true), checksum offload
 * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
 * gso_type is %SKB_GSO_TCPV4 or %SKB_GSO_TCPV6, TCP checksum offload as
 * part of the GSO operation is implied. If a checksum is being offloaded
 * with GSO then ip_summed is %CHECKSUM_PARTIAL, and both csum_start and
 * csum_offset are set to refer to the outermost checksum being offloaded
 * (two offloaded checksums are possible with UDP encapsulation).
 */

/* Don't change this without changing skb_csum_unnecessary! */
#define CHECKSUM_NONE                0
#define CHECKSUM_UNNECESSARY        1
#define CHECKSUM_COMPLETE        2
#define CHECKSUM_PARTIAL        3

/* Maximum value in skb->csum_level */
#define SKB_MAX_CSUM_LEVEL        3

#define SKB_DATA_ALIGN(X)        ALIGN(X, SMP_CACHE_BYTES)
#define SKB_WITH_OVERHEAD(X)        \
        ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

/* For X bytes available in skb->head, what is the minimal
 * allocation needed, knowing struct skb_shared_info needs
 * to be aligned.
 */
#define SKB_HEAD_ALIGN(X) (SKB_DATA_ALIGN(X) + \
        SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

#define SKB_MAX_ORDER(X, ORDER) \
        SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
#define SKB_MAX_HEAD(X)                (SKB_MAX_ORDER((X), 0))
#define SKB_MAX_ALLOC                (SKB_MAX_ORDER(0, 2))

/* return minimum truesize of one skb containing X bytes of data */
#define SKB_TRUESIZE(X) ((X) +                                                \
                         SKB_DATA_ALIGN(sizeof(struct sk_buff)) +        \
                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))

struct ahash_request;
struct net_device;
struct scatterlist;
struct pipe_inode_info;
struct iov_iter;
struct napi_struct;
struct bpf_prog;
union bpf_attr;
struct skb_ext;
struct ts_config;

#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info {
        enum {
                BRNF_PROTO_UNCHANGED,
                BRNF_PROTO_8021Q,
                BRNF_PROTO_PPPOE
        } orig_proto:8;
        u8                        pkt_otherhost:1;
        u8                        in_prerouting:1;
        u8                        bridged_dnat:1;
        u8                        sabotage_in_done:1;
        __u16                        frag_max_size;
        int                        physinif;

        /* always valid & non-NULL from FORWARD on, for physdev match */
        struct net_device        *physoutdev;
        union {
                /* prerouting: detect dnat in orig/reply direction */
                __be32          ipv4_daddr;
                struct in6_addr ipv6_daddr;

                /* after prerouting + nat detected: store original source
                 * mac since neigh resolution overwrites it, only used while
                 * skb is out in neigh layer.
                 */
                char neigh_header[8];
        };
};
#endif

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
/* Chain in tc_skb_ext will be used to share the tc chain with
 * ovs recirc_id. It will be set to the current chain by tc
 * and read by ovs to recirc_id.
 */
struct tc_skb_ext {
        union {
                u64 act_miss_cookie;
                __u32 chain;
        };
        __u16 mru;
        __u16 zone;
        u8 post_ct:1;
        u8 post_ct_snat:1;
        u8 post_ct_dnat:1;
        u8 act_miss:1; /* Set if act_miss_cookie is used */
        u8 l2_miss:1; /* Set by bridge upon FDB or MDB miss */
};
#endif

struct sk_buff_head {
        /* These two members must be first to match sk_buff. */
        struct_group_tagged(sk_buff_list, list,
                struct sk_buff        *next;
                struct sk_buff        *prev;
        );

        __u32                qlen;
        spinlock_t        lock;
};

struct sk_buff;

#ifndef CONFIG_MAX_SKB_FRAGS
# define CONFIG_MAX_SKB_FRAGS 17
#endif

#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS

/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
 * segment using its current segmentation instead.
 */
#define GSO_BY_FRAGS        0xFFFF

typedef struct skb_frag {
        netmem_ref netmem;
        unsigned int len;
        unsigned int offset;
} skb_frag_t;

/**
 * skb_frag_size() - Returns the size of a skb fragment
 * @frag: skb fragment
 */
static inline unsigned int skb_frag_size(const skb_frag_t *frag)
{
        return frag->len;
}

/**
 * skb_frag_size_set() - Sets the size of a skb fragment
 * @frag: skb fragment
 * @size: size of fragment
 */
static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
{
        frag->len = size;
}

/**
 * skb_frag_size_add() - Increments the size of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to add
 */
static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
{
        frag->len += delta;
}

/**
 * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to subtract
 */
static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
{
        frag->len -= delta;
}

/**
 * skb_frag_must_loop - Test if %p is a high memory page
 * @p: fragment's page
 */
static inline bool skb_frag_must_loop(struct page *p)
{
#if defined(CONFIG_HIGHMEM)
        if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) || PageHighMem(p))
                return true;
#endif
        return false;
}

/**
 *        skb_frag_foreach_page - loop over pages in a fragment
 *
 *        @f:                skb frag to operate on
 *        @f_off:                offset from start of f->netmem
 *        @f_len:                length from f_off to loop over
 *        @p:                (temp var) current page
 *        @p_off:                (temp var) offset from start of current page,
 *                                   non-zero only on first page.
 *        @p_len:                (temp var) length in current page,
 *                                   < PAGE_SIZE only on first and last page.
 *        @copied:        (temp var) length so far, excluding current p_len.
 *
 *        A fragment can hold a compound page, in which case per-page
 *        operations, notably kmap_atomic, must be called for each
 *        regular page.
 */
#define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied)        \
        for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT),                \
             p_off = (f_off) & (PAGE_SIZE - 1),                                \
             p_len = skb_frag_must_loop(p) ?                                \
             min_t(u32, f_len, PAGE_SIZE - p_off) : f_len,                \
             copied = 0;                                                \
             copied < f_len;                                                \
             copied += p_len, p++, p_off = 0,                                \
             p_len = min_t(u32, f_len - copied, PAGE_SIZE))                \

/**
 * struct skb_shared_hwtstamps - hardware time stamps
 * @hwtstamp:                hardware time stamp transformed into duration
 *                        since arbitrary point in time
 * @netdev_data:        address/cookie of network device driver used as
 *                        reference to actual hardware time stamp
 *
 * Software time stamps generated by ktime_get_real() are stored in
 * skb->tstamp.
 *
 * hwtstamps can only be compared against other hwtstamps from
 * the same device.
 *
 * This structure is attached to packets as part of the
 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
 */
struct skb_shared_hwtstamps {
        union {
                ktime_t        hwtstamp;
                void *netdev_data;
        };
};

/* Definitions for tx_flags in struct skb_shared_info */
enum {
        /* generate hardware time stamp */
        SKBTX_HW_TSTAMP = 1 << 0,

        /* generate software time stamp when queueing packet to NIC */
        SKBTX_SW_TSTAMP = 1 << 1,

        /* device driver is going to provide hardware time stamp */
        SKBTX_IN_PROGRESS = 1 << 2,

        /* generate hardware time stamp based on cycles if supported */
        SKBTX_HW_TSTAMP_USE_CYCLES = 1 << 3,

        /* generate wifi status information (where possible) */
        SKBTX_WIFI_STATUS = 1 << 4,

        /* determine hardware time stamp based on time or cycles */
        SKBTX_HW_TSTAMP_NETDEV = 1 << 5,

        /* generate software time stamp when entering packet scheduling */
        SKBTX_SCHED_TSTAMP = 1 << 6,
};

#define SKBTX_ANY_SW_TSTAMP        (SKBTX_SW_TSTAMP    | \
                                 SKBTX_SCHED_TSTAMP)
#define SKBTX_ANY_TSTAMP        (SKBTX_HW_TSTAMP | \
                                 SKBTX_HW_TSTAMP_USE_CYCLES | \
                                 SKBTX_ANY_SW_TSTAMP)

/* Definitions for flags in struct skb_shared_info */
enum {
        /* use zcopy routines */
        SKBFL_ZEROCOPY_ENABLE = BIT(0),

        /* This indicates at least one fragment might be overwritten
         * (as in vmsplice(), sendfile() ...)
         * If we need to compute a TX checksum, we'll need to copy
         * all frags to avoid possible bad checksum
         */
        SKBFL_SHARED_FRAG = BIT(1),

        /* segment contains only zerocopy data and should not be
         * charged to the kernel memory.
         */
        SKBFL_PURE_ZEROCOPY = BIT(2),

        SKBFL_DONT_ORPHAN = BIT(3),

        /* page references are managed by the ubuf_info, so it's safe to
         * use frags only up until ubuf_info is released
         */
        SKBFL_MANAGED_FRAG_REFS = BIT(4),
};

#define SKBFL_ZEROCOPY_FRAG        (SKBFL_ZEROCOPY_ENABLE | SKBFL_SHARED_FRAG)
#define SKBFL_ALL_ZEROCOPY        (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
                                 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)

struct ubuf_info_ops {
        void (*complete)(struct sk_buff *, struct ubuf_info *,
                         bool zerocopy_success);
        /* has to be compatible with skb_zcopy_set() */
        int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
};

/*
 * The callback notifies userspace to release buffers when skb DMA is done in
 * lower device, the skb last reference should be 0 when calling this.
 * The zerocopy_success argument is true if zero copy transmit occurred,
 * false on data copy or out of memory error caused by data copy attempt.
 * The ctx field is used to track device context.
 * The desc field is used to track userspace buffer index.
 */
struct ubuf_info {
        const struct ubuf_info_ops *ops;
        refcount_t refcnt;
        u8 flags;
};

struct ubuf_info_msgzc {
        struct ubuf_info ubuf;

        union {
                struct {
                        unsigned long desc;
                        void *ctx;
                };
                struct {
                        u32 id;
                        u16 len;
                        u16 zerocopy:1;
                        u32 bytelen;
                };
        };

        struct mmpin {
                struct user_struct *user;
                unsigned int num_pg;
        } mmp;
};

#define skb_uarg(SKB)        ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
#define uarg_to_msgzc(ubuf_ptr)        container_of((ubuf_ptr), struct ubuf_info_msgzc, \
                                             ubuf)

int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
void mm_unaccount_pinned_pages(struct mmpin *mmp);

/* Preserve some data across TX submission and completion.
 *
 * Note, this state is stored in the driver. Extending the layout
 * might need some special care.
 */
struct xsk_tx_metadata_compl {
        __u64 *tx_timestamp;
};

/* This data is invariant across clones and lives at
 * the end of the header data, ie. at skb->end.
 */
struct skb_shared_info {
        __u8                flags;
        __u8                meta_len;
        __u8                nr_frags;
        __u8                tx_flags;
        unsigned short        gso_size;
        /* Warning: this field is not always filled in (UFO)! */
        unsigned short        gso_segs;
        struct sk_buff        *frag_list;
        union {
                struct skb_shared_hwtstamps hwtstamps;
                struct xsk_tx_metadata_compl xsk_meta;
        };
        unsigned int        gso_type;
        u32                tskey;

        /*
         * Warning : all fields before dataref are cleared in __alloc_skb()
         */
        atomic_t        dataref;
        unsigned int        xdp_frags_size;

        /* Intermediate layers must ensure that destructor_arg
         * remains valid until skb destructor */
        void *                destructor_arg;

        /* must be last field, see pskb_expand_head() */
        skb_frag_t        frags[MAX_SKB_FRAGS];
};

/**
 * DOC: dataref and headerless skbs
 *
 * Transport layers send out clones of payload skbs they hold for
 * retransmissions. To allow lower layers of the stack to prepend their headers
 * we split &skb_shared_info.dataref into two halves.
 * The lower 16 bits count the overall number of references.
 * The higher 16 bits indicate how many of the references are payload-only.
 * skb_header_cloned() checks if skb is allowed to add / write the headers.
 *
 * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr
 * (via __skb_header_release()). Any clone created from marked skb will get
 * &sk_buff.hdr_len populated with the available headroom.
 * If there's the only clone in existence it's able to modify the headroom
 * at will. The sequence of calls inside the transport layer is::
 *
 *  <alloc skb>
 *  skb_reserve()
 *  __skb_header_release()
 *  skb_clone()
 *  // send the clone down the stack
 *
 * This is not a very generic construct and it depends on the transport layers
 * doing the right thing. In practice there's usually only one payload-only skb.
 * Having multiple payload-only skbs with different lengths of hdr_len is not
 * possible. The payload-only skbs should never leave their owner.
 */
#define SKB_DATAREF_SHIFT 16
#define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)


enum {
        SKB_FCLONE_UNAVAILABLE,        /* skb has no fclone (from head_cache) */
        SKB_FCLONE_ORIG,        /* orig skb (from fclone_cache) */
        SKB_FCLONE_CLONE,        /* companion fclone skb (from fclone_cache) */
};

enum {
        SKB_GSO_TCPV4 = 1 << 0,

        /* This indicates the skb is from an untrusted source. */
        SKB_GSO_DODGY = 1 << 1,

        /* This indicates the tcp segment has CWR set. */
        SKB_GSO_TCP_ECN = 1 << 2,

        SKB_GSO_TCP_FIXEDID = 1 << 3,

        SKB_GSO_TCPV6 = 1 << 4,

        SKB_GSO_FCOE = 1 << 5,

        SKB_GSO_GRE = 1 << 6,

        SKB_GSO_GRE_CSUM = 1 << 7,

        SKB_GSO_IPXIP4 = 1 << 8,

        SKB_GSO_IPXIP6 = 1 << 9,

        SKB_GSO_UDP_TUNNEL = 1 << 10,

        SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,

        SKB_GSO_PARTIAL = 1 << 12,

        SKB_GSO_TUNNEL_REMCSUM = 1 << 13,

        SKB_GSO_SCTP = 1 << 14,

        SKB_GSO_ESP = 1 << 15,

        SKB_GSO_UDP = 1 << 16,

        SKB_GSO_UDP_L4 = 1 << 17,

        SKB_GSO_FRAGLIST = 1 << 18,
};

#if BITS_PER_LONG > 32
#define NET_SKBUFF_DATA_USES_OFFSET 1
#endif

#ifdef NET_SKBUFF_DATA_USES_OFFSET
typedef unsigned int sk_buff_data_t;
#else
typedef unsigned char *sk_buff_data_t;
#endif

enum skb_tstamp_type {
        SKB_CLOCK_REALTIME,
        SKB_CLOCK_MONOTONIC,
        SKB_CLOCK_TAI,
        __SKB_CLOCK_MAX = SKB_CLOCK_TAI,
};

/**
 * DOC: Basic sk_buff geometry
 *
 * struct sk_buff itself is a metadata structure and does not hold any packet
 * data. All the data is held in associated buffers.
 *
 * &sk_buff.head points to the main "head" buffer. The head buffer is divided
 * into two parts:
 *
 *  - data buffer, containing headers and sometimes payload;
 *    this is the part of the skb operated on by the common helpers
 *    such as skb_put() or skb_pull();
 *  - shared info (struct skb_shared_info) which holds an array of pointers
 *    to read-only data in the (page, offset, length) format.
 *
 * Optionally &skb_shared_info.frag_list may point to another skb.
 *
 * Basic diagram may look like this::
 *
 *                                  ---------------
 *                                 | sk_buff       |
 *                                  ---------------
 *     ,---------------------------  + head
 *    /          ,-----------------  + data
 *   /          /      ,-----------  + tail
 *  |          |      |            , + end
 *  |          |      |           |
 *  v          v      v           v
 *   -----------------------------------------------
 *  | headroom | data |  tailroom | skb_shared_info |
 *   -----------------------------------------------
 *                                 + [page frag]
 *                                 + [page frag]
 *                                 + [page frag]
 *                                 + [page frag]       ---------
 *                                 + frag_list    --> | sk_buff |
 *                                                     ---------
 *
 */

/**
 *        struct sk_buff - socket buffer
 *        @next: Next buffer in list
 *        @prev: Previous buffer in list
 *        @tstamp: Time we arrived/left
 *        @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
 *                for retransmit timer
 *        @rbnode: RB tree node, alternative to next/prev for netem/tcp
 *        @list: queue head
 *        @ll_node: anchor in an llist (eg socket defer_list)
 *        @sk: Socket we are owned by
 *        @dev: Device we arrived on/are leaving by
 *        @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
 *        @cb: Control buffer. Free for use by every layer. Put private vars here
 *        @_skb_refdst: destination entry (with norefcount bit)
 *        @len: Length of actual data
 *        @data_len: Data length
 *        @mac_len: Length of link layer header
 *        @hdr_len: writable header length of cloned skb
 *        @csum: Checksum (must include start/offset pair)
 *        @csum_start: Offset from skb->head where checksumming should start
 *        @csum_offset: Offset from csum_start where checksum should be stored
 *        @priority: Packet queueing priority
 *        @ignore_df: allow local fragmentation
 *        @cloned: Head may be cloned (check refcnt to be sure)
 *        @ip_summed: Driver fed us an IP checksum
 *        @nohdr: Payload reference only, must not modify header
 *        @pkt_type: Packet class
 *        @fclone: skbuff clone status
 *        @ipvs_property: skbuff is owned by ipvs
 *        @inner_protocol_type: whether the inner protocol is
 *                ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
 *        @remcsum_offload: remote checksum offload is enabled
 *        @offload_fwd_mark: Packet was L2-forwarded in hardware
 *        @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
 *        @tc_skip_classify: do not classify packet. set by IFB device
 *        @tc_at_ingress: used within tc_classify to distinguish in/egress
 *        @redirected: packet was redirected by packet classifier
 *        @from_ingress: packet was redirected from the ingress path
 *        @nf_skip_egress: packet shall skip nf egress - see netfilter_netdev.h
 *        @peeked: this packet has been seen already, so stats have been
 *                done for it, don't do them again
 *        @nf_trace: netfilter packet trace flag
 *        @protocol: Packet protocol from driver
 *        @destructor: Destruct function
 *        @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
 *        @_sk_redir: socket redirection information for skmsg
 *        @_nfct: Associated connection, if any (with nfctinfo bits)
 *        @skb_iif: ifindex of device we arrived on
 *        @tc_index: Traffic control index
 *        @hash: the packet hash
 *        @queue_mapping: Queue mapping for multiqueue devices
 *        @head_frag: skb was allocated from page fragments,
 *                not allocated by kmalloc() or vmalloc().
 *        @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
 *        @pp_recycle: mark the packet for recycling instead of freeing (implies
 *                page_pool support on driver)
 *        @active_extensions: active extensions (skb_ext_id types)
 *        @ndisc_nodetype: router type (from link layer)
 *        @ooo_okay: allow the mapping of a socket to a queue to be changed
 *        @l4_hash: indicate hash is a canonical 4-tuple hash over transport
 *                ports.
 *        @sw_hash: indicates hash was computed in software stack
 *        @wifi_acked_valid: wifi_acked was set
 *        @wifi_acked: whether frame was acked on wifi or not
 *        @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
 *        @encapsulation: indicates the inner headers in the skbuff are valid
 *        @encap_hdr_csum: software checksum is needed
 *        @csum_valid: checksum is already valid
 *        @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
 *        @csum_complete_sw: checksum was completed by software
 *        @csum_level: indicates the number of consecutive checksums found in
 *                the packet minus one that have been verified as
 *                CHECKSUM_UNNECESSARY (max 3)
 *        @unreadable: indicates that at least 1 of the fragments in this skb is
 *                unreadable.
 *        @dst_pending_confirm: need to confirm neighbour
 *        @decrypted: Decrypted SKB
 *        @slow_gro: state present at GRO time, slower prepare step required
 *        @tstamp_type: When set, skb->tstamp has the
 *                delivery_time clock base of skb->tstamp.
 *        @napi_id: id of the NAPI struct this skb came from
 *        @sender_cpu: (aka @napi_id) source CPU in XPS
 *        @alloc_cpu: CPU which did the skb allocation.
 *        @secmark: security marking
 *        @mark: Generic packet mark
 *        @reserved_tailroom: (aka @mark) number of bytes of free space available
 *                at the tail of an sk_buff
 *        @vlan_all: vlan fields (proto & tci)
 *        @vlan_proto: vlan encapsulation protocol
 *        @vlan_tci: vlan tag control information
 *        @inner_protocol: Protocol (encapsulation)
 *        @inner_ipproto: (aka @inner_protocol) stores ipproto when
 *                skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
 *        @inner_transport_header: Inner transport layer header (encapsulation)
 *        @inner_network_header: Network layer header (encapsulation)
 *        @inner_mac_header: Link layer header (encapsulation)
 *        @transport_header: Transport layer header
 *        @network_header: Network layer header
 *        @mac_header: Link layer header
 *        @kcov_handle: KCOV remote handle for remote coverage collection
 *        @tail: Tail pointer
 *        @end: End pointer
 *        @head: Head of buffer
 *        @data: Data head pointer
 *        @truesize: Buffer size
 *        @users: User count - see {datagram,tcp}.c
 *        @extensions: allocated extensions, valid if active_extensions is nonzero
 */

struct sk_buff {
        union {
                struct {
                        /* These two members must be first to match sk_buff_head. */
                        struct sk_buff                *next;
                        struct sk_buff                *prev;

                        union {
                                struct net_device        *dev;
                                /* Some protocols might use this space to store information,
                                 * while device pointer would be NULL.
                                 * UDP receive path is one user.
                                 */
                                unsigned long                dev_scratch;
                        };
                };
                struct rb_node                rbnode; /* used in netem, ip4 defrag, and tcp stack */
                struct list_head        list;
                struct llist_node        ll_node;
        };

        struct sock                *sk;

        union {
                ktime_t                tstamp;
                u64                skb_mstamp_ns; /* earliest departure time */
        };
        /*
         * This is the control buffer. It is free to use for every
         * layer. Please put your private variables there. If you
         * want to keep them across layers you have to do a skb_clone()
         * first. This is owned by whoever has the skb queued ATM.
         */
        char                        cb[48] __aligned(8);

        union {
                struct {
                        unsigned long        _skb_refdst;
                        void                (*destructor)(struct sk_buff *skb);
                };
                struct list_head        tcp_tsorted_anchor;
#ifdef CONFIG_NET_SOCK_MSG
                unsigned long                _sk_redir;
#endif
        };

#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        unsigned long                 _nfct;
#endif
        unsigned int                len,
                                data_len;
        __u16                        mac_len,
                                hdr_len;

        /* Following fields are _not_ copied in __copy_skb_header()
         * Note that queue_mapping is here mostly to fill a hole.
         */
        __u16                        queue_mapping;

/* if you move cloned around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define CLONED_MASK        (1 << 7)
#else
#define CLONED_MASK        1
#endif
#define CLONED_OFFSET                offsetof(struct sk_buff, __cloned_offset)

        /* private: */
        __u8                        __cloned_offset[0];
        /* public: */
        __u8                        cloned:1,
                                nohdr:1,
                                fclone:2,
                                peeked:1,
                                head_frag:1,
                                pfmemalloc:1,
                                pp_recycle:1; /* page_pool recycle indicator */
#ifdef CONFIG_SKB_EXTENSIONS
        __u8                        active_extensions;
#endif

        /* Fields enclosed in headers group are copied
         * using a single memcpy() in __copy_skb_header()
         */
        struct_group(headers,

        /* private: */
        __u8                        __pkt_type_offset[0];
        /* public: */
        __u8                        pkt_type:3; /* see PKT_TYPE_MAX */
        __u8                        ignore_df:1;
        __u8                        dst_pending_confirm:1;
        __u8                        ip_summed:2;
        __u8                        ooo_okay:1;

        /* private: */
        __u8                        __mono_tc_offset[0];
        /* public: */
        __u8                        tstamp_type:2;        /* See skb_tstamp_type */
#ifdef CONFIG_NET_XGRESS
        __u8                        tc_at_ingress:1;        /* See TC_AT_INGRESS_MASK */
        __u8                        tc_skip_classify:1;
#endif
        __u8                        remcsum_offload:1;
        __u8                        csum_complete_sw:1;
        __u8                        csum_level:2;
        __u8                        inner_protocol_type:1;

        __u8                        l4_hash:1;
        __u8                        sw_hash:1;
#ifdef CONFIG_WIRELESS
        __u8                        wifi_acked_valid:1;
        __u8                        wifi_acked:1;
#endif
        __u8                        no_fcs:1;
        /* Indicates the inner headers are valid in the skbuff. */
        __u8                        encapsulation:1;
        __u8                        encap_hdr_csum:1;
        __u8                        csum_valid:1;
#ifdef CONFIG_IPV6_NDISC_NODETYPE
        __u8                        ndisc_nodetype:2;
#endif

#if IS_ENABLED(CONFIG_IP_VS)
        __u8                        ipvs_property:1;
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        __u8                        nf_trace:1;
#endif
#ifdef CONFIG_NET_SWITCHDEV
        __u8                        offload_fwd_mark:1;
        __u8                        offload_l3_fwd_mark:1;
#endif
        __u8                        redirected:1;
#ifdef CONFIG_NET_REDIRECT
        __u8                        from_ingress:1;
#endif
#ifdef CONFIG_NETFILTER_SKIP_EGRESS
        __u8                        nf_skip_egress:1;
#endif
#ifdef CONFIG_SKB_DECRYPTED
        __u8                        decrypted:1;
#endif
        __u8                        slow_gro:1;
#if IS_ENABLED(CONFIG_IP_SCTP)
        __u8                        csum_not_inet:1;
#endif
        __u8                        unreadable:1;
#if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS)
        __u16                        tc_index;        /* traffic control index */
#endif

        u16                        alloc_cpu;

        union {
                __wsum                csum;
                struct {
                        __u16        csum_start;
                        __u16        csum_offset;
                };
        };
        __u32                        priority;
        int                        skb_iif;
        __u32                        hash;
        union {
                u32                vlan_all;
                struct {
                        __be16        vlan_proto;
                        __u16        vlan_tci;
                };
        };
#if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
        union {
                unsigned int        napi_id;
                unsigned int        sender_cpu;
        };
#endif
#ifdef CONFIG_NETWORK_SECMARK
        __u32                secmark;
#endif

        union {
                __u32                mark;
                __u32                reserved_tailroom;
        };

        union {
                __be16                inner_protocol;
                __u8                inner_ipproto;
        };

        __u16                        inner_transport_header;
        __u16                        inner_network_header;
        __u16                        inner_mac_header;

        __be16                        protocol;
        __u16                        transport_header;
        __u16                        network_header;
        __u16                        mac_header;

#ifdef CONFIG_KCOV
        u64                        kcov_handle;
#endif

        ); /* end headers group */

        /* These elements must be at the end, see alloc_skb() for details.  */
        sk_buff_data_t                tail;
        sk_buff_data_t                end;
        unsigned char                *head,
                                *data;
        unsigned int                truesize;
        refcount_t                users;

#ifdef CONFIG_SKB_EXTENSIONS
        /* only usable after checking ->active_extensions != 0 */
        struct skb_ext                *extensions;
#endif
};

/* if you move pkt_type around you also must adapt those constants */
#ifdef __BIG_ENDIAN_BITFIELD
#define PKT_TYPE_MAX        (7 << 5)
#else
#define PKT_TYPE_MAX        7
#endif
#define PKT_TYPE_OFFSET                offsetof(struct sk_buff, __pkt_type_offset)

/* if you move tc_at_ingress or tstamp_type
 * around, you also must adapt these constants.
 */
#ifdef __BIG_ENDIAN_BITFIELD
#define SKB_TSTAMP_TYPE_MASK                (3 << 6)
#define SKB_TSTAMP_TYPE_RSHIFT                (6)
#define TC_AT_INGRESS_MASK                (1 << 5)
#else
#define SKB_TSTAMP_TYPE_MASK                (3)
#define TC_AT_INGRESS_MASK                (1 << 2)
#endif
#define SKB_BF_MONO_TC_OFFSET                offsetof(struct sk_buff, __mono_tc_offset)

#ifdef __KERNEL__
/*
 *        Handling routines are only of interest to the kernel
 */

#define SKB_ALLOC_FCLONE        0x01
#define SKB_ALLOC_RX                0x02
#define SKB_ALLOC_NAPI                0x04

/**
 * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
 * @skb: buffer
 */
static inline bool skb_pfmemalloc(const struct sk_buff *skb)
{
        return unlikely(skb->pfmemalloc);
}

/*
 * skb might have a dst pointer attached, refcounted or not.
 * _skb_refdst low order bit is set if refcount was _not_ taken
 */
#define SKB_DST_NOREF        1UL
#define SKB_DST_PTRMASK        ~(SKB_DST_NOREF)

/**
 * skb_dst - returns skb dst_entry
 * @skb: buffer
 *
 * Returns skb dst_entry, regardless of reference taken or not.
 */
static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
{
        /* If refdst was not refcounted, check we still are in a
         * rcu_read_lock section
         */
        WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
                !rcu_read_lock_held() &&
                !rcu_read_lock_bh_held());
        return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
}

/**
 * skb_dst_set - sets skb dst
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was taken on dst and should
 * be released by skb_dst_drop()
 */
static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
{
        skb->slow_gro |= !!dst;
        skb->_skb_refdst = (unsigned long)dst;
}

/**
 * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
 * @skb: buffer
 * @dst: dst entry
 *
 * Sets skb dst, assuming a reference was not taken on dst.
 * If dst entry is cached, we do not take reference and dst_release
 * will be avoided by refdst_drop. If dst entry is not cached, we take
 * reference, so that last dst_release can destroy the dst immediately.
 */
static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
{
        WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
        skb->slow_gro |= !!dst;
        skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
}

/**
 * skb_dst_is_noref - Test if skb dst isn't refcounted
 * @skb: buffer
 */
static inline bool skb_dst_is_noref(const struct sk_buff *skb)
{
        return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
}

/* For mangling skb->pkt_type from user space side from applications
 * such as nft, tc, etc, we only allow a conservative subset of
 * possible pkt_types to be set.
*/
static inline bool skb_pkt_type_ok(u32 ptype)
{
        return ptype <= PACKET_OTHERHOST;
}

/**
 * skb_napi_id - Returns the skb's NAPI id
 * @skb: buffer
 */
static inline unsigned int skb_napi_id(const struct sk_buff *skb)
{
#ifdef CONFIG_NET_RX_BUSY_POLL
        return skb->napi_id;
#else
        return 0;
#endif
}

static inline bool skb_wifi_acked_valid(const struct sk_buff *skb)
{
#ifdef CONFIG_WIRELESS
        return skb->wifi_acked_valid;
#else
        return 0;
#endif
}

/**
 * skb_unref - decrement the skb's reference count
 * @skb: buffer
 *
 * Returns true if we can free the skb.
 */
static inline bool skb_unref(struct sk_buff *skb)
{
        if (unlikely(!skb))
                return false;
        if (!IS_ENABLED(CONFIG_DEBUG_NET) && likely(refcount_read(&skb->users) == 1))
                smp_rmb();
        else if (likely(!refcount_dec_and_test(&skb->users)))
                return false;

        return true;
}

static inline bool skb_data_unref(const struct sk_buff *skb,
                                  struct skb_shared_info *shinfo)
{
        int bias;

        if (!skb->cloned)
                return true;

        bias = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;

        if (atomic_read(&shinfo->dataref) == bias)
                smp_rmb();
        else if (atomic_sub_return(bias, &shinfo->dataref))
                return false;

        return true;
}

void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
                                      enum skb_drop_reason reason);

static inline void
kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
{
        sk_skb_reason_drop(NULL, skb, reason);
}

/**
 *        kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
 *        @skb: buffer to free
 */
static inline void kfree_skb(struct sk_buff *skb)
{
        kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
}

void skb_release_head_state(struct sk_buff *skb);
void kfree_skb_list_reason(struct sk_buff *segs,
                           enum skb_drop_reason reason);
void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
void skb_tx_error(struct sk_buff *skb);

static inline void kfree_skb_list(struct sk_buff *segs)
{
        kfree_skb_list_reason(segs, SKB_DROP_REASON_NOT_SPECIFIED);
}

#ifdef CONFIG_TRACEPOINTS
void consume_skb(struct sk_buff *skb);
#else
static inline void consume_skb(struct sk_buff *skb)
{
        return kfree_skb(skb);
}
#endif

void __consume_stateless_skb(struct sk_buff *skb);
void  __kfree_skb(struct sk_buff *skb);

void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
                      bool *fragstolen, int *delta_truesize);

struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
                            int node);
struct sk_buff *__build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb(void *data, unsigned int frag_size);
struct sk_buff *build_skb_around(struct sk_buff *skb,
                                 void *data, unsigned int frag_size);
void skb_attempt_defer_free(struct sk_buff *skb);

struct sk_buff *napi_build_skb(void *data, unsigned int frag_size);
struct sk_buff *slab_build_skb(void *data);

/**
 * alloc_skb - allocate a network buffer
 * @size: size to allocate
 * @priority: allocation mask
 *
 * This function is a convenient wrapper around __alloc_skb().
 */
static inline struct sk_buff *alloc_skb(unsigned int size,
                                        gfp_t priority)
{
        return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
}

struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
                                     unsigned long data_len,
                                     int max_page_order,
                                     int *errcode,
                                     gfp_t gfp_mask);
struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);

/* Layout of fast clones : [skb1][skb2][fclone_ref] */
struct sk_buff_fclones {
        struct sk_buff        skb1;

        struct sk_buff        skb2;

        refcount_t        fclone_ref;
};

/**
 *        skb_fclone_busy - check if fclone is busy
 *        @sk: socket
 *        @skb: buffer
 *
 * Returns true if skb is a fast clone, and its clone is not freed.
 * Some drivers call skb_orphan() in their ndo_start_xmit(),
 * so we also check that didn't happen.
 */
static inline bool skb_fclone_busy(const struct sock *sk,
                                   const struct sk_buff *skb)
{
        const struct sk_buff_fclones *fclones;

        fclones = container_of(skb, struct sk_buff_fclones, skb1);

        return skb->fclone == SKB_FCLONE_ORIG &&
               refcount_read(&fclones->fclone_ref) > 1 &&
               READ_ONCE(fclones->skb2.sk) == sk;
}

/**
 * alloc_skb_fclone - allocate a network buffer from fclone cache
 * @size: size to allocate
 * @priority: allocation mask
 *
 * This function is a convenient wrapper around __alloc_skb().
 */
static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
                                               gfp_t priority)
{
        return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
}

struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
void skb_headers_offset_update(struct sk_buff *skb, int off);
int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
                                   gfp_t gfp_mask, bool fclone);
static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
                                          gfp_t gfp_mask)
{
        return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
}

int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
                                     unsigned int headroom);
struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom);
struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
                                int newtailroom, gfp_t priority);
int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
                                     int offset, int len);
int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
                              int offset, int len);
int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);

/**
 *        skb_pad                        -        zero pad the tail of an skb
 *        @skb: buffer to pad
 *        @pad: space to pad
 *
 *        Ensure that a buffer is followed by a padding area that is zero
 *        filled. Used by network drivers which may DMA or transfer data
 *        beyond the buffer end onto the wire.
 *
 *        May return error in out of memory cases. The skb is freed on error.
 */
static inline int skb_pad(struct sk_buff *skb, int pad)
{
        return __skb_pad(skb, pad, true);
}
#define dev_kfree_skb(a)        consume_skb(a)

int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
                         int offset, size_t size, size_t max_frags);

struct skb_seq_state {
        __u32                lower_offset;
        __u32                upper_offset;
        __u32                frag_idx;
        __u32                stepped_offset;
        struct sk_buff        *root_skb;
        struct sk_buff        *cur_skb;
        __u8                *frag_data;
        __u32                frag_off;
};

void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
                          unsigned int to, struct skb_seq_state *st);
unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
                          struct skb_seq_state *st);
void skb_abort_seq_read(struct skb_seq_state *st);
int skb_copy_seq_read(struct skb_seq_state *st, int offset, void *to, int len);

unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
                           unsigned int to, struct ts_config *config);

/*
 * Packet hash types specify the type of hash in skb_set_hash.
 *
 * Hash types refer to the protocol layer addresses which are used to
 * construct a packet's hash. The hashes are used to differentiate or identify
 * flows of the protocol layer for the hash type. Hash types are either
 * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
 *
 * Properties of hashes:
 *
 * 1) Two packets in different flows have different hash values
 * 2) Two packets in the same flow should have the same hash value
 *
 * A hash at a higher layer is considered to be more specific. A driver should
 * set the most specific hash possible.
 *
 * A driver cannot indicate a more specific hash than the layer at which a hash
 * was computed. For instance an L3 hash cannot be set as an L4 hash.
 *
 * A driver may indicate a hash level which is less specific than the
 * actual layer the hash was computed on. For instance, a hash computed
 * at L4 may be considered an L3 hash. This should only be done if the
 * driver can't unambiguously determine that the HW computed the hash at
 * the higher layer. Note that the "should" in the second property above
 * permits this.
 */
enum pkt_hash_types {
        PKT_HASH_TYPE_NONE,        /* Undefined type */
        PKT_HASH_TYPE_L2,        /* Input: src_MAC, dest_MAC */
        PKT_HASH_TYPE_L3,        /* Input: src_IP, dst_IP */
        PKT_HASH_TYPE_L4,        /* Input: src_IP, dst_IP, src_port, dst_port */
};

static inline void skb_clear_hash(struct sk_buff *skb)
{
        skb->hash = 0;
        skb->sw_hash = 0;
        skb->l4_hash = 0;
}

static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
{
        if (!skb->l4_hash)
                skb_clear_hash(skb);
}

static inline void
__skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
{
        skb->l4_hash = is_l4;
        skb->sw_hash = is_sw;
        skb->hash = hash;
}

static inline void
skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
{
        /* Used by drivers to set hash from HW */
        __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
}

static inline void
__skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
{
        __skb_set_hash(skb, hash, true, is_l4);
}

u32 __skb_get_hash_symmetric_net(const struct net *net, const struct sk_buff *skb);

static inline u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
{
        return __skb_get_hash_symmetric_net(NULL, skb);
}

void __skb_get_hash_net(const struct net *net, struct sk_buff *skb);
u32 skb_get_poff(const struct sk_buff *skb);
u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
                   const struct flow_keys_basic *keys, int hlen);
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
                            const void *data, int hlen_proto);

static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
                                        int thoff, u8 ip_proto)
{
        return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
}

void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
                             const struct flow_dissector_key *key,
                             unsigned int key_count);

struct bpf_flow_dissector;
u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
                     __be16 proto, int nhoff, int hlen, unsigned int flags);

bool __skb_flow_dissect(const struct net *net,
                        const struct sk_buff *skb,
                        struct flow_dissector *flow_dissector,
                        void *target_container, const void *data,
                        __be16 proto, int nhoff, int hlen, unsigned int flags);

static inline bool skb_flow_dissect(const struct sk_buff *skb,
                                    struct flow_dissector *flow_dissector,
                                    void *target_container, unsigned int flags)
{
        return __skb_flow_dissect(NULL, skb, flow_dissector,
                                  target_container, NULL, 0, 0, 0, flags);
}

static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
                                              struct flow_keys *flow,
                                              unsigned int flags)
{
        memset(flow, 0, sizeof(*flow));
        return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
                                  flow, NULL, 0, 0, 0, flags);
}

static inline bool
skb_flow_dissect_flow_keys_basic(const struct net *net,
                                 const struct sk_buff *skb,
                                 struct flow_keys_basic *flow,
                                 const void *data, __be16 proto,
                                 int nhoff, int hlen, unsigned int flags)
{
        memset(flow, 0, sizeof(*flow));
        return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
                                  data, proto, nhoff, hlen, flags);
}

void skb_flow_dissect_meta(const struct sk_buff *skb,
                           struct flow_dissector *flow_dissector,
                           void *target_container);

/* Gets a skb connection tracking info, ctinfo map should be a
 * map of mapsize to translate enum ip_conntrack_info states
 * to user states.
 */
void
skb_flow_dissect_ct(const struct sk_buff *skb,
                    struct flow_dissector *flow_dissector,
                    void *target_container,
                    u16 *ctinfo_map, size_t mapsize,
                    bool post_ct, u16 zone);
void
skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
                             struct flow_dissector *flow_dissector,
                             void *target_container);

void skb_flow_dissect_hash(const struct sk_buff *skb,
                           struct flow_dissector *flow_dissector,
                           void *target_container);

static inline __u32 skb_get_hash_net(const struct net *net, struct sk_buff *skb)
{
        if (!skb->l4_hash && !skb->sw_hash)
                __skb_get_hash_net(net, skb);

        return skb->hash;
}

static inline __u32 skb_get_hash(struct sk_buff *skb)
{
        if (!skb->l4_hash && !skb->sw_hash)
                __skb_get_hash_net(NULL, skb);

        return skb->hash;
}

static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
{
        if (!skb->l4_hash && !skb->sw_hash) {
                struct flow_keys keys;
                __u32 hash = __get_hash_from_flowi6(fl6, &keys);

                __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
        }

        return skb->hash;
}

__u32 skb_get_hash_perturb(const struct sk_buff *skb,
                           const siphash_key_t *perturb);

static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
{
        return skb->hash;
}

static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
{
        to->hash = from->hash;
        to->sw_hash = from->sw_hash;
        to->l4_hash = from->l4_hash;
};

static inline int skb_cmp_decrypted(const struct sk_buff *skb1,
                                    const struct sk_buff *skb2)
{
#ifdef CONFIG_SKB_DECRYPTED
        return skb2->decrypted - skb1->decrypted;
#else
        return 0;
#endif
}

static inline bool skb_is_decrypted(const struct sk_buff *skb)
{
#ifdef CONFIG_SKB_DECRYPTED
        return skb->decrypted;
#else
        return false;
#endif
}

static inline void skb_copy_decrypted(struct sk_buff *to,
                                      const struct sk_buff *from)
{
#ifdef CONFIG_SKB_DECRYPTED
        to->decrypted = from->decrypted;
#endif
}

#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
        return skb->head + skb->end;
}

static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
        return skb->end;
}

static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
        skb->end = offset;
}
#else
static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
{
        return skb->end;
}

static inline unsigned int skb_end_offset(const struct sk_buff *skb)
{
        return skb->end - skb->head;
}

static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
{
        skb->end = skb->head + offset;
}
#endif

extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;

struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                                       struct ubuf_info *uarg);

void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);

int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
                            struct sk_buff *skb, struct iov_iter *from,
                            size_t length);

int zerocopy_fill_skb_from_iter(struct sk_buff *skb,
                                struct iov_iter *from, size_t length);

static inline int skb_zerocopy_iter_dgram(struct sk_buff *skb,
                                          struct msghdr *msg, int len)
{
        return __zerocopy_sg_from_iter(msg, skb->sk, skb, &msg->msg_iter, len);
}

int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
                             struct msghdr *msg, int len,
                             struct ubuf_info *uarg);

/* Internal */
#define skb_shinfo(SKB)        ((struct skb_shared_info *)(skb_end_pointer(SKB)))

static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
{
        return &skb_shinfo(skb)->hwtstamps;
}

static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
{
        bool is_zcopy = skb && skb_shinfo(skb)->flags & SKBFL_ZEROCOPY_ENABLE;

        return is_zcopy ? skb_uarg(skb) : NULL;
}

static inline bool skb_zcopy_pure(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->flags & SKBFL_PURE_ZEROCOPY;
}

static inline bool skb_zcopy_managed(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->flags & SKBFL_MANAGED_FRAG_REFS;
}

static inline bool skb_pure_zcopy_same(const struct sk_buff *skb1,
                                       const struct sk_buff *skb2)
{
        return skb_zcopy_pure(skb1) == skb_zcopy_pure(skb2);
}

static inline void net_zcopy_get(struct ubuf_info *uarg)
{
        refcount_inc(&uarg->refcnt);
}

static inline void skb_zcopy_init(struct sk_buff *skb, struct ubuf_info *uarg)
{
        skb_shinfo(skb)->destructor_arg = uarg;
        skb_shinfo(skb)->flags |= uarg->flags;
}

static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
                                 bool *have_ref)
{
        if (skb && uarg && !skb_zcopy(skb)) {
                if (unlikely(have_ref && *have_ref))
                        *have_ref = false;
                else
                        net_zcopy_get(uarg);
                skb_zcopy_init(skb, uarg);
        }
}

static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
{
        skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
        skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
}

static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
{
        return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
}

static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
{
        return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
}

static inline void net_zcopy_put(struct ubuf_info *uarg)
{
        if (uarg)
                uarg->ops->complete(NULL, uarg, true);
}

static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
{
        if (uarg) {
                if (uarg->ops == &msg_zerocopy_ubuf_ops)
                        msg_zerocopy_put_abort(uarg, have_uref);
                else if (have_uref)
                        net_zcopy_put(uarg);
        }
}

/* Release a reference on a zerocopy structure */
static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
{
        struct ubuf_info *uarg = skb_zcopy(skb);

        if (uarg) {
                if (!skb_zcopy_is_nouarg(skb))
                        uarg->ops->complete(skb, uarg, zerocopy_success);

                skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
        }
}

void __skb_zcopy_downgrade_managed(struct sk_buff *skb);

static inline void skb_zcopy_downgrade_managed(struct sk_buff *skb)
{
        if (unlikely(skb_zcopy_managed(skb)))
                __skb_zcopy_downgrade_managed(skb);
}

/* Return true if frags in this skb are readable by the host. */
static inline bool skb_frags_readable(const struct sk_buff *skb)
{
        return !skb->unreadable;
}

static inline void skb_mark_not_on_list(struct sk_buff *skb)
{
        skb->next = NULL;
}

static inline void skb_poison_list(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
        skb->next = SKB_LIST_POISON_NEXT;
#endif
}

/* Iterate through singly-linked GSO fragments of an skb. */
#define skb_list_walk_safe(first, skb, next_skb)                               \
        for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb);  \
             (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)

static inline void skb_list_del_init(struct sk_buff *skb)
{
        __list_del_entry(&skb->list);
        skb_mark_not_on_list(skb);
}

/**
 *        skb_queue_empty - check if a queue is empty
 *        @list: queue head
 *
 *        Returns true if the queue is empty, false otherwise.
 */
static inline int skb_queue_empty(const struct sk_buff_head *list)
{
        return list->next == (const struct sk_buff *) list;
}

/**
 *        skb_queue_empty_lockless - check if a queue is empty
 *        @list: queue head
 *
 *        Returns true if the queue is empty, false otherwise.
 *        This variant can be used in lockless contexts.
 */
static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
{
        return READ_ONCE(list->next) == (const struct sk_buff *) list;
}


/**
 *        skb_queue_is_last - check if skb is the last entry in the queue
 *        @list: queue head
 *        @skb: buffer
 *
 *        Returns true if @skb is the last buffer on the list.
 */
static inline bool skb_queue_is_last(const struct sk_buff_head *list,
                                     const struct sk_buff *skb)
{
        return skb->next == (const struct sk_buff *) list;
}

/**
 *        skb_queue_is_first - check if skb is the first entry in the queue
 *        @list: queue head
 *        @skb: buffer
 *
 *        Returns true if @skb is the first buffer on the list.
 */
static inline bool skb_queue_is_first(const struct sk_buff_head *list,
                                      const struct sk_buff *skb)
{
        return skb->prev == (const struct sk_buff *) list;
}

/**
 *        skb_queue_next - return the next packet in the queue
 *        @list: queue head
 *        @skb: current buffer
 *
 *        Return the next packet in @list after @skb.  It is only valid to
 *        call this if skb_queue_is_last() evaluates to false.
 */
static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
                                             const struct sk_buff *skb)
{
        /* This BUG_ON may seem severe, but if we just return then we
         * are going to dereference garbage.
         */
        BUG_ON(skb_queue_is_last(list, skb));
        return skb->next;
}

/**
 *        skb_queue_prev - return the prev packet in the queue
 *        @list: queue head
 *        @skb: current buffer
 *
 *        Return the prev packet in @list before @skb.  It is only valid to
 *        call this if skb_queue_is_first() evaluates to false.
 */
static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
                                             const struct sk_buff *skb)
{
        /* This BUG_ON may seem severe, but if we just return then we
         * are going to dereference garbage.
         */
        BUG_ON(skb_queue_is_first(list, skb));
        return skb->prev;
}

/**
 *        skb_get - reference buffer
 *        @skb: buffer to reference
 *
 *        Makes another reference to a socket buffer and returns a pointer
 *        to the buffer.
 */
static inline struct sk_buff *skb_get(struct sk_buff *skb)
{
        refcount_inc(&skb->users);
        return skb;
}

/*
 * If users == 1, we are the only owner and can avoid redundant atomic changes.
 */

/**
 *        skb_cloned - is the buffer a clone
 *        @skb: buffer to check
 *
 *        Returns true if the buffer was generated with skb_clone() and is
 *        one of multiple shared copies of the buffer. Cloned buffers are
 *        shared data so must not be written to under normal circumstances.
 */
static inline int skb_cloned(const struct sk_buff *skb)
{
        return skb->cloned &&
               (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
}

static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_cloned(skb))
                return pskb_expand_head(skb, 0, 0, pri);

        return 0;
}

/* This variant of skb_unclone() makes sure skb->truesize
 * and skb_end_offset() are not changed, whenever a new skb->head is needed.
 *
 * Indeed there is no guarantee that ksize(kmalloc(X)) == ksize(kmalloc(X))
 * when various debugging features are in place.
 */
int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri);
static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_cloned(skb))
                return __skb_unclone_keeptruesize(skb, pri);
        return 0;
}

/**
 *        skb_header_cloned - is the header a clone
 *        @skb: buffer to check
 *
 *        Returns true if modifying the header part of the buffer requires
 *        the data to be copied.
 */
static inline int skb_header_cloned(const struct sk_buff *skb)
{
        int dataref;

        if (!skb->cloned)
                return 0;

        dataref = atomic_read(&skb_shinfo(skb)->dataref);
        dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
        return dataref != 1;
}

static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));

        if (skb_header_cloned(skb))
                return pskb_expand_head(skb, 0, 0, pri);

        return 0;
}

/**
 * __skb_header_release() - allow clones to use the headroom
 * @skb: buffer to operate on
 *
 * See "DOC: dataref and headerless skbs".
 */
static inline void __skb_header_release(struct sk_buff *skb)
{
        skb->nohdr = 1;
        atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
}


/**
 *        skb_shared - is the buffer shared
 *        @skb: buffer to check
 *
 *        Returns true if more than one person has a reference to this
 *        buffer.
 */
static inline int skb_shared(const struct sk_buff *skb)
{
        return refcount_read(&skb->users) != 1;
}

/**
 *        skb_share_check - check if buffer is shared and if so clone it
 *        @skb: buffer to check
 *        @pri: priority for memory allocation
 *
 *        If the buffer is shared the buffer is cloned and the old copy
 *        drops a reference. A new clone with a single reference is returned.
 *        If the buffer is not shared the original buffer is returned. When
 *        being called from interrupt status or with spinlocks held pri must
 *        be GFP_ATOMIC.
 *
 *        NULL is returned on a memory allocation failure.
 */
static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));
        if (skb_shared(skb)) {
                struct sk_buff *nskb = skb_clone(skb, pri);

                if (likely(nskb))
                        consume_skb(skb);
                else
                        kfree_skb(skb);
                skb = nskb;
        }
        return skb;
}

/*
 *        Copy shared buffers into a new sk_buff. We effectively do COW on
 *        packets to handle cases where we have a local reader and forward
 *        and a couple of other messy ones. The normal one is tcpdumping
 *        a packet that's being forwarded.
 */

/**
 *        skb_unshare - make a copy of a shared buffer
 *        @skb: buffer to check
 *        @pri: priority for memory allocation
 *
 *        If the socket buffer is a clone then this function creates a new
 *        copy of the data, drops a reference count on the old copy and returns
 *        the new copy with the reference count at 1. If the buffer is not a clone
 *        the original buffer is returned. When called with a spinlock held or
 *        from interrupt state @pri must be %GFP_ATOMIC
 *
 *        %NULL is returned on a memory allocation failure.
 */
static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
                                          gfp_t pri)
{
        might_sleep_if(gfpflags_allow_blocking(pri));
        if (skb_cloned(skb)) {
                struct sk_buff *nskb = skb_copy(skb, pri);

                /* Free our shared copy */
                if (likely(nskb))
                        consume_skb(skb);
                else
                        kfree_skb(skb);
                skb = nskb;
        }
        return skb;
}

/**
 *        skb_peek - peek at the head of an &sk_buff_head
 *        @list_: list to peek at
 *
 *        Peek an &sk_buff. Unlike most other operations you _MUST_
 *        be careful with this one. A peek leaves the buffer on the
 *        list and someone else may run off with it. You must hold
 *        the appropriate locks or have a private queue to do this.
 *
 *        Returns %NULL for an empty list or a pointer to the head element.
 *        The reference count is not incremented and the reference is therefore
 *        volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
{
        struct sk_buff *skb = list_->next;

        if (skb == (struct sk_buff *)list_)
                skb = NULL;
        return skb;
}

/**
 *        __skb_peek - peek at the head of a non-empty &sk_buff_head
 *        @list_: list to peek at
 *
 *        Like skb_peek(), but the caller knows that the list is not empty.
 */
static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
{
        return list_->next;
}

/**
 *        skb_peek_next - peek skb following the given one from a queue
 *        @skb: skb to start from
 *        @list_: list to peek at
 *
 *        Returns %NULL when the end of the list is met or a pointer to the
 *        next element. The reference count is not incremented and the
 *        reference is therefore volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
                const struct sk_buff_head *list_)
{
        struct sk_buff *next = skb->next;

        if (next == (struct sk_buff *)list_)
                next = NULL;
        return next;
}

/**
 *        skb_peek_tail - peek at the tail of an &sk_buff_head
 *        @list_: list to peek at
 *
 *        Peek an &sk_buff. Unlike most other operations you _MUST_
 *        be careful with this one. A peek leaves the buffer on the
 *        list and someone else may run off with it. You must hold
 *        the appropriate locks or have a private queue to do this.
 *
 *        Returns %NULL for an empty list or a pointer to the tail element.
 *        The reference count is not incremented and the reference is therefore
 *        volatile. Use with caution.
 */
static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
{
        struct sk_buff *skb = READ_ONCE(list_->prev);

        if (skb == (struct sk_buff *)list_)
                skb = NULL;
        return skb;

}

/**
 *        skb_queue_len        - get queue length
 *        @list_: list to measure
 *
 *        Return the length of an &sk_buff queue.
 */
static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
{
        return list_->qlen;
}

/**
 *        skb_queue_len_lockless        - get queue length
 *        @list_: list to measure
 *
 *        Return the length of an &sk_buff queue.
 *        This variant can be used in lockless contexts.
 */
static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
{
        return READ_ONCE(list_->qlen);
}

/**
 *        __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
 *        @list: queue to initialize
 *
 *        This initializes only the list and queue length aspects of
 *        an sk_buff_head object.  This allows to initialize the list
 *        aspects of an sk_buff_head without reinitializing things like
 *        the spinlock.  It can also be used for on-stack sk_buff_head
 *        objects where the spinlock is known to not be used.
 */
static inline void __skb_queue_head_init(struct sk_buff_head *list)
{
        list->prev = list->next = (struct sk_buff *)list;
        list->qlen = 0;
}

/*
 * This function creates a split out lock class for each invocation;
 * this is needed for now since a whole lot of users of the skb-queue
 * infrastructure in drivers have different locking usage (in hardirq)
 * than the networking core (in softirq only). In the long run either the
 * network layer or drivers should need annotation to consolidate the
 * main types of usage into 3 classes.
 */
static inline void skb_queue_head_init(struct sk_buff_head *list)
{
        spin_lock_init(&list->lock);
        __skb_queue_head_init(list);
}

static inline void skb_queue_head_init_class(struct sk_buff_head *list,
                struct lock_class_key *class)
{
        skb_queue_head_init(list);
        lockdep_set_class(&list->lock, class);
}

/*
 *        Insert an sk_buff on a list.
 *
 *        The "__skb_xxxx()" functions are the non-atomic ones that
 *        can only be called with interrupts disabled.
 */
static inline void __skb_insert(struct sk_buff *newsk,
                                struct sk_buff *prev, struct sk_buff *next,
                                struct sk_buff_head *list)
{
        /* See skb_queue_empty_lockless() and skb_peek_tail()
         * for the opposite READ_ONCE()
         */
        WRITE_ONCE(newsk->next, next);
        WRITE_ONCE(newsk->prev, prev);
        WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk);
        WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk);
        WRITE_ONCE(list->qlen, list->qlen + 1);
}

static inline void __skb_queue_splice(const struct sk_buff_head *list,
                                      struct sk_buff *prev,
                                      struct sk_buff *next)
{
        struct sk_buff *first = list->next;
        struct sk_buff *last = list->prev;

        WRITE_ONCE(first->prev, prev);
        WRITE_ONCE(prev->next, first);

        WRITE_ONCE(last->next, next);
        WRITE_ONCE(next->prev, last);
}

/**
 *        skb_queue_splice - join two skb lists, this is designed for stacks
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 */
static inline void skb_queue_splice(const struct sk_buff_head *list,
                                    struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, (struct sk_buff *) head, head->next);
                head->qlen += list->qlen;
        }
}

/**
 *        skb_queue_splice_init - join two skb lists and reinitialise the emptied list
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 *
 *        The list at @list is reinitialised
 */
static inline void skb_queue_splice_init(struct sk_buff_head *list,
                                         struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, (struct sk_buff *) head, head->next);
                head->qlen += list->qlen;
                __skb_queue_head_init(list);
        }
}

/**
 *        skb_queue_splice_tail - join two skb lists, each list being a queue
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 */
static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
                                         struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
                head->qlen += list->qlen;
        }
}

/**
 *        skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
 *        @list: the new list to add
 *        @head: the place to add it in the first list
 *
 *        Each of the lists is a queue.
 *        The list at @list is reinitialised
 */
static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
                                              struct sk_buff_head *head)
{
        if (!skb_queue_empty(list)) {
                __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
                head->qlen += list->qlen;
                __skb_queue_head_init(list);
        }
}

/**
 *        __skb_queue_after - queue a buffer at the list head
 *        @list: list to use
 *        @prev: place after this buffer
 *        @newsk: buffer to queue
 *
 *        Queue a buffer int the middle of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_after(struct sk_buff_head *list,
                                     struct sk_buff *prev,
                                     struct sk_buff *newsk)
{
        __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list);
}

void skb_append(struct sk_buff *old, struct sk_buff *newsk,
                struct sk_buff_head *list);

static inline void __skb_queue_before(struct sk_buff_head *list,
                                      struct sk_buff *next,
                                      struct sk_buff *newsk)
{
        __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list);
}

/**
 *        __skb_queue_head - queue a buffer at the list head
 *        @list: list to use
 *        @newsk: buffer to queue
 *
 *        Queue a buffer at the start of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_head(struct sk_buff_head *list,
                                    struct sk_buff *newsk)
{
        __skb_queue_after(list, (struct sk_buff *)list, newsk);
}
void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);

/**
 *        __skb_queue_tail - queue a buffer at the list tail
 *        @list: list to use
 *        @newsk: buffer to queue
 *
 *        Queue a buffer at the end of a list. This function takes no locks
 *        and you must therefore hold required locks before calling it.
 *
 *        A buffer cannot be placed on two lists at the same time.
 */
static inline void __skb_queue_tail(struct sk_buff_head *list,
                                   struct sk_buff *newsk)
{
        __skb_queue_before(list, (struct sk_buff *)list, newsk);
}
void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);

/*
 * remove sk_buff from list. _Must_ be called atomically, and with
 * the list known..
 */
void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
{
        struct sk_buff *next, *prev;

        WRITE_ONCE(list->qlen, list->qlen - 1);
        next           = skb->next;
        prev           = skb->prev;
        skb->next  = skb->prev = NULL;
        WRITE_ONCE(next->prev, prev);
        WRITE_ONCE(prev->next, next);
}

/**
 *        __skb_dequeue - remove from the head of the queue
 *        @list: list to dequeue from
 *
 *        Remove the head of the list. This function does not take any locks
 *        so must be used with appropriate locks held only. The head item is
 *        returned or %NULL if the list is empty.
 */
static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
{
        struct sk_buff *skb = skb_peek(list);
        if (skb)
                __skb_unlink(skb, list);
        return skb;
}
struct sk_buff *skb_dequeue(struct sk_buff_head *list);

/**
 *        __skb_dequeue_tail - remove from the tail of the queue
 *        @list: list to dequeue from
 *
 *        Remove the tail of the list. This function does not take any locks
 *        so must be used with appropriate locks held only. The tail item is
 *        returned or %NULL if the list is empty.
 */
static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
{
        struct sk_buff *skb = skb_peek_tail(list);
        if (skb)
                __skb_unlink(skb, list);
        return skb;
}
struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);


static inline bool skb_is_nonlinear(const struct sk_buff *skb)
{
        return skb->data_len;
}

static inline unsigned int skb_headlen(const struct sk_buff *skb)
{
        return skb->len - skb->data_len;
}

static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
{
        unsigned int i, len = 0;

        for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
                len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
        return len;
}

static inline unsigned int skb_pagelen(const struct sk_buff *skb)
{
        return skb_headlen(skb) + __skb_pagelen(skb);
}

static inline void skb_frag_fill_netmem_desc(skb_frag_t *frag,
                                             netmem_ref netmem, int off,
                                             int size)
{
        frag->netmem = netmem;
        frag->offset = off;
        skb_frag_size_set(frag, size);
}

static inline void skb_frag_fill_page_desc(skb_frag_t *frag,
                                           struct page *page,
                                           int off, int size)
{
        skb_frag_fill_netmem_desc(frag, page_to_netmem(page), off, size);
}

static inline void __skb_fill_netmem_desc_noacc(struct skb_shared_info *shinfo,
                                                int i, netmem_ref netmem,
                                                int off, int size)
{
        skb_frag_t *frag = &shinfo->frags[i];

        skb_frag_fill_netmem_desc(frag, netmem, off, size);
}

static inline void __skb_fill_page_desc_noacc(struct skb_shared_info *shinfo,
                                              int i, struct page *page,
                                              int off, int size)
{
        __skb_fill_netmem_desc_noacc(shinfo, i, page_to_netmem(page), off,
                                     size);
}

/**
 * skb_len_add - adds a number to len fields of skb
 * @skb: buffer to add len to
 * @delta: number of bytes to add
 */
static inline void skb_len_add(struct sk_buff *skb, int delta)
{
        skb->len += delta;
        skb->data_len += delta;
        skb->truesize += delta;
}

/**
 * __skb_fill_netmem_desc - initialise a fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: fragment index to initialise
 * @netmem: the netmem to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * Initialises the @i'th fragment of @skb to point to &size bytes at
 * offset @off within @page.
 *
 * Does not take any additional reference on the fragment.
 */
static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i,
                                          netmem_ref netmem, int off, int size)
{
        struct page *page;

        __skb_fill_netmem_desc_noacc(skb_shinfo(skb), i, netmem, off, size);

        if (netmem_is_net_iov(netmem)) {
                skb->unreadable = true;
                return;
        }

        page = netmem_to_page(netmem);

        /* Propagate page pfmemalloc to the skb if we can. The problem is
         * that not all callers have unique ownership of the page but rely
         * on page_is_pfmemalloc doing the right thing(tm).
         */
        page = compound_head(page);
        if (page_is_pfmemalloc(page))
                skb->pfmemalloc = true;
}

static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
                                        struct page *page, int off, int size)
{
        __skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}

static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i,
                                        netmem_ref netmem, int off, int size)
{
        __skb_fill_netmem_desc(skb, i, netmem, off, size);
        skb_shinfo(skb)->nr_frags = i + 1;
}

/**
 * skb_fill_page_desc - initialise a paged fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: paged fragment index to initialise
 * @page: the page to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
 * @skb to point to @size bytes at offset @off within @page. In
 * addition updates @skb such that @i is the last fragment.
 *
 * Does not take any additional reference on the fragment.
 */
static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
                                      struct page *page, int off, int size)
{
        skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size);
}

/**
 * skb_fill_page_desc_noacc - initialise a paged fragment in an skb
 * @skb: buffer containing fragment to be initialised
 * @i: paged fragment index to initialise
 * @page: the page to use for this fragment
 * @off: the offset to the data with @page
 * @size: the length of the data
 *
 * Variant of skb_fill_page_desc() which does not deal with
 * pfmemalloc, if page is not owned by us.
 */
static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i,
                                            struct page *page, int off,
                                            int size)
{
        struct skb_shared_info *shinfo = skb_shinfo(skb);

        __skb_fill_page_desc_noacc(shinfo, i, page, off, size);
        shinfo->nr_frags = i + 1;
}

void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem,
                            int off, int size, unsigned int truesize);

static inline void skb_add_rx_frag(struct sk_buff *skb, int i,
                                   struct page *page, int off, int size,
                                   unsigned int truesize)
{
        skb_add_rx_frag_netmem(skb, i, page_to_netmem(page), off, size,
                               truesize);
}

void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
                          unsigned int truesize);

#define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))

#ifdef NET_SKBUFF_DATA_USES_OFFSET
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
        return skb->head + skb->tail;
}

static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
        skb->tail = skb->data - skb->head;
}

static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
        skb_reset_tail_pointer(skb);
        skb->tail += offset;
}

#else /* NET_SKBUFF_DATA_USES_OFFSET */
static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
{
        return skb->tail;
}

static inline void skb_reset_tail_pointer(struct sk_buff *skb)
{
        skb->tail = skb->data;
}

static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
{
        skb->tail = skb->data + offset;
}

#endif /* NET_SKBUFF_DATA_USES_OFFSET */

static inline void skb_assert_len(struct sk_buff *skb)
{
#ifdef CONFIG_DEBUG_NET
        if (WARN_ONCE(!skb->len, "%s\n", __func__))
                DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
#endif /* CONFIG_DEBUG_NET */
}

#if defined(CONFIG_FAIL_SKB_REALLOC)
void skb_might_realloc(struct sk_buff *skb);
#else
static inline void skb_might_realloc(struct sk_buff *skb) {}
#endif

/*
 *        Add data to an sk_buff
 */
void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
void *skb_put(struct sk_buff *skb, unsigned int len);
static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
{
        void *tmp = skb_tail_pointer(skb);
        SKB_LINEAR_ASSERT(skb);
        skb->tail += len;
        skb->len  += len;
        return tmp;
}

static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
{
        void *tmp = __skb_put(skb, len);

        memset(tmp, 0, len);
        return tmp;
}

static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
                                   unsigned int len)
{
        void *tmp = __skb_put(skb, len);

        memcpy(tmp, data, len);
        return tmp;
}

static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
{
        *(u8 *)__skb_put(skb, 1) = val;
}

static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
{
        void *tmp = skb_put(skb, len);

        memset(tmp, 0, len);

        return tmp;
}

static inline void *skb_put_data(struct sk_buff *skb, const void *data,
                                 unsigned int len)
{
        void *tmp = skb_put(skb, len);

        memcpy(tmp, data, len);

        return tmp;
}

static inline void skb_put_u8(struct sk_buff *skb, u8 val)
{
        *(u8 *)skb_put(skb, 1) = val;
}

void *skb_push(struct sk_buff *skb, unsigned int len);
static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);

        skb->data -= len;
        skb->len  += len;
        return skb->data;
}

void *skb_pull(struct sk_buff *skb, unsigned int len);
static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);

        skb->len -= len;
        if (unlikely(skb->len < skb->data_len)) {
#if defined(CONFIG_DEBUG_NET)
                skb->len += len;
                pr_err("__skb_pull(len=%u)\n", len);
                skb_dump(KERN_ERR, skb, false);
#endif
                BUG();
        }
        return skb->data += len;
}

static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
{
        return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
}

void *skb_pull_data(struct sk_buff *skb, size_t len);

void *__pskb_pull_tail(struct sk_buff *skb, int delta);

static inline enum skb_drop_reason
pskb_may_pull_reason(struct sk_buff *skb, unsigned int len)
{
        DEBUG_NET_WARN_ON_ONCE(len > INT_MAX);
        skb_might_realloc(skb);

        if (likely(len <= skb_headlen(skb)))
                return SKB_NOT_DROPPED_YET;

        if (unlikely(len > skb->len))
                return SKB_DROP_REASON_PKT_TOO_SMALL;

        if (unlikely(!__pskb_pull_tail(skb, len - skb_headlen(skb))))
                return SKB_DROP_REASON_NOMEM;

        return SKB_NOT_DROPPED_YET;
}

static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
{
        return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
}

static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
{
        if (!pskb_may_pull(skb, len))
                return NULL;

        skb->len -= len;
        return skb->data += len;
}

void skb_condense(struct sk_buff *skb);

/**
 *        skb_headroom - bytes at buffer head
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the head of an &sk_buff.
 */
static inline unsigned int skb_headroom(const struct sk_buff *skb)
{
        return skb->data - skb->head;
}

/**
 *        skb_tailroom - bytes at buffer end
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the tail of an sk_buff
 */
static inline int skb_tailroom(const struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
}

/**
 *        skb_availroom - bytes at buffer end
 *        @skb: buffer to check
 *
 *        Return the number of bytes of free space at the tail of an sk_buff
 *        allocated by sk_stream_alloc()
 */
static inline int skb_availroom(const struct sk_buff *skb)
{
        if (skb_is_nonlinear(skb))
                return 0;

        return skb->end - skb->tail - skb->reserved_tailroom;
}

/**
 *        skb_reserve - adjust headroom
 *        @skb: buffer to alter
 *        @len: bytes to move
 *
 *        Increase the headroom of an empty &sk_buff by reducing the tail
 *        room. This is only allowed for an empty buffer.
 */
static inline void skb_reserve(struct sk_buff *skb, int len)
{
        skb->data += len;
        skb->tail += len;
}

/**
 *        skb_tailroom_reserve - adjust reserved_tailroom
 *        @skb: buffer to alter
 *        @mtu: maximum amount of headlen permitted
 *        @needed_tailroom: minimum amount of reserved_tailroom
 *
 *        Set reserved_tailroom so that headlen can be as large as possible but
 *        not larger than mtu and tailroom cannot be smaller than
 *        needed_tailroom.
 *        The required headroom should already have been reserved before using
 *        this function.
 */
static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
                                        unsigned int needed_tailroom)
{
        SKB_LINEAR_ASSERT(skb);
        if (mtu < skb_tailroom(skb) - needed_tailroom)
                /* use at most mtu */
                skb->reserved_tailroom = skb_tailroom(skb) - mtu;
        else
                /* use up to all available space */
                skb->reserved_tailroom = needed_tailroom;
}

#define ENCAP_TYPE_ETHER        0
#define ENCAP_TYPE_IPPROTO        1

static inline void skb_set_inner_protocol(struct sk_buff *skb,
                                          __be16 protocol)
{
        skb->inner_protocol = protocol;
        skb->inner_protocol_type = ENCAP_TYPE_ETHER;
}

static inline void skb_set_inner_ipproto(struct sk_buff *skb,
                                         __u8 ipproto)
{
        skb->inner_ipproto = ipproto;
        skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
}

static inline void skb_reset_inner_headers(struct sk_buff *skb)
{
        skb->inner_mac_header = skb->mac_header;
        skb->inner_network_header = skb->network_header;
        skb->inner_transport_header = skb->transport_header;
}

static inline int skb_mac_header_was_set(const struct sk_buff *skb)
{
        return skb->mac_header != (typeof(skb->mac_header))~0U;
}

static inline void skb_reset_mac_len(struct sk_buff *skb)
{
        if (!skb_mac_header_was_set(skb)) {
                DEBUG_NET_WARN_ON_ONCE(1);
                skb->mac_len = 0;
        } else {
                skb->mac_len = skb->network_header - skb->mac_header;
        }
}

static inline unsigned char *skb_inner_transport_header(const struct sk_buff
                                                        *skb)
{
        return skb->head + skb->inner_transport_header;
}

static inline int skb_inner_transport_offset(const struct sk_buff *skb)
{
        return skb_inner_transport_header(skb) - skb->data;
}

static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_transport_header))offset);
        skb->inner_transport_header = offset;
}

static inline void skb_set_inner_transport_header(struct sk_buff *skb,
                                                   const int offset)
{
        skb_reset_inner_transport_header(skb);
        skb->inner_transport_header += offset;
}

static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
{
        return skb->head + skb->inner_network_header;
}

static inline void skb_reset_inner_network_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_network_header))offset);
        skb->inner_network_header = offset;
}

static inline void skb_set_inner_network_header(struct sk_buff *skb,
                                                const int offset)
{
        skb_reset_inner_network_header(skb);
        skb->inner_network_header += offset;
}

static inline bool skb_inner_network_header_was_set(const struct sk_buff *skb)
{
        return skb->inner_network_header > 0;
}

static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
{
        return skb->head + skb->inner_mac_header;
}

static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->inner_mac_header))offset);
        skb->inner_mac_header = offset;
}

static inline void skb_set_inner_mac_header(struct sk_buff *skb,
                                            const int offset)
{
        skb_reset_inner_mac_header(skb);
        skb->inner_mac_header += offset;
}
static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
{
        return skb->transport_header != (typeof(skb->transport_header))~0U;
}

static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
        return skb->head + skb->transport_header;
}

static inline void skb_reset_transport_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->transport_header))offset);
        skb->transport_header = offset;
}

static inline void skb_set_transport_header(struct sk_buff *skb,
                                            const int offset)
{
        skb_reset_transport_header(skb);
        skb->transport_header += offset;
}

static inline unsigned char *skb_network_header(const struct sk_buff *skb)
{
        return skb->head + skb->network_header;
}

static inline void skb_reset_network_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->network_header))offset);
        skb->network_header = offset;
}

static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
{
        skb_reset_network_header(skb);
        skb->network_header += offset;
}

static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
        return skb->head + skb->mac_header;
}

static inline int skb_mac_offset(const struct sk_buff *skb)
{
        return skb_mac_header(skb) - skb->data;
}

static inline u32 skb_mac_header_len(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_mac_header_was_set(skb));
        return skb->network_header - skb->mac_header;
}

static inline void skb_unset_mac_header(struct sk_buff *skb)
{
        skb->mac_header = (typeof(skb->mac_header))~0U;
}

static inline void skb_reset_mac_header(struct sk_buff *skb)
{
        long offset = skb->data - skb->head;

        DEBUG_NET_WARN_ON_ONCE(offset != (typeof(skb->mac_header))offset);
        skb->mac_header = offset;
}

static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
{
        skb_reset_mac_header(skb);
        skb->mac_header += offset;
}

static inline void skb_pop_mac_header(struct sk_buff *skb)
{
        skb->mac_header = skb->network_header;
}

static inline void skb_probe_transport_header(struct sk_buff *skb)
{
        struct flow_keys_basic keys;

        if (skb_transport_header_was_set(skb))
                return;

        if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
                                             NULL, 0, 0, 0, 0))
                skb_set_transport_header(skb, keys.control.thoff);
}

static inline void skb_mac_header_rebuild(struct sk_buff *skb)
{
        if (skb_mac_header_was_set(skb)) {
                const unsigned char *old_mac = skb_mac_header(skb);

                skb_set_mac_header(skb, -skb->mac_len);
                memmove(skb_mac_header(skb), old_mac, skb->mac_len);
        }
}

/* Move the full mac header up to current network_header.
 * Leaves skb->data pointing at offset skb->mac_len into the mac_header.
 * Must be provided the complete mac header length.
 */
static inline void skb_mac_header_rebuild_full(struct sk_buff *skb, u32 full_mac_len)
{
        if (skb_mac_header_was_set(skb)) {
                const unsigned char *old_mac = skb_mac_header(skb);

                skb_set_mac_header(skb, -full_mac_len);
                memmove(skb_mac_header(skb), old_mac, full_mac_len);
                __skb_push(skb, full_mac_len - skb->mac_len);
        }
}

static inline int skb_checksum_start_offset(const struct sk_buff *skb)
{
        return skb->csum_start - skb_headroom(skb);
}

static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
{
        return skb->head + skb->csum_start;
}

static inline int skb_transport_offset(const struct sk_buff *skb)
{
        return skb_transport_header(skb) - skb->data;
}

static inline u32 skb_network_header_len(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb));
        return skb->transport_header - skb->network_header;
}

static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
{
        return skb->inner_transport_header - skb->inner_network_header;
}

static inline int skb_network_offset(const struct sk_buff *skb)
{
        return skb_network_header(skb) - skb->data;
}

static inline int skb_inner_network_offset(const struct sk_buff *skb)
{
        return skb_inner_network_header(skb) - skb->data;
}

static inline enum skb_drop_reason
pskb_network_may_pull_reason(struct sk_buff *skb, unsigned int len)
{
        return pskb_may_pull_reason(skb, skb_network_offset(skb) + len);
}

static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
{
        return pskb_network_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET;
}

/*
 * CPUs often take a performance hit when accessing unaligned memory
 * locations. The actual performance hit varies, it can be small if the
 * hardware handles it or large if we have to take an exception and fix it
 * in software.
 *
 * Since an ethernet header is 14 bytes network drivers often end up with
 * the IP header at an unaligned offset. The IP header can be aligned by
 * shifting the start of the packet by 2 bytes. Drivers should do this
 * with:
 *
 * skb_reserve(skb, NET_IP_ALIGN);
 *
 * The downside to this alignment of the IP header is that the DMA is now
 * unaligned. On some architectures the cost of an unaligned DMA is high
 * and this cost outweighs the gains made by aligning the IP header.
 *
 * Since this trade off varies between architectures, we allow NET_IP_ALIGN
 * to be overridden.
 */
#ifndef NET_IP_ALIGN
#define NET_IP_ALIGN        2
#endif

/*
 * The networking layer reserves some headroom in skb data (via
 * dev_alloc_skb). This is used to avoid having to reallocate skb data when
 * the header has to grow. In the default case, if the header has to grow
 * 32 bytes or less we avoid the reallocation.
 *
 * Unfortunately this headroom changes the DMA alignment of the resulting
 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
 * on some architectures. An architecture can override this value,
 * perhaps setting it to a cacheline in size (since that will maintain
 * cacheline alignment of the DMA). It must be a power of 2.
 *
 * Various parts of the networking layer expect at least 32 bytes of
 * headroom, you should not reduce this.
 *
 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
 * to reduce average number of cache lines per packet.
 * get_rps_cpu() for example only access one 64 bytes aligned block :
 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
 */
#ifndef NET_SKB_PAD
#define NET_SKB_PAD        max(32, L1_CACHE_BYTES)
#endif

int ___pskb_trim(struct sk_buff *skb, unsigned int len);

static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
{
        if (WARN_ON(skb_is_nonlinear(skb)))
                return;
        skb->len = len;
        skb_set_tail_pointer(skb, len);
}

static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
{
        __skb_set_length(skb, len);
}

void skb_trim(struct sk_buff *skb, unsigned int len);

static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
{
        if (skb->data_len)
                return ___pskb_trim(skb, len);
        __skb_trim(skb, len);
        return 0;
}

static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
{
        skb_might_realloc(skb);
        return (len < skb->len) ? __pskb_trim(skb, len) : 0;
}

/**
 *        pskb_trim_unique - remove end from a paged unique (not cloned) buffer
 *        @skb: buffer to alter
 *        @len: new length
 *
 *        This is identical to pskb_trim except that the caller knows that
 *        the skb is not cloned so we should never get an error due to out-
 *        of-memory.
 */
static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
{
        int err = pskb_trim(skb, len);
        BUG_ON(err);
}

static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
{
        unsigned int diff = len - skb->len;

        if (skb_tailroom(skb) < diff) {
                int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
                                           GFP_ATOMIC);
                if (ret)
                        return ret;
        }
        __skb_set_length(skb, len);
        return 0;
}

/**
 *        skb_orphan - orphan a buffer
 *        @skb: buffer to orphan
 *
 *        If a buffer currently has an owner then we call the owner's
 *        destructor function and make the @skb unowned. The buffer continues
 *        to exist but is no longer charged to its former owner.
 */
static inline void skb_orphan(struct sk_buff *skb)
{
        if (skb->destructor) {
                skb->destructor(skb);
                skb->destructor = NULL;
                skb->sk                = NULL;
        } else {
                BUG_ON(skb->sk);
        }
}

/**
 *        skb_orphan_frags - orphan the frags contained in a buffer
 *        @skb: buffer to orphan frags from
 *        @gfp_mask: allocation mask for replacement pages
 *
 *        For each frag in the SKB which needs a destructor (i.e. has an
 *        owner) create a copy of that frag and release the original
 *        page by calling the destructor.
 */
static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
{
        if (likely(!skb_zcopy(skb)))
                return 0;
        if (skb_shinfo(skb)->flags & SKBFL_DONT_ORPHAN)
                return 0;
        return skb_copy_ubufs(skb, gfp_mask);
}

/* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
{
        if (likely(!skb_zcopy(skb)))
                return 0;
        return skb_copy_ubufs(skb, gfp_mask);
}

/**
 *        __skb_queue_purge_reason - empty a list
 *        @list: list to empty
 *        @reason: drop reason
 *
 *        Delete all buffers on an &sk_buff list. Each buffer is removed from
 *        the list and one reference dropped. This function does not take the
 *        list lock and the caller must hold the relevant locks to use it.
 */
static inline void __skb_queue_purge_reason(struct sk_buff_head *list,
                                            enum skb_drop_reason reason)
{
        struct sk_buff *skb;

        while ((skb = __skb_dequeue(list)) != NULL)
                kfree_skb_reason(skb, reason);
}

static inline void __skb_queue_purge(struct sk_buff_head *list)
{
        __skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}

void skb_queue_purge_reason(struct sk_buff_head *list,
                            enum skb_drop_reason reason);

static inline void skb_queue_purge(struct sk_buff_head *list)
{
        skb_queue_purge_reason(list, SKB_DROP_REASON_QUEUE_PURGE);
}

unsigned int skb_rbtree_purge(struct rb_root *root);
void skb_errqueue_purge(struct sk_buff_head *list);

void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);

/**
 * netdev_alloc_frag - allocate a page fragment
 * @fragsz: fragment size
 *
 * Allocates a frag from a page for receive buffer.
 * Uses GFP_ATOMIC allocations.
 */
static inline void *netdev_alloc_frag(unsigned int fragsz)
{
        return __netdev_alloc_frag_align(fragsz, ~0u);
}

static inline void *netdev_alloc_frag_align(unsigned int fragsz,
                                            unsigned int align)
{
        WARN_ON_ONCE(!is_power_of_2(align));
        return __netdev_alloc_frag_align(fragsz, -align);
}

struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
                                   gfp_t gfp_mask);

/**
 *        netdev_alloc_skb - allocate an skbuff for rx on a specific device
 *        @dev: network device to receive on
 *        @length: length to allocate
 *
 *        Allocate a new &sk_buff and assign it a usage count of one. The
 *        buffer has unspecified headroom built in. Users should allocate
 *        the headroom they think they need without accounting for the
 *        built in space. The built in space is used for optimisations.
 *
 *        %NULL is returned if there is no free memory. Although this function
 *        allocates memory it can be called from an interrupt.
 */
static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
                                               unsigned int length)
{
        return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
}

/* legacy helper around __netdev_alloc_skb() */
static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
                                              gfp_t gfp_mask)
{
        return __netdev_alloc_skb(NULL, length, gfp_mask);
}

/* legacy helper around netdev_alloc_skb() */
static inline struct sk_buff *dev_alloc_skb(unsigned int length)
{
        return netdev_alloc_skb(NULL, length);
}


static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
                unsigned int length, gfp_t gfp)
{
        struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);

        if (NET_IP_ALIGN && skb)
                skb_reserve(skb, NET_IP_ALIGN);
        return skb;
}

static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
                unsigned int length)
{
        return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
}

static inline void skb_free_frag(void *addr)
{
        page_frag_free(addr);
}

void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask);

static inline void *napi_alloc_frag(unsigned int fragsz)
{
        return __napi_alloc_frag_align(fragsz, ~0u);
}

static inline void *napi_alloc_frag_align(unsigned int fragsz,
                                          unsigned int align)
{
        WARN_ON_ONCE(!is_power_of_2(align));
        return __napi_alloc_frag_align(fragsz, -align);
}

struct sk_buff *napi_alloc_skb(struct napi_struct *napi, unsigned int length);
void napi_consume_skb(struct sk_buff *skb, int budget);

void napi_skb_free_stolen_head(struct sk_buff *skb);
void __napi_kfree_skb(struct sk_buff *skb, enum skb_drop_reason reason);

/**
 * __dev_alloc_pages - allocate page for network Rx
 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
 * @order: size of the allocation
 *
 * Allocate a new page.
 *
 * %NULL is returned if there is no free memory.
*/
static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask,
                                             unsigned int order)
{
        /* This piece of code contains several assumptions.
         * 1.  This is for device Rx, therefore a cold page is preferred.
         * 2.  The expectation is the user wants a compound page.
         * 3.  If requesting a order 0 page it will not be compound
         *     due to the check to see if order has a value in prep_new_page
         * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
         *     code in gfp_to_alloc_flags that should be enforcing this.
         */
        gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;

        return alloc_pages_node_noprof(NUMA_NO_NODE, gfp_mask, order);
}
#define __dev_alloc_pages(...)        alloc_hooks(__dev_alloc_pages_noprof(__VA_ARGS__))

/*
 * This specialized allocator has to be a macro for its allocations to be
 * accounted separately (to have a separate alloc_tag).
 */
#define dev_alloc_pages(_order) __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, _order)

/**
 * __dev_alloc_page - allocate a page for network Rx
 * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
 *
 * Allocate a new page.
 *
 * %NULL is returned if there is no free memory.
 */
static inline struct page *__dev_alloc_page_noprof(gfp_t gfp_mask)
{
        return __dev_alloc_pages_noprof(gfp_mask, 0);
}
#define __dev_alloc_page(...)        alloc_hooks(__dev_alloc_page_noprof(__VA_ARGS__))

/*
 * This specialized allocator has to be a macro for its allocations to be
 * accounted separately (to have a separate alloc_tag).
 */
#define dev_alloc_page()        dev_alloc_pages(0)

/**
 * dev_page_is_reusable - check whether a page can be reused for network Rx
 * @page: the page to test
 *
 * A page shouldn't be considered for reusing/recycling if it was allocated
 * under memory pressure or at a distant memory node.
 *
 * Returns false if this page should be returned to page allocator, true
 * otherwise.
 */
static inline bool dev_page_is_reusable(const struct page *page)
{
        return likely(page_to_nid(page) == numa_mem_id() &&
                      !page_is_pfmemalloc(page));
}

/**
 *        skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
 *        @page: The page that was allocated from skb_alloc_page
 *        @skb: The skb that may need pfmemalloc set
 */
static inline void skb_propagate_pfmemalloc(const struct page *page,
                                            struct sk_buff *skb)
{
        if (page_is_pfmemalloc(page))
                skb->pfmemalloc = true;
}

/**
 * skb_frag_off() - Returns the offset of a skb fragment
 * @frag: the paged fragment
 */
static inline unsigned int skb_frag_off(const skb_frag_t *frag)
{
        return frag->offset;
}

/**
 * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
 * @frag: skb fragment
 * @delta: value to add
 */
static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
{
        frag->offset += delta;
}

/**
 * skb_frag_off_set() - Sets the offset of a skb fragment
 * @frag: skb fragment
 * @offset: offset of fragment
 */
static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
{
        frag->offset = offset;
}

/**
 * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
 * @fragto: skb fragment where offset is set
 * @fragfrom: skb fragment offset is copied from
 */
static inline void skb_frag_off_copy(skb_frag_t *fragto,
                                     const skb_frag_t *fragfrom)
{
        fragto->offset = fragfrom->offset;
}

/* Return: true if the skb_frag contains a net_iov. */
static inline bool skb_frag_is_net_iov(const skb_frag_t *frag)
{
        return netmem_is_net_iov(frag->netmem);
}

/**
 * skb_frag_net_iov - retrieve the net_iov referred to by fragment
 * @frag: the fragment
 *
 * Return: the &struct net_iov associated with @frag. Returns NULL if this
 * frag has no associated net_iov.
 */
static inline struct net_iov *skb_frag_net_iov(const skb_frag_t *frag)
{
        if (!skb_frag_is_net_iov(frag))
                return NULL;

        return netmem_to_net_iov(frag->netmem);
}

/**
 * skb_frag_page - retrieve the page referred to by a paged fragment
 * @frag: the paged fragment
 *
 * Return: the &struct page associated with @frag. Returns NULL if this frag
 * has no associated page.
 */
static inline struct page *skb_frag_page(const skb_frag_t *frag)
{
        if (skb_frag_is_net_iov(frag))
                return NULL;

        return netmem_to_page(frag->netmem);
}

/**
 * skb_frag_netmem - retrieve the netmem referred to by a fragment
 * @frag: the fragment
 *
 * Return: the &netmem_ref associated with @frag.
 */
static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag)
{
        return frag->netmem;
}

int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
                    unsigned int headroom);
int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
                         struct bpf_prog *prog);

/**
 * skb_frag_address - gets the address of the data contained in a paged fragment
 * @frag: the paged fragment buffer
 *
 * Returns the address of the data within @frag. The page must already
 * be mapped.
 */
static inline void *skb_frag_address(const skb_frag_t *frag)
{
        if (!skb_frag_page(frag))
                return NULL;

        return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
}

/**
 * skb_frag_address_safe - gets the address of the data contained in a paged fragment
 * @frag: the paged fragment buffer
 *
 * Returns the address of the data within @frag. Checks that the page
 * is mapped and returns %NULL otherwise.
 */
static inline void *skb_frag_address_safe(const skb_frag_t *frag)
{
        void *ptr = page_address(skb_frag_page(frag));
        if (unlikely(!ptr))
                return NULL;

        return ptr + skb_frag_off(frag);
}

/**
 * skb_frag_page_copy() - sets the page in a fragment from another fragment
 * @fragto: skb fragment where page is set
 * @fragfrom: skb fragment page is copied from
 */
static inline void skb_frag_page_copy(skb_frag_t *fragto,
                                      const skb_frag_t *fragfrom)
{
        fragto->netmem = fragfrom->netmem;
}

bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);

/**
 * skb_frag_dma_map - maps a paged fragment via the DMA API
 * @dev: the device to map the fragment to
 * @frag: the paged fragment to map
 * @offset: the offset within the fragment (starting at the
 *          fragment's own offset)
 * @size: the number of bytes to map
 * @dir: the direction of the mapping (``PCI_DMA_*``)
 *
 * Maps the page associated with @frag to @device.
 */
static inline dma_addr_t skb_frag_dma_map(struct device *dev,
                                          const skb_frag_t *frag,
                                          size_t offset, size_t size,
                                          enum dma_data_direction dir)
{
        return dma_map_page(dev, skb_frag_page(frag),
                            skb_frag_off(frag) + offset, size, dir);
}

static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
                                        gfp_t gfp_mask)
{
        return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
}


static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
                                                  gfp_t gfp_mask)
{
        return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
}


/**
 *        skb_clone_writable - is the header of a clone writable
 *        @skb: buffer to check
 *        @len: length up to which to write
 *
 *        Returns true if modifying the header part of the cloned buffer
 *        does not requires the data to be copied.
 */
static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
{
        return !skb_header_cloned(skb) &&
               skb_headroom(skb) + len <= skb->hdr_len;
}

static inline int skb_try_make_writable(struct sk_buff *skb,
                                        unsigned int write_len)
{
        return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
               pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
}

static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
                            int cloned)
{
        int delta = 0;

        if (headroom > skb_headroom(skb))
                delta = headroom - skb_headroom(skb);

        if (delta || cloned)
                return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
                                        GFP_ATOMIC);
        return 0;
}

/**
 *        skb_cow - copy header of skb when it is required
 *        @skb: buffer to cow
 *        @headroom: needed headroom
 *
 *        If the skb passed lacks sufficient headroom or its data part
 *        is shared, data is reallocated. If reallocation fails, an error
 *        is returned and original skb is not changed.
 *
 *        The result is skb with writable area skb->head...skb->tail
 *        and at least @headroom of space at head.
 */
static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
{
        return __skb_cow(skb, headroom, skb_cloned(skb));
}

/**
 *        skb_cow_head - skb_cow but only making the head writable
 *        @skb: buffer to cow
 *        @headroom: needed headroom
 *
 *        This function is identical to skb_cow except that we replace the
 *        skb_cloned check by skb_header_cloned.  It should be used when
 *        you only need to push on some header and do not need to modify
 *        the data.
 */
static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
{
        return __skb_cow(skb, headroom, skb_header_cloned(skb));
}

/**
 *        skb_padto        - pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error.
 */
static inline int skb_padto(struct sk_buff *skb, unsigned int len)
{
        unsigned int size = skb->len;
        if (likely(size >= len))
                return 0;
        return skb_pad(skb, len - size);
}

/**
 *        __skb_put_padto - increase size and pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *        @free_on_error: free buffer on error
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error if @free_on_error is true.
 */
static inline int __must_check __skb_put_padto(struct sk_buff *skb,
                                               unsigned int len,
                                               bool free_on_error)
{
        unsigned int size = skb->len;

        if (unlikely(size < len)) {
                len -= size;
                if (__skb_pad(skb, len, free_on_error))
                        return -ENOMEM;
                __skb_put(skb, len);
        }
        return 0;
}

/**
 *        skb_put_padto - increase size and pad an skbuff up to a minimal size
 *        @skb: buffer to pad
 *        @len: minimal length
 *
 *        Pads up a buffer to ensure the trailing bytes exist and are
 *        blanked. If the buffer already contains sufficient data it
 *        is untouched. Otherwise it is extended. Returns zero on
 *        success. The skb is freed on error.
 */
static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
{
        return __skb_put_padto(skb, len, true);
}

bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i)
        __must_check;

static inline int skb_add_data(struct sk_buff *skb,
                               struct iov_iter *from, int copy)
{
        const int off = skb->len;

        if (skb->ip_summed == CHECKSUM_NONE) {
                __wsum csum = 0;
                if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy,
                                                 &csum, from)) {
                        skb->csum = csum_block_add(skb->csum, csum, off);
                        return 0;
                }
        } else if (copy_from_iter_full(skb_put(skb, copy), copy, from))
                return 0;

        __skb_trim(skb, off);
        return -EFAULT;
}

static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
                                    const struct page *page, int off)
{
        if (skb_zcopy(skb))
                return false;
        if (i) {
                const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];

                return page == skb_frag_page(frag) &&
                       off == skb_frag_off(frag) + skb_frag_size(frag);
        }
        return false;
}

static inline int __skb_linearize(struct sk_buff *skb)
{
        return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
}

/**
 *        skb_linearize - convert paged skb to linear one
 *        @skb: buffer to linarize
 *
 *        If there is no free memory -ENOMEM is returned, otherwise zero
 *        is returned and the old skb data released.
 */
static inline int skb_linearize(struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
}

/**
 * skb_has_shared_frag - can any frag be overwritten
 * @skb: buffer to test
 *
 * Return true if the skb has at least one frag that might be modified
 * by an external entity (as in vmsplice()/sendfile())
 */
static inline bool skb_has_shared_frag(const struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) &&
               skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG;
}

/**
 *        skb_linearize_cow - make sure skb is linear and writable
 *        @skb: buffer to process
 *
 *        If there is no free memory -ENOMEM is returned, otherwise zero
 *        is returned and the old skb data released.
 */
static inline int skb_linearize_cow(struct sk_buff *skb)
{
        return skb_is_nonlinear(skb) || skb_cloned(skb) ?
               __skb_linearize(skb) : 0;
}

static __always_inline void
__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
                     unsigned int off)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = csum_block_sub(skb->csum,
                                           csum_partial(start, len, 0), off);
        else if (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) < 0)
                skb->ip_summed = CHECKSUM_NONE;
}

/**
 *        skb_postpull_rcsum - update checksum for received skb after pull
 *        @skb: buffer to update
 *        @start: start of data before pull
 *        @len: length of data pulled
 *
 *        After doing a pull on a received packet, you need to call this to
 *        update the CHECKSUM_COMPLETE checksum, or set ip_summed to
 *        CHECKSUM_NONE so that it can be recomputed from scratch.
 */
static inline void skb_postpull_rcsum(struct sk_buff *skb,
                                      const void *start, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = wsum_negate(csum_partial(start, len,
                                                     wsum_negate(skb->csum)));
        else if (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) < 0)
                skb->ip_summed = CHECKSUM_NONE;
}

static __always_inline void
__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
                     unsigned int off)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->csum = csum_block_add(skb->csum,
                                           csum_partial(start, len, 0), off);
}

/**
 *        skb_postpush_rcsum - update checksum for received skb after push
 *        @skb: buffer to update
 *        @start: start of data after push
 *        @len: length of data pushed
 *
 *        After doing a push on a received packet, you need to call this to
 *        update the CHECKSUM_COMPLETE checksum.
 */
static inline void skb_postpush_rcsum(struct sk_buff *skb,
                                      const void *start, unsigned int len)
{
        __skb_postpush_rcsum(skb, start, len, 0);
}

void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);

/**
 *        skb_push_rcsum - push skb and update receive checksum
 *        @skb: buffer to update
 *        @len: length of data pulled
 *
 *        This function performs an skb_push on the packet and updates
 *        the CHECKSUM_COMPLETE checksum.  It should be used on
 *        receive path processing instead of skb_push unless you know
 *        that the checksum difference is zero (e.g., a valid IP header)
 *        or you are setting ip_summed to CHECKSUM_NONE.
 */
static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
{
        skb_push(skb, len);
        skb_postpush_rcsum(skb, skb->data, len);
        return skb->data;
}

int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
/**
 *        pskb_trim_rcsum - trim received skb and update checksum
 *        @skb: buffer to trim
 *        @len: new length
 *
 *        This is exactly the same as pskb_trim except that it ensures the
 *        checksum of received packets are still valid after the operation.
 *        It can change skb pointers.
 */

static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
        skb_might_realloc(skb);
        if (likely(len >= skb->len))
                return 0;
        return pskb_trim_rcsum_slow(skb, len);
}

static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
        __skb_trim(skb, len);
        return 0;
}

static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
        return __skb_grow(skb, len);
}

#define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
#define skb_rb_first(root) rb_to_skb(rb_first(root))
#define skb_rb_last(root)  rb_to_skb(rb_last(root))
#define skb_rb_next(skb)   rb_to_skb(rb_next(&(skb)->rbnode))
#define skb_rb_prev(skb)   rb_to_skb(rb_prev(&(skb)->rbnode))

#define skb_queue_walk(queue, skb) \
                for (skb = (queue)->next;                                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = skb->next)

#define skb_queue_walk_safe(queue, skb, tmp)                                        \
                for (skb = (queue)->next, tmp = skb->next;                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->next)

#define skb_queue_walk_from(queue, skb)                                                \
                for (; skb != (struct sk_buff *)(queue);                        \
                     skb = skb->next)

#define skb_rbtree_walk(skb, root)                                                \
                for (skb = skb_rb_first(root); skb != NULL;                        \
                     skb = skb_rb_next(skb))

#define skb_rbtree_walk_from(skb)                                                \
                for (; skb != NULL;                                                \
                     skb = skb_rb_next(skb))

#define skb_rbtree_walk_from_safe(skb, tmp)                                        \
                for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL);        \
                     skb = tmp)

#define skb_queue_walk_from_safe(queue, skb, tmp)                                \
                for (tmp = skb->next;                                                \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->next)

#define skb_queue_reverse_walk(queue, skb) \
                for (skb = (queue)->prev;                                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = skb->prev)

#define skb_queue_reverse_walk_safe(queue, skb, tmp)                                \
                for (skb = (queue)->prev, tmp = skb->prev;                        \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->prev)

#define skb_queue_reverse_walk_from_safe(queue, skb, tmp)                        \
                for (tmp = skb->prev;                                                \
                     skb != (struct sk_buff *)(queue);                                \
                     skb = tmp, tmp = skb->prev)

static inline bool skb_has_frag_list(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->frag_list != NULL;
}

static inline void skb_frag_list_init(struct sk_buff *skb)
{
        skb_shinfo(skb)->frag_list = NULL;
}

#define skb_walk_frags(skb, iter)        \
        for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)


int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
                                int *err, long *timeo_p,
                                const struct sk_buff *skb);
struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
                                          struct sk_buff_head *queue,
                                          unsigned int flags,
                                          int *off, int *err,
                                          struct sk_buff **last);
struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
                                        struct sk_buff_head *queue,
                                        unsigned int flags, int *off, int *err,
                                        struct sk_buff **last);
struct sk_buff *__skb_recv_datagram(struct sock *sk,
                                    struct sk_buff_head *sk_queue,
                                    unsigned int flags, int *off, int *err);
struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, int *err);
__poll_t datagram_poll(struct file *file, struct socket *sock,
                           struct poll_table_struct *wait);
int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
                           struct iov_iter *to, int size);
static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
                                        struct msghdr *msg, int size)
{
        return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
}
int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
                                   struct msghdr *msg);
int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
                           struct iov_iter *to, int len,
                           struct ahash_request *hash);
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
                                 struct iov_iter *from, int len);
int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
                              int len);
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
                    struct pipe_inode_info *pipe, unsigned int len,
                    unsigned int flags);
int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
                         int len);
int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len);
void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
                 int len, int hlen);
void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
void skb_scrub_packet(struct sk_buff *skb, bool xnet);
struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
                                 unsigned int offset);
struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
int skb_ensure_writable(struct sk_buff *skb, unsigned int write_len);
int skb_ensure_writable_head_tail(struct sk_buff *skb, struct net_device *dev);
int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
int skb_vlan_pop(struct sk_buff *skb);
int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
int skb_eth_pop(struct sk_buff *skb);
int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
                 const unsigned char *src);
int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
                  int mac_len, bool ethernet);
int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
                 bool ethernet);
int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
int skb_mpls_dec_ttl(struct sk_buff *skb);
struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
                             gfp_t gfp);

static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
{
        return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
}

static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
{
        return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
}

struct skb_checksum_ops {
        __wsum (*update)(const void *mem, int len, __wsum wsum);
        __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
};

extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;

__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
                      __wsum csum, const struct skb_checksum_ops *ops);
__wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
                    __wsum csum);

static inline void * __must_check
__skb_header_pointer(const struct sk_buff *skb, int offset, int len,
                     const void *data, int hlen, void *buffer)
{
        if (likely(hlen - offset >= len))
                return (void *)data + offset;

        if (!skb || unlikely(skb_copy_bits(skb, offset, buffer, len) < 0))
                return NULL;

        return buffer;
}

static inline void * __must_check
skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
{
        return __skb_header_pointer(skb, offset, len, skb->data,
                                    skb_headlen(skb), buffer);
}

static inline void * __must_check
skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
{
        if (likely(skb_headlen(skb) - offset >= len))
                return skb->data + offset;
        return NULL;
}

/**
 *        skb_needs_linearize - check if we need to linearize a given skb
 *                              depending on the given device features.
 *        @skb: socket buffer to check
 *        @features: net device features
 *
 *        Returns true if either:
 *        1. skb has frag_list and the device doesn't support FRAGLIST, or
 *        2. skb is fragmented and the device does not support SG.
 */
static inline bool skb_needs_linearize(struct sk_buff *skb,
                                       netdev_features_t features)
{
        return skb_is_nonlinear(skb) &&
               ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
                (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
}

static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
                                             void *to,
                                             const unsigned int len)
{
        memcpy(to, skb->data, len);
}

static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
                                                    const int offset, void *to,
                                                    const unsigned int len)
{
        memcpy(to, skb->data + offset, len);
}

static inline void skb_copy_to_linear_data(struct sk_buff *skb,
                                           const void *from,
                                           const unsigned int len)
{
        memcpy(skb->data, from, len);
}

static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
                                                  const int offset,
                                                  const void *from,
                                                  const unsigned int len)
{
        memcpy(skb->data + offset, from, len);
}

void skb_init(void);

static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
{
        return skb->tstamp;
}

/**
 *        skb_get_timestamp - get timestamp from a skb
 *        @skb: skb to get stamp from
 *        @stamp: pointer to struct __kernel_old_timeval to store stamp in
 *
 *        Timestamps are stored in the skb as offsets to a base timestamp.
 *        This function converts the offset back to a struct timeval and stores
 *        it in stamp.
 */
static inline void skb_get_timestamp(const struct sk_buff *skb,
                                     struct __kernel_old_timeval *stamp)
{
        *stamp = ns_to_kernel_old_timeval(skb->tstamp);
}

static inline void skb_get_new_timestamp(const struct sk_buff *skb,
                                         struct __kernel_sock_timeval *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_usec = ts.tv_nsec / 1000;
}

static inline void skb_get_timestampns(const struct sk_buff *skb,
                                       struct __kernel_old_timespec *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_nsec = ts.tv_nsec;
}

static inline void skb_get_new_timestampns(const struct sk_buff *skb,
                                           struct __kernel_timespec *stamp)
{
        struct timespec64 ts = ktime_to_timespec64(skb->tstamp);

        stamp->tv_sec = ts.tv_sec;
        stamp->tv_nsec = ts.tv_nsec;
}

static inline void __net_timestamp(struct sk_buff *skb)
{
        skb->tstamp = ktime_get_real();
        skb->tstamp_type = SKB_CLOCK_REALTIME;
}

static inline ktime_t net_timedelta(ktime_t t)
{
        return ktime_sub(ktime_get_real(), t);
}

static inline void skb_set_delivery_time(struct sk_buff *skb, ktime_t kt,
                                         u8 tstamp_type)
{
        skb->tstamp = kt;

        if (kt)
                skb->tstamp_type = tstamp_type;
        else
                skb->tstamp_type = SKB_CLOCK_REALTIME;
}

static inline void skb_set_delivery_type_by_clockid(struct sk_buff *skb,
                                                    ktime_t kt, clockid_t clockid)
{
        u8 tstamp_type = SKB_CLOCK_REALTIME;

        switch (clockid) {
        case CLOCK_REALTIME:
                break;
        case CLOCK_MONOTONIC:
                tstamp_type = SKB_CLOCK_MONOTONIC;
                break;
        case CLOCK_TAI:
                tstamp_type = SKB_CLOCK_TAI;
                break;
        default:
                WARN_ON_ONCE(1);
                kt = 0;
        }

        skb_set_delivery_time(skb, kt, tstamp_type);
}

DECLARE_STATIC_KEY_FALSE(netstamp_needed_key);

/* It is used in the ingress path to clear the delivery_time.
 * If needed, set the skb->tstamp to the (rcv) timestamp.
 */
static inline void skb_clear_delivery_time(struct sk_buff *skb)
{
        if (skb->tstamp_type) {
                skb->tstamp_type = SKB_CLOCK_REALTIME;
                if (static_branch_unlikely(&netstamp_needed_key))
                        skb->tstamp = ktime_get_real();
                else
                        skb->tstamp = 0;
        }
}

static inline void skb_clear_tstamp(struct sk_buff *skb)
{
        if (skb->tstamp_type)
                return;

        skb->tstamp = 0;
}

static inline ktime_t skb_tstamp(const struct sk_buff *skb)
{
        if (skb->tstamp_type)
                return 0;

        return skb->tstamp;
}

static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond)
{
        if (skb->tstamp_type != SKB_CLOCK_MONOTONIC && skb->tstamp)
                return skb->tstamp;

        if (static_branch_unlikely(&netstamp_needed_key) || cond)
                return ktime_get_real();

        return 0;
}

static inline u8 skb_metadata_len(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->meta_len;
}

static inline void *skb_metadata_end(const struct sk_buff *skb)
{
        return skb_mac_header(skb);
}

static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
                                          const struct sk_buff *skb_b,
                                          u8 meta_len)
{
        const void *a = skb_metadata_end(skb_a);
        const void *b = skb_metadata_end(skb_b);
        u64 diffs = 0;

        if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
            BITS_PER_LONG != 64)
                goto slow;

        /* Using more efficient variant than plain call to memcmp(). */
        switch (meta_len) {
#define __it(x, op) (x -= sizeof(u##op))
#define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
        case 32: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 24: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 16: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case  8: diffs |= __it_diff(a, b, 64);
                break;
        case 28: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 20: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case 12: diffs |= __it_diff(a, b, 64);
                fallthrough;
        case  4: diffs |= __it_diff(a, b, 32);
                break;
        default:
slow:
                return memcmp(a - meta_len, b - meta_len, meta_len);
        }
        return diffs;
}

static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
                                        const struct sk_buff *skb_b)
{
        u8 len_a = skb_metadata_len(skb_a);
        u8 len_b = skb_metadata_len(skb_b);

        if (!(len_a | len_b))
                return false;

        return len_a != len_b ?
               true : __skb_metadata_differs(skb_a, skb_b, len_a);
}

static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
{
        skb_shinfo(skb)->meta_len = meta_len;
}

static inline void skb_metadata_clear(struct sk_buff *skb)
{
        skb_metadata_set(skb, 0);
}

struct sk_buff *skb_clone_sk(struct sk_buff *skb);

#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING

void skb_clone_tx_timestamp(struct sk_buff *skb);
bool skb_defer_rx_timestamp(struct sk_buff *skb);

#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */

static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
{
}

static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
{
        return false;
}

#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */

/**
 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
 *
 * PHY drivers may accept clones of transmitted packets for
 * timestamping via their phy_driver.txtstamp method. These drivers
 * must call this function to return the skb back to the stack with a
 * timestamp.
 *
 * @skb: clone of the original outgoing packet
 * @hwtstamps: hardware time stamps
 *
 */
void skb_complete_tx_timestamp(struct sk_buff *skb,
                               struct skb_shared_hwtstamps *hwtstamps);

void __skb_tstamp_tx(struct sk_buff *orig_skb, const struct sk_buff *ack_skb,
                     struct skb_shared_hwtstamps *hwtstamps,
                     struct sock *sk, int tstype);

/**
 * skb_tstamp_tx - queue clone of skb with send time stamps
 * @orig_skb:        the original outgoing packet
 * @hwtstamps:        hardware time stamps, may be NULL if not available
 *
 * If the skb has a socket associated, then this function clones the
 * skb (thus sharing the actual data and optional structures), stores
 * the optional hardware time stamping information (if non NULL) or
 * generates a software time stamp (otherwise), then queues the clone
 * to the error queue of the socket.  Errors are silently ignored.
 */
void skb_tstamp_tx(struct sk_buff *orig_skb,
                   struct skb_shared_hwtstamps *hwtstamps);

/**
 * skb_tx_timestamp() - Driver hook for transmit timestamping
 *
 * Ethernet MAC Drivers should call this function in their hard_xmit()
 * function immediately before giving the sk_buff to the MAC hardware.
 *
 * Specifically, one should make absolutely sure that this function is
 * called before TX completion of this packet can trigger.  Otherwise
 * the packet could potentially already be freed.
 *
 * @skb: A socket buffer.
 */
static inline void skb_tx_timestamp(struct sk_buff *skb)
{
        skb_clone_tx_timestamp(skb);
        if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
                skb_tstamp_tx(skb, NULL);
}

/**
 * skb_complete_wifi_ack - deliver skb with wifi status
 *
 * @skb: the original outgoing packet
 * @acked: ack status
 *
 */
void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);

__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
__sum16 __skb_checksum_complete(struct sk_buff *skb);

static inline int skb_csum_unnecessary(const struct sk_buff *skb)
{
        return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
                skb->csum_valid ||
                (skb->ip_summed == CHECKSUM_PARTIAL &&
                 skb_checksum_start_offset(skb) >= 0));
}

/**
 *        skb_checksum_complete - Calculate checksum of an entire packet
 *        @skb: packet to process
 *
 *        This function calculates the checksum over the entire packet plus
 *        the value of skb->csum.  The latter can be used to supply the
 *        checksum of a pseudo header as used by TCP/UDP.  It returns the
 *        checksum.
 *
 *        For protocols that contain complete checksums such as ICMP/TCP/UDP,
 *        this function can be used to verify that checksum on received
 *        packets.  In that case the function should return zero if the
 *        checksum is correct.  In particular, this function will return zero
 *        if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
 *        hardware has already verified the correctness of the checksum.
 */
static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
{
        return skb_csum_unnecessary(skb) ?
               0 : __skb_checksum_complete(skb);
}

static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                if (skb->csum_level == 0)
                        skb->ip_summed = CHECKSUM_NONE;
                else
                        skb->csum_level--;
        }
}

static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
                        skb->csum_level++;
        } else if (skb->ip_summed == CHECKSUM_NONE) {
                skb->ip_summed = CHECKSUM_UNNECESSARY;
                skb->csum_level = 0;
        }
}

static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
                skb->ip_summed = CHECKSUM_NONE;
                skb->csum_level = 0;
        }
}

/* Check if we need to perform checksum complete validation.
 *
 * Returns true if checksum complete is needed, false otherwise
 * (either checksum is unnecessary or zero checksum is allowed).
 */
static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
                                                  bool zero_okay,
                                                  __sum16 check)
{
        if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
                skb->csum_valid = 1;
                __skb_decr_checksum_unnecessary(skb);
                return false;
        }

        return true;
}

/* For small packets <= CHECKSUM_BREAK perform checksum complete directly
 * in checksum_init.
 */
#define CHECKSUM_BREAK 76

/* Unset checksum-complete
 *
 * Unset checksum complete can be done when packet is being modified
 * (uncompressed for instance) and checksum-complete value is
 * invalidated.
 */
static inline void skb_checksum_complete_unset(struct sk_buff *skb)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
}

/* Validate (init) checksum based on checksum complete.
 *
 * Return values:
 *   0: checksum is validated or try to in skb_checksum_complete. In the latter
 *        case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
 *        checksum is stored in skb->csum for use in __skb_checksum_complete
 *   non-zero: value of invalid checksum
 *
 */
static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
                                                       bool complete,
                                                       __wsum psum)
{
        if (skb->ip_summed == CHECKSUM_COMPLETE) {
                if (!csum_fold(csum_add(psum, skb->csum))) {
                        skb->csum_valid = 1;
                        return 0;
                }
        }

        skb->csum = psum;

        if (complete || skb->len <= CHECKSUM_BREAK) {
                __sum16 csum;

                csum = __skb_checksum_complete(skb);
                skb->csum_valid = !csum;
                return csum;
        }

        return 0;
}

static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
{
        return 0;
}

/* Perform checksum validate (init). Note that this is a macro since we only
 * want to calculate the pseudo header which is an input function if necessary.
 * First we try to validate without any computation (checksum unnecessary) and
 * then calculate based on checksum complete calling the function to compute
 * pseudo header.
 *
 * Return values:
 *   0: checksum is validated or try to in skb_checksum_complete
 *   non-zero: value of invalid checksum
 */
#define __skb_checksum_validate(skb, proto, complete,                        \
                                zero_okay, check, compute_pseudo)        \
({                                                                        \
        __sum16 __ret = 0;                                                \
        skb->csum_valid = 0;                                                \
        if (__skb_checksum_validate_needed(skb, zero_okay, check))        \
                __ret = __skb_checksum_validate_complete(skb,                \
                                complete, compute_pseudo(skb, proto));        \
        __ret;                                                                \
})

#define skb_checksum_init(skb, proto, compute_pseudo)                        \
        __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)

#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo)        \
        __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)

#define skb_checksum_validate(skb, proto, compute_pseudo)                \
        __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)

#define skb_checksum_validate_zero_check(skb, proto, check,                \
                                         compute_pseudo)                \
        __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)

#define skb_checksum_simple_validate(skb)                                \
        __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)

static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
{
        return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
}

static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
{
        skb->csum = ~pseudo;
        skb->ip_summed = CHECKSUM_COMPLETE;
}

#define skb_checksum_try_convert(skb, proto, compute_pseudo)        \
do {                                                                        \
        if (__skb_checksum_convert_check(skb))                                \
                __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
} while (0)

static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
                                              u16 start, u16 offset)
{
        skb->ip_summed = CHECKSUM_PARTIAL;
        skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
        skb->csum_offset = offset - start;
}

/* Update skbuf and packet to reflect the remote checksum offload operation.
 * When called, ptr indicates the starting point for skb->csum when
 * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
 * here, skb_postpull_rcsum is done so skb->csum start is ptr.
 */
static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
                                       int start, int offset, bool nopartial)
{
        __wsum delta;

        if (!nopartial) {
                skb_remcsum_adjust_partial(skb, ptr, start, offset);
                return;
        }

        if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
                __skb_checksum_complete(skb);
                skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
        }

        delta = remcsum_adjust(ptr, skb->csum, start, offset);

        /* Adjust skb->csum since we changed the packet */
        skb->csum = csum_add(skb->csum, delta);
}

static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return (void *)(skb->_nfct & NFCT_PTRMASK);
#else
        return NULL;
#endif
}

static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return skb->_nfct;
#else
        return 0UL;
#endif
}

static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        skb->slow_gro |= !!nfct;
        skb->_nfct = nfct;
#endif
}

#ifdef CONFIG_SKB_EXTENSIONS
enum skb_ext_id {
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
        SKB_EXT_BRIDGE_NF,
#endif
#ifdef CONFIG_XFRM
        SKB_EXT_SEC_PATH,
#endif
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
        TC_SKB_EXT,
#endif
#if IS_ENABLED(CONFIG_MPTCP)
        SKB_EXT_MPTCP,
#endif
#if IS_ENABLED(CONFIG_MCTP_FLOWS)
        SKB_EXT_MCTP,
#endif
        SKB_EXT_NUM, /* must be last */
};

/**
 *        struct skb_ext - sk_buff extensions
 *        @refcnt: 1 on allocation, deallocated on 0
 *        @offset: offset to add to @data to obtain extension address
 *        @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
 *        @data: start of extension data, variable sized
 *
 *        Note: offsets/lengths are stored in chunks of 8 bytes, this allows
 *        to use 'u8' types while allowing up to 2kb worth of extension data.
 */
struct skb_ext {
        refcount_t refcnt;
        u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
        u8 chunks;                /* same */
        char data[] __aligned(8);
};

struct skb_ext *__skb_ext_alloc(gfp_t flags);
void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
                    struct skb_ext *ext);
void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
void __skb_ext_put(struct skb_ext *ext);

static inline void skb_ext_put(struct sk_buff *skb)
{
        if (skb->active_extensions)
                __skb_ext_put(skb->extensions);
}

static inline void __skb_ext_copy(struct sk_buff *dst,
                                  const struct sk_buff *src)
{
        dst->active_extensions = src->active_extensions;

        if (src->active_extensions) {
                struct skb_ext *ext = src->extensions;

                refcount_inc(&ext->refcnt);
                dst->extensions = ext;
        }
}

static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
{
        skb_ext_put(dst);
        __skb_ext_copy(dst, src);
}

static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
{
        return !!ext->offset[i];
}

static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
{
        return skb->active_extensions & (1 << id);
}

static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
{
        if (skb_ext_exist(skb, id))
                __skb_ext_del(skb, id);
}

static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
{
        if (skb_ext_exist(skb, id)) {
                struct skb_ext *ext = skb->extensions;

                return (void *)ext + (ext->offset[id] << 3);
        }

        return NULL;
}

static inline void skb_ext_reset(struct sk_buff *skb)
{
        if (unlikely(skb->active_extensions)) {
                __skb_ext_put(skb->extensions);
                skb->active_extensions = 0;
        }
}

static inline bool skb_has_extensions(struct sk_buff *skb)
{
        return unlikely(skb->active_extensions);
}
#else
static inline void skb_ext_put(struct sk_buff *skb) {}
static inline void skb_ext_reset(struct sk_buff *skb) {}
static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
#endif /* CONFIG_SKB_EXTENSIONS */

static inline void nf_reset_ct(struct sk_buff *skb)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        nf_conntrack_put(skb_nfct(skb));
        skb->_nfct = 0;
#endif
}

static inline void nf_reset_trace(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        skb->nf_trace = 0;
#endif
}

static inline void ipvs_reset(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_VS)
        skb->ipvs_property = 0;
#endif
}

/* Note: This doesn't put any conntrack info in dst. */
static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
                             bool copy)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        dst->_nfct = src->_nfct;
        nf_conntrack_get(skb_nfct(src));
#endif
#if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || IS_ENABLED(CONFIG_NF_TABLES)
        if (copy)
                dst->nf_trace = src->nf_trace;
#endif
}

static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
{
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
        nf_conntrack_put(skb_nfct(dst));
#endif
        dst->slow_gro = src->slow_gro;
        __nf_copy(dst, src, true);
}

#ifdef CONFIG_NETWORK_SECMARK
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{
        to->secmark = from->secmark;
}

static inline void skb_init_secmark(struct sk_buff *skb)
{
        skb->secmark = 0;
}
#else
static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
{ }

static inline void skb_init_secmark(struct sk_buff *skb)
{ }
#endif

static inline int secpath_exists(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
#else
        return 0;
#endif
}

static inline bool skb_irq_freeable(const struct sk_buff *skb)
{
        return !skb->destructor &&
                !secpath_exists(skb) &&
                !skb_nfct(skb) &&
                !skb->_skb_refdst &&
                !skb_has_frag_list(skb);
}

static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
{
        skb->queue_mapping = queue_mapping;
}

static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
{
        return skb->queue_mapping;
}

static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
{
        to->queue_mapping = from->queue_mapping;
}

static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
{
        skb->queue_mapping = rx_queue + 1;
}

static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
{
        return skb->queue_mapping - 1;
}

static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
{
        return skb->queue_mapping != 0;
}

static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
{
        skb->dst_pending_confirm = val;
}

static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
{
        return skb->dst_pending_confirm != 0;
}

static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
{
#ifdef CONFIG_XFRM
        return skb_ext_find(skb, SKB_EXT_SEC_PATH);
#else
        return NULL;
#endif
}

static inline bool skb_is_gso(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_size;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_v6(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
}

/* Note: Should be called only if skb_is_gso(skb) is true */
static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
{
        return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
}

static inline void skb_gso_reset(struct sk_buff *skb)
{
        skb_shinfo(skb)->gso_size = 0;
        skb_shinfo(skb)->gso_segs = 0;
        skb_shinfo(skb)->gso_type = 0;
}

static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
                                         u16 increment)
{
        if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
                return;
        shinfo->gso_size += increment;
}

static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
                                         u16 decrement)
{
        if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
                return;
        shinfo->gso_size -= decrement;
}

void __skb_warn_lro_forwarding(const struct sk_buff *skb);

static inline bool skb_warn_if_lro(const struct sk_buff *skb)
{
        /* LRO sets gso_size but not gso_type, whereas if GSO is really
         * wanted then gso_type will be set. */
        const struct skb_shared_info *shinfo = skb_shinfo(skb);

        if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
            unlikely(shinfo->gso_type == 0)) {
                __skb_warn_lro_forwarding(skb);
                return true;
        }
        return false;
}

static inline void skb_forward_csum(struct sk_buff *skb)
{
        /* Unfortunately we don't support this one.  Any brave souls? */
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                skb->ip_summed = CHECKSUM_NONE;
}

/**
 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
 * @skb: skb to check
 *
 * fresh skbs have their ip_summed set to CHECKSUM_NONE.
 * Instead of forcing ip_summed to CHECKSUM_NONE, we can
 * use this helper, to document places where we make this assertion.
 */
static inline void skb_checksum_none_assert(const struct sk_buff *skb)
{
        DEBUG_NET_WARN_ON_ONCE(skb->ip_summed != CHECKSUM_NONE);
}

bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);

int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
                                     unsigned int transport_len,
                                     __sum16(*skb_chkf)(struct sk_buff *skb));

/**
 * skb_head_is_locked - Determine if the skb->head is locked down
 * @skb: skb to check
 *
 * The head on skbs build around a head frag can be removed if they are
 * not cloned.  This function returns true if the skb head is locked down
 * due to either being allocated via kmalloc, or by being a clone with
 * multiple references to the head.
 */
static inline bool skb_head_is_locked(const struct sk_buff *skb)
{
        return !skb->head_frag || skb_cloned(skb);
}

/* Local Checksum Offload.
 * Compute outer checksum based on the assumption that the
 * inner checksum will be offloaded later.
 * See Documentation/networking/checksum-offloads.rst for
 * explanation of how this works.
 * Fill in outer checksum adjustment (e.g. with sum of outer
 * pseudo-header) before calling.
 * Also ensure that inner checksum is in linear data area.
 */
static inline __wsum lco_csum(struct sk_buff *skb)
{
        unsigned char *csum_start = skb_checksum_start(skb);
        unsigned char *l4_hdr = skb_transport_header(skb);
        __wsum partial;

        /* Start with complement of inner checksum adjustment */
        partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
                                                    skb->csum_offset));

        /* Add in checksum of our headers (incl. outer checksum
         * adjustment filled in by caller) and return result.
         */
        return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
}

static inline bool skb_is_redirected(const struct sk_buff *skb)
{
        return skb->redirected;
}

static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
{
        skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
        skb->from_ingress = from_ingress;
        if (skb->from_ingress)
                skb_clear_tstamp(skb);
#endif
}

static inline void skb_reset_redirect(struct sk_buff *skb)
{
        skb->redirected = 0;
}

static inline void skb_set_redirected_noclear(struct sk_buff *skb,
                                              bool from_ingress)
{
        skb->redirected = 1;
#ifdef CONFIG_NET_REDIRECT
        skb->from_ingress = from_ingress;
#endif
}

static inline bool skb_csum_is_sctp(struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_IP_SCTP)
        return skb->csum_not_inet;
#else
        return 0;
#endif
}

static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
{
        skb->ip_summed = CHECKSUM_NONE;
#if IS_ENABLED(CONFIG_IP_SCTP)
        skb->csum_not_inet = 0;
#endif
}

static inline void skb_set_kcov_handle(struct sk_buff *skb,
                                       const u64 kcov_handle)
{
#ifdef CONFIG_KCOV
        skb->kcov_handle = kcov_handle;
#endif
}

static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
{
#ifdef CONFIG_KCOV
        return skb->kcov_handle;
#else
        return 0;
#endif
}

static inline void skb_mark_for_recycle(struct sk_buff *skb)
{
#ifdef CONFIG_PAGE_POOL
        skb->pp_recycle = 1;
#endif
}

ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
                             ssize_t maxsize, gfp_t gfp);

#endif        /* __KERNEL__ */
#endif        /* _LINUX_SKBUFF_H */

































































   20 
   19 












   20 


   17 



    2 
   10 
    1 








   16 

   15 




   15 


    9 

    9 
    1 

    8 

   15 
   15 
   10 
    6 



   14 






    3 






    1 




    2 


   12 












    1 


   12 
    1 

    6 

    5 









   12 

   11 





    2 



    9 








































































   70 


   70 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                "Ping" sockets
 *
 * Based on ipv4/ping.c code.
 *
 * Authors:        Lorenzo Colitti (IPv6 support)
 *                Vasiliy Kulikov / Openwall (IPv4 implementation, for Linux 2.6),
 *                Pavel Kankovsky (IPv4 implementation, for Linux 2.4.32)
 */

#include <net/addrconf.h>
#include <net/ipv6.h>
#include <net/ip6_route.h>
#include <net/protocol.h>
#include <net/udp.h>
#include <net/transp_v6.h>
#include <linux/proc_fs.h>
#include <linux/bpf-cgroup.h>
#include <net/ping.h>

/* Compatibility glue so we can support IPv6 when it's compiled as a module */
static int dummy_ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len,
                                 int *addr_len)
{
        return -EAFNOSUPPORT;
}
static void dummy_ip6_datagram_recv_ctl(struct sock *sk, struct msghdr *msg,
                                       struct sk_buff *skb)
{
}
static int dummy_icmpv6_err_convert(u8 type, u8 code, int *err)
{
        return -EAFNOSUPPORT;
}
static void dummy_ipv6_icmp_error(struct sock *sk, struct sk_buff *skb, int err,
                                  __be16 port, u32 info, u8 *payload) {}
static int dummy_ipv6_chk_addr(struct net *net, const struct in6_addr *addr,
                               const struct net_device *dev, int strict)
{
        return 0;
}

static int ping_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
                               int addr_len)
{
        /* This check is replicated from __ip6_datagram_connect() and
         * intended to prevent BPF program called below from accessing
         * bytes that are out of the bound specified by user in addr_len.
         */

        if (addr_len < SIN6_LEN_RFC2133)
                return -EINVAL;

        return BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr, &addr_len);
}

static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{
        struct inet_sock *inet = inet_sk(sk);
        struct ipv6_pinfo *np = inet6_sk(sk);
        struct icmp6hdr user_icmph;
        int addr_type;
        struct in6_addr *daddr;
        int oif = 0;
        struct flowi6 fl6;
        int err;
        struct dst_entry *dst;
        struct rt6_info *rt;
        struct pingfakehdr pfh;
        struct ipcm6_cookie ipc6;

        err = ping_common_sendmsg(AF_INET6, msg, len, &user_icmph,
                                  sizeof(user_icmph));
        if (err)
                return err;

        memset(&fl6, 0, sizeof(fl6));

        if (msg->msg_name) {
                DECLARE_SOCKADDR(struct sockaddr_in6 *, u, msg->msg_name);
                if (msg->msg_namelen < sizeof(*u))
                        return -EINVAL;
                if (u->sin6_family != AF_INET6) {
                        return -EAFNOSUPPORT;
                }
                daddr = &(u->sin6_addr);
                if (inet6_test_bit(SNDFLOW, sk))
                        fl6.flowlabel = u->sin6_flowinfo & IPV6_FLOWINFO_MASK;
                if (__ipv6_addr_needs_scope_id(ipv6_addr_type(daddr)))
                        oif = u->sin6_scope_id;
        } else {
                if (sk->sk_state != TCP_ESTABLISHED)
                        return -EDESTADDRREQ;
                daddr = &sk->sk_v6_daddr;
                fl6.flowlabel = np->flow_label;
        }

        if (!oif)
                oif = sk->sk_bound_dev_if;

        if (!oif)
                oif = np->sticky_pktinfo.ipi6_ifindex;

        if (!oif && ipv6_addr_is_multicast(daddr))
                oif = READ_ONCE(np->mcast_oif);
        else if (!oif)
                oif = READ_ONCE(np->ucast_oif);

        addr_type = ipv6_addr_type(daddr);
        if ((__ipv6_addr_needs_scope_id(addr_type) && !oif) ||
            (addr_type & IPV6_ADDR_MAPPED) ||
            (oif && sk->sk_bound_dev_if && oif != sk->sk_bound_dev_if &&
             l3mdev_master_ifindex_by_index(sock_net(sk), oif) != sk->sk_bound_dev_if))
                return -EINVAL;

        ipcm6_init_sk(&ipc6, sk);
        ipc6.sockc.tsflags = READ_ONCE(sk->sk_tsflags);
        ipc6.sockc.mark = READ_ONCE(sk->sk_mark);

        fl6.flowi6_oif = oif;

        if (msg->msg_controllen) {
                struct ipv6_txoptions opt = {};

                opt.tot_len = sizeof(opt);
                ipc6.opt = &opt;

                err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6, &ipc6);
                if (err < 0)
                        return err;

                /* Changes to txoptions and flow info are not implemented, yet.
                 * Drop the options.
                 */
                ipc6.opt = NULL;
        }

        fl6.flowi6_proto = IPPROTO_ICMPV6;
        fl6.saddr = np->saddr;
        fl6.daddr = *daddr;
        fl6.flowi6_mark = ipc6.sockc.mark;
        fl6.flowi6_uid = sk->sk_uid;
        fl6.fl6_icmp_type = user_icmph.icmp6_type;
        fl6.fl6_icmp_code = user_icmph.icmp6_code;
        security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));

        fl6.flowlabel = ip6_make_flowinfo(ipc6.tclass, fl6.flowlabel);

        dst = ip6_sk_dst_lookup_flow(sk, &fl6, daddr, false);
        if (IS_ERR(dst))
                return PTR_ERR(dst);
        rt = dst_rt6_info(dst);

        if (!fl6.flowi6_oif && ipv6_addr_is_multicast(&fl6.daddr))
                fl6.flowi6_oif = READ_ONCE(np->mcast_oif);
        else if (!fl6.flowi6_oif)
                fl6.flowi6_oif = READ_ONCE(np->ucast_oif);

        pfh.icmph.type = user_icmph.icmp6_type;
        pfh.icmph.code = user_icmph.icmp6_code;
        pfh.icmph.checksum = 0;
        pfh.icmph.un.echo.id = inet->inet_sport;
        pfh.icmph.un.echo.sequence = user_icmph.icmp6_sequence;
        pfh.msg = msg;
        pfh.wcheck = 0;
        pfh.family = AF_INET6;

        if (ipc6.hlimit < 0)
                ipc6.hlimit = ip6_sk_dst_hoplimit(np, &fl6, dst);

        lock_sock(sk);
        err = ip6_append_data(sk, ping_getfrag, &pfh, len,
                              sizeof(struct icmp6hdr), &ipc6, &fl6, rt,
                              MSG_DONTWAIT);

        if (err) {
                ICMP6_INC_STATS(sock_net(sk), rt->rt6i_idev,
                                ICMP6_MIB_OUTERRORS);
                ip6_flush_pending_frames(sk);
        } else {
                icmpv6_push_pending_frames(sk, &fl6,
                                           (struct icmp6hdr *)&pfh.icmph, len);
        }
        release_sock(sk);

        dst_release(dst);

        if (err)
                return err;

        return len;
}

struct proto pingv6_prot = {
        .name =                "PINGv6",
        .owner =        THIS_MODULE,
        .init =                ping_init_sock,
        .close =        ping_close,
        .pre_connect =        ping_v6_pre_connect,
        .connect =        ip6_datagram_connect_v6_only,
        .disconnect =        __udp_disconnect,
        .setsockopt =        ipv6_setsockopt,
        .getsockopt =        ipv6_getsockopt,
        .sendmsg =        ping_v6_sendmsg,
        .recvmsg =        ping_recvmsg,
        .bind =                ping_bind,
        .backlog_rcv =        ping_queue_rcv_skb,
        .hash =                ping_hash,
        .unhash =        ping_unhash,
        .get_port =        ping_get_port,
        .put_port =        ping_unhash,
        .obj_size =        sizeof(struct raw6_sock),
        .ipv6_pinfo_offset = offsetof(struct raw6_sock, inet6),
};
EXPORT_SYMBOL_GPL(pingv6_prot);

static struct inet_protosw pingv6_protosw = {
        .type =      SOCK_DGRAM,
        .protocol =  IPPROTO_ICMPV6,
        .prot =      &pingv6_prot,
        .ops =       &inet6_sockraw_ops,
        .flags =     INET_PROTOSW_REUSE,
};

#ifdef CONFIG_PROC_FS
static void *ping_v6_seq_start(struct seq_file *seq, loff_t *pos)
{
        return ping_seq_start(seq, pos, AF_INET6);
}

static int ping_v6_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, IPV6_SEQ_DGRAM_HEADER);
        } else {
                int bucket = ((struct ping_iter_state *) seq->private)->bucket;
                struct inet_sock *inet = inet_sk((struct sock *)v);
                __u16 srcp = ntohs(inet->inet_sport);
                __u16 destp = ntohs(inet->inet_dport);
                ip6_dgram_sock_seq_show(seq, v, srcp, destp, bucket);
        }
        return 0;
}

static const struct seq_operations ping_v6_seq_ops = {
        .start                = ping_v6_seq_start,
        .show                = ping_v6_seq_show,
        .next                = ping_seq_next,
        .stop                = ping_seq_stop,
};

static int __net_init ping_v6_proc_init_net(struct net *net)
{
        if (!proc_create_net("icmp6", 0444, net->proc_net, &ping_v6_seq_ops,
                        sizeof(struct ping_iter_state)))
                return -ENOMEM;
        return 0;
}

static void __net_exit ping_v6_proc_exit_net(struct net *net)
{
        remove_proc_entry("icmp6", net->proc_net);
}

static struct pernet_operations ping_v6_net_ops = {
        .init = ping_v6_proc_init_net,
        .exit = ping_v6_proc_exit_net,
};
#endif

int __init pingv6_init(void)
{
#ifdef CONFIG_PROC_FS
        int ret = register_pernet_subsys(&ping_v6_net_ops);
        if (ret)
                return ret;
#endif
        pingv6_ops.ipv6_recv_error = ipv6_recv_error;
        pingv6_ops.ip6_datagram_recv_common_ctl = ip6_datagram_recv_common_ctl;
        pingv6_ops.ip6_datagram_recv_specific_ctl =
                ip6_datagram_recv_specific_ctl;
        pingv6_ops.icmpv6_err_convert = icmpv6_err_convert;
        pingv6_ops.ipv6_icmp_error = ipv6_icmp_error;
        pingv6_ops.ipv6_chk_addr = ipv6_chk_addr;
        return inet6_register_protosw(&pingv6_protosw);
}

/* This never gets called because it's not possible to unload the ipv6 module,
 * but just in case.
 */
void pingv6_exit(void)
{
        pingv6_ops.ipv6_recv_error = dummy_ipv6_recv_error;
        pingv6_ops.ip6_datagram_recv_common_ctl = dummy_ip6_datagram_recv_ctl;
        pingv6_ops.ip6_datagram_recv_specific_ctl = dummy_ip6_datagram_recv_ctl;
        pingv6_ops.icmpv6_err_convert = dummy_icmpv6_err_convert;
        pingv6_ops.ipv6_icmp_error = dummy_ipv6_icmp_error;
        pingv6_ops.ipv6_chk_addr = dummy_ipv6_chk_addr;
#ifdef CONFIG_PROC_FS
        unregister_pernet_subsys(&ping_v6_net_ops);
#endif
        inet6_unregister_protosw(&pingv6_protosw);
}











































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  204 




  106 

  105 



  106 




   39 

   38 




  204 




























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Monitoring code for network dropped packet alerts
 *
 * Copyright (C) 2009 Neil Horman <nhorman@tuxdriver.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/string.h>
#include <linux/if_arp.h>
#include <linux/inetdevice.h>
#include <linux/inet.h>
#include <linux/interrupt.h>
#include <linux/netpoll.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <linux/netlink.h>
#include <linux/net_dropmon.h>
#include <linux/bitfield.h>
#include <linux/percpu.h>
#include <linux/timer.h>
#include <linux/bitops.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/genetlink.h>
#include <net/netevent.h>
#include <net/flow_offload.h>
#include <net/dropreason.h>
#include <net/devlink.h>

#include <trace/events/skb.h>
#include <trace/events/napi.h>
#include <trace/events/devlink.h>

#include <linux/unaligned.h>

#define TRACE_ON 1
#define TRACE_OFF 0

/*
 * Globals, our netlink socket pointer
 * and the work handle that will send up
 * netlink alerts
 */
static int trace_state = TRACE_OFF;
static bool monitor_hw;

/* net_dm_mutex
 *
 * An overall lock guarding every operation coming from userspace.
 */
static DEFINE_MUTEX(net_dm_mutex);

struct net_dm_stats {
        u64_stats_t dropped;
        struct u64_stats_sync syncp;
};

#define NET_DM_MAX_HW_TRAP_NAME_LEN 40

struct net_dm_hw_entry {
        char trap_name[NET_DM_MAX_HW_TRAP_NAME_LEN];
        u32 count;
};

struct net_dm_hw_entries {
        u32 num_entries;
        struct net_dm_hw_entry entries[];
};

struct per_cpu_dm_data {
        raw_spinlock_t                lock;        /* Protects 'skb', 'hw_entries' and
                                         * 'send_timer'
                                         */
        union {
                struct sk_buff                        *skb;
                struct net_dm_hw_entries        *hw_entries;
        };
        struct sk_buff_head        drop_queue;
        struct work_struct        dm_alert_work;
        struct timer_list        send_timer;
        struct net_dm_stats        stats;
};

struct dm_hw_stat_delta {
        unsigned long last_rx;
        unsigned long last_drop_val;
        struct rcu_head rcu;
};

static struct genl_family net_drop_monitor_family;

static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_cpu_data);
static DEFINE_PER_CPU(struct per_cpu_dm_data, dm_hw_cpu_data);

static int dm_hit_limit = 64;
static int dm_delay = 1;
static unsigned long dm_hw_check_delta = 2*HZ;

static enum net_dm_alert_mode net_dm_alert_mode = NET_DM_ALERT_MODE_SUMMARY;
static u32 net_dm_trunc_len;
static u32 net_dm_queue_len = 1000;

struct net_dm_alert_ops {
        void (*kfree_skb_probe)(void *ignore, struct sk_buff *skb,
                                void *location,
                                enum skb_drop_reason reason,
                                struct sock *rx_sk);
        void (*napi_poll_probe)(void *ignore, struct napi_struct *napi,
                                int work, int budget);
        void (*work_item_func)(struct work_struct *work);
        void (*hw_work_item_func)(struct work_struct *work);
        void (*hw_trap_probe)(void *ignore, const struct devlink *devlink,
                              struct sk_buff *skb,
                              const struct devlink_trap_metadata *metadata);
};

struct net_dm_skb_cb {
        union {
                struct devlink_trap_metadata *hw_metadata;
                void *pc;
        };
        enum skb_drop_reason reason;
};

#define NET_DM_SKB_CB(__skb) ((struct net_dm_skb_cb *)&((__skb)->cb[0]))

static struct sk_buff *reset_per_cpu_data(struct per_cpu_dm_data *data)
{
        size_t al;
        struct net_dm_alert_msg *msg;
        struct nlattr *nla;
        struct sk_buff *skb;
        unsigned long flags;
        void *msg_header;

        al = sizeof(struct net_dm_alert_msg);
        al += dm_hit_limit * sizeof(struct net_dm_drop_point);
        al += sizeof(struct nlattr);

        skb = genlmsg_new(al, GFP_KERNEL);

        if (!skb)
                goto err;

        msg_header = genlmsg_put(skb, 0, 0, &net_drop_monitor_family,
                                 0, NET_DM_CMD_ALERT);
        if (!msg_header) {
                nlmsg_free(skb);
                skb = NULL;
                goto err;
        }
        nla = nla_reserve(skb, NLA_UNSPEC,
                          sizeof(struct net_dm_alert_msg));
        if (!nla) {
                nlmsg_free(skb);
                skb = NULL;
                goto err;
        }
        msg = nla_data(nla);
        memset(msg, 0, al);
        goto out;

err:
        mod_timer(&data->send_timer, jiffies + HZ / 10);
out:
        raw_spin_lock_irqsave(&data->lock, flags);
        swap(data->skb, skb);
        raw_spin_unlock_irqrestore(&data->lock, flags);

        if (skb) {
                struct nlmsghdr *nlh = (struct nlmsghdr *)skb->data;
                struct genlmsghdr *gnlh = (struct genlmsghdr *)nlmsg_data(nlh);

                genlmsg_end(skb, genlmsg_data(gnlh));
        }

        return skb;
}

static const struct genl_multicast_group dropmon_mcgrps[] = {
        { .name = "events", .flags = GENL_MCAST_CAP_SYS_ADMIN, },
};

static void send_dm_alert(struct work_struct *work)
{
        struct sk_buff *skb;
        struct per_cpu_dm_data *data;

        data = container_of(work, struct per_cpu_dm_data, dm_alert_work);

        skb = reset_per_cpu_data(data);

        if (skb)
                genlmsg_multicast(&net_drop_monitor_family, skb, 0,
                                  0, GFP_KERNEL);
}

/*
 * This is the timer function to delay the sending of an alert
 * in the event that more drops will arrive during the
 * hysteresis period.
 */
static void sched_send_work(struct timer_list *t)
{
        struct per_cpu_dm_data *data = from_timer(data, t, send_timer);

        schedule_work(&data->dm_alert_work);
}

static void trace_drop_common(struct sk_buff *skb, void *location)
{
        struct net_dm_alert_msg *msg;
        struct net_dm_drop_point *point;
        struct nlmsghdr *nlh;
        struct nlattr *nla;
        int i;
        struct sk_buff *dskb;
        struct per_cpu_dm_data *data;
        unsigned long flags;

        local_irq_save(flags);
        data = this_cpu_ptr(&dm_cpu_data);
        raw_spin_lock(&data->lock);
        dskb = data->skb;

        if (!dskb)
                goto out;

        nlh = (struct nlmsghdr *)dskb->data;
        nla = genlmsg_data(nlmsg_data(nlh));
        msg = nla_data(nla);
        point = msg->points;
        for (i = 0; i < msg->entries; i++) {
                if (!memcmp(&location, &point->pc, sizeof(void *))) {
                        point->count++;
                        goto out;
                }
                point++;
        }
        if (msg->entries == dm_hit_limit)
                goto out;
        /*
         * We need to create a new entry
         */
        __nla_reserve_nohdr(dskb, sizeof(struct net_dm_drop_point));
        nla->nla_len += NLA_ALIGN(sizeof(struct net_dm_drop_point));
        memcpy(point->pc, &location, sizeof(void *));
        point->count = 1;
        msg->entries++;

        if (!timer_pending(&data->send_timer)) {
                data->send_timer.expires = jiffies + dm_delay * HZ;
                add_timer(&data->send_timer);
        }

out:
        raw_spin_unlock_irqrestore(&data->lock, flags);
}

static void trace_kfree_skb_hit(void *ignore, struct sk_buff *skb,
                                void *location,
                                enum skb_drop_reason reason,
                                struct sock *rx_sk)
{
        trace_drop_common(skb, location);
}

static void trace_napi_poll_hit(void *ignore, struct napi_struct *napi,
                                int work, int budget)
{
        struct net_device *dev = napi->dev;
        struct dm_hw_stat_delta *stat;
        /*
         * Don't check napi structures with no associated device
         */
        if (!dev)
                return;

        rcu_read_lock();
        stat = rcu_dereference(dev->dm_private);
        if (stat) {
                /*
                 * only add a note to our monitor buffer if:
                 * 1) its after the last_rx delta
                 * 2) our rx_dropped count has gone up
                 */
                if (time_after(jiffies, stat->last_rx + dm_hw_check_delta) &&
                    (dev->stats.rx_dropped != stat->last_drop_val)) {
                        trace_drop_common(NULL, NULL);
                        stat->last_drop_val = dev->stats.rx_dropped;
                        stat->last_rx = jiffies;
                }
        }
        rcu_read_unlock();
}

static struct net_dm_hw_entries *
net_dm_hw_reset_per_cpu_data(struct per_cpu_dm_data *hw_data)
{
        struct net_dm_hw_entries *hw_entries;
        unsigned long flags;

        hw_entries = kzalloc(struct_size(hw_entries, entries, dm_hit_limit),
                             GFP_KERNEL);
        if (!hw_entries) {
                /* If the memory allocation failed, we try to perform another
                 * allocation in 1/10 second. Otherwise, the probe function
                 * will constantly bail out.
                 */
                mod_timer(&hw_data->send_timer, jiffies + HZ / 10);
        }

        raw_spin_lock_irqsave(&hw_data->lock, flags);
        swap(hw_data->hw_entries, hw_entries);
        raw_spin_unlock_irqrestore(&hw_data->lock, flags);

        return hw_entries;
}

static int net_dm_hw_entry_put(struct sk_buff *msg,
                               const struct net_dm_hw_entry *hw_entry)
{
        struct nlattr *attr;

        attr = nla_nest_start(msg, NET_DM_ATTR_HW_ENTRY);
        if (!attr)
                return -EMSGSIZE;

        if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_NAME, hw_entry->trap_name))
                goto nla_put_failure;

        if (nla_put_u32(msg, NET_DM_ATTR_HW_TRAP_COUNT, hw_entry->count))
                goto nla_put_failure;

        nla_nest_end(msg, attr);

        return 0;

nla_put_failure:
        nla_nest_cancel(msg, attr);
        return -EMSGSIZE;
}

static int net_dm_hw_entries_put(struct sk_buff *msg,
                                 const struct net_dm_hw_entries *hw_entries)
{
        struct nlattr *attr;
        int i;

        attr = nla_nest_start(msg, NET_DM_ATTR_HW_ENTRIES);
        if (!attr)
                return -EMSGSIZE;

        for (i = 0; i < hw_entries->num_entries; i++) {
                int rc;

                rc = net_dm_hw_entry_put(msg, &hw_entries->entries[i]);
                if (rc)
                        goto nla_put_failure;
        }

        nla_nest_end(msg, attr);

        return 0;

nla_put_failure:
        nla_nest_cancel(msg, attr);
        return -EMSGSIZE;
}

static int
net_dm_hw_summary_report_fill(struct sk_buff *msg,
                              const struct net_dm_hw_entries *hw_entries)
{
        struct net_dm_alert_msg anc_hdr = { 0 };
        void *hdr;
        int rc;

        hdr = genlmsg_put(msg, 0, 0, &net_drop_monitor_family, 0,
                          NET_DM_CMD_ALERT);
        if (!hdr)
                return -EMSGSIZE;

        /* We need to put the ancillary header in order not to break user
         * space.
         */
        if (nla_put(msg, NLA_UNSPEC, sizeof(anc_hdr), &anc_hdr))
                goto nla_put_failure;

        rc = net_dm_hw_entries_put(msg, hw_entries);
        if (rc)
                goto nla_put_failure;

        genlmsg_end(msg, hdr);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static void net_dm_hw_summary_work(struct work_struct *work)
{
        struct net_dm_hw_entries *hw_entries;
        struct per_cpu_dm_data *hw_data;
        struct sk_buff *msg;
        int rc;

        hw_data = container_of(work, struct per_cpu_dm_data, dm_alert_work);

        hw_entries = net_dm_hw_reset_per_cpu_data(hw_data);
        if (!hw_entries)
                return;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                goto out;

        rc = net_dm_hw_summary_report_fill(msg, hw_entries);
        if (rc) {
                nlmsg_free(msg);
                goto out;
        }

        genlmsg_multicast(&net_drop_monitor_family, msg, 0, 0, GFP_KERNEL);

out:
        kfree(hw_entries);
}

static void
net_dm_hw_trap_summary_probe(void *ignore, const struct devlink *devlink,
                             struct sk_buff *skb,
                             const struct devlink_trap_metadata *metadata)
{
        struct net_dm_hw_entries *hw_entries;
        struct net_dm_hw_entry *hw_entry;
        struct per_cpu_dm_data *hw_data;
        unsigned long flags;
        int i;

        if (metadata->trap_type == DEVLINK_TRAP_TYPE_CONTROL)
                return;

        hw_data = this_cpu_ptr(&dm_hw_cpu_data);
        raw_spin_lock_irqsave(&hw_data->lock, flags);
        hw_entries = hw_data->hw_entries;

        if (!hw_entries)
                goto out;

        for (i = 0; i < hw_entries->num_entries; i++) {
                hw_entry = &hw_entries->entries[i];
                if (!strncmp(hw_entry->trap_name, metadata->trap_name,
                             NET_DM_MAX_HW_TRAP_NAME_LEN - 1)) {
                        hw_entry->count++;
                        goto out;
                }
        }
        if (WARN_ON_ONCE(hw_entries->num_entries == dm_hit_limit))
                goto out;

        hw_entry = &hw_entries->entries[hw_entries->num_entries];
        strscpy(hw_entry->trap_name, metadata->trap_name,
                NET_DM_MAX_HW_TRAP_NAME_LEN - 1);
        hw_entry->count = 1;
        hw_entries->num_entries++;

        if (!timer_pending(&hw_data->send_timer)) {
                hw_data->send_timer.expires = jiffies + dm_delay * HZ;
                add_timer(&hw_data->send_timer);
        }

out:
        raw_spin_unlock_irqrestore(&hw_data->lock, flags);
}

static const struct net_dm_alert_ops net_dm_alert_summary_ops = {
        .kfree_skb_probe        = trace_kfree_skb_hit,
        .napi_poll_probe        = trace_napi_poll_hit,
        .work_item_func                = send_dm_alert,
        .hw_work_item_func        = net_dm_hw_summary_work,
        .hw_trap_probe                = net_dm_hw_trap_summary_probe,
};

static void net_dm_packet_trace_kfree_skb_hit(void *ignore,
                                              struct sk_buff *skb,
                                              void *location,
                                              enum skb_drop_reason reason,
                                              struct sock *rx_sk)
{
        ktime_t tstamp = ktime_get_real();
        struct per_cpu_dm_data *data;
        struct net_dm_skb_cb *cb;
        struct sk_buff *nskb;
        unsigned long flags;

        if (!skb_mac_header_was_set(skb))
                return;

        nskb = skb_clone(skb, GFP_ATOMIC);
        if (!nskb)
                return;

        cb = NET_DM_SKB_CB(nskb);
        cb->reason = reason;
        cb->pc = location;
        /* Override the timestamp because we care about the time when the
         * packet was dropped.
         */
        nskb->tstamp = tstamp;

        data = this_cpu_ptr(&dm_cpu_data);

        spin_lock_irqsave(&data->drop_queue.lock, flags);
        if (skb_queue_len(&data->drop_queue) < net_dm_queue_len)
                __skb_queue_tail(&data->drop_queue, nskb);
        else
                goto unlock_free;
        spin_unlock_irqrestore(&data->drop_queue.lock, flags);

        schedule_work(&data->dm_alert_work);

        return;

unlock_free:
        spin_unlock_irqrestore(&data->drop_queue.lock, flags);
        u64_stats_update_begin(&data->stats.syncp);
        u64_stats_inc(&data->stats.dropped);
        u64_stats_update_end(&data->stats.syncp);
        consume_skb(nskb);
}

static void net_dm_packet_trace_napi_poll_hit(void *ignore,
                                              struct napi_struct *napi,
                                              int work, int budget)
{
}

static size_t net_dm_in_port_size(void)
{
               /* NET_DM_ATTR_IN_PORT nest */
        return nla_total_size(0) +
               /* NET_DM_ATTR_PORT_NETDEV_IFINDEX */
               nla_total_size(sizeof(u32)) +
               /* NET_DM_ATTR_PORT_NETDEV_NAME */
               nla_total_size(IFNAMSIZ + 1);
}

#define NET_DM_MAX_SYMBOL_LEN 40
#define NET_DM_MAX_REASON_LEN 50

static size_t net_dm_packet_report_size(size_t payload_len)
{
        size_t size;

        size = nlmsg_msg_size(GENL_HDRLEN + net_drop_monitor_family.hdrsize);

        return NLMSG_ALIGN(size) +
               /* NET_DM_ATTR_ORIGIN */
               nla_total_size(sizeof(u16)) +
               /* NET_DM_ATTR_PC */
               nla_total_size(sizeof(u64)) +
               /* NET_DM_ATTR_SYMBOL */
               nla_total_size(NET_DM_MAX_SYMBOL_LEN + 1) +
               /* NET_DM_ATTR_IN_PORT */
               net_dm_in_port_size() +
               /* NET_DM_ATTR_TIMESTAMP */
               nla_total_size(sizeof(u64)) +
               /* NET_DM_ATTR_ORIG_LEN */
               nla_total_size(sizeof(u32)) +
               /* NET_DM_ATTR_PROTO */
               nla_total_size(sizeof(u16)) +
               /* NET_DM_ATTR_REASON */
               nla_total_size(NET_DM_MAX_REASON_LEN + 1) +
               /* NET_DM_ATTR_PAYLOAD */
               nla_total_size(payload_len);
}

static int net_dm_packet_report_in_port_put(struct sk_buff *msg, int ifindex,
                                            const char *name)
{
        struct nlattr *attr;

        attr = nla_nest_start(msg, NET_DM_ATTR_IN_PORT);
        if (!attr)
                return -EMSGSIZE;

        if (ifindex &&
            nla_put_u32(msg, NET_DM_ATTR_PORT_NETDEV_IFINDEX, ifindex))
                goto nla_put_failure;

        if (name && nla_put_string(msg, NET_DM_ATTR_PORT_NETDEV_NAME, name))
                goto nla_put_failure;

        nla_nest_end(msg, attr);

        return 0;

nla_put_failure:
        nla_nest_cancel(msg, attr);
        return -EMSGSIZE;
}

static int net_dm_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb,
                                     size_t payload_len)
{
        struct net_dm_skb_cb *cb = NET_DM_SKB_CB(skb);
        const struct drop_reason_list *list = NULL;
        unsigned int subsys, subsys_reason;
        char buf[NET_DM_MAX_SYMBOL_LEN];
        struct nlattr *attr;
        void *hdr;
        int rc;

        hdr = genlmsg_put(msg, 0, 0, &net_drop_monitor_family, 0,
                          NET_DM_CMD_PACKET_ALERT);
        if (!hdr)
                return -EMSGSIZE;

        if (nla_put_u16(msg, NET_DM_ATTR_ORIGIN, NET_DM_ORIGIN_SW))
                goto nla_put_failure;

        if (nla_put_u64_64bit(msg, NET_DM_ATTR_PC, (u64)(uintptr_t)cb->pc,
                              NET_DM_ATTR_PAD))
                goto nla_put_failure;

        rcu_read_lock();
        subsys = u32_get_bits(cb->reason, SKB_DROP_REASON_SUBSYS_MASK);
        if (subsys < SKB_DROP_REASON_SUBSYS_NUM)
                list = rcu_dereference(drop_reasons_by_subsys[subsys]);
        subsys_reason = cb->reason & ~SKB_DROP_REASON_SUBSYS_MASK;
        if (!list ||
            subsys_reason >= list->n_reasons ||
            !list->reasons[subsys_reason] ||
            strlen(list->reasons[subsys_reason]) > NET_DM_MAX_REASON_LEN) {
                list = rcu_dereference(drop_reasons_by_subsys[SKB_DROP_REASON_SUBSYS_CORE]);
                subsys_reason = SKB_DROP_REASON_NOT_SPECIFIED;
        }
        if (nla_put_string(msg, NET_DM_ATTR_REASON,
                           list->reasons[subsys_reason])) {
                rcu_read_unlock();
                goto nla_put_failure;
        }
        rcu_read_unlock();

        snprintf(buf, sizeof(buf), "%pS", cb->pc);
        if (nla_put_string(msg, NET_DM_ATTR_SYMBOL, buf))
                goto nla_put_failure;

        rc = net_dm_packet_report_in_port_put(msg, skb->skb_iif, NULL);
        if (rc)
                goto nla_put_failure;

        if (nla_put_u64_64bit(msg, NET_DM_ATTR_TIMESTAMP,
                              ktime_to_ns(skb->tstamp), NET_DM_ATTR_PAD))
                goto nla_put_failure;

        if (nla_put_u32(msg, NET_DM_ATTR_ORIG_LEN, skb->len))
                goto nla_put_failure;

        if (!payload_len)
                goto out;

        if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol)))
                goto nla_put_failure;

        attr = skb_put(msg, nla_total_size(payload_len));
        attr->nla_type = NET_DM_ATTR_PAYLOAD;
        attr->nla_len = nla_attr_size(payload_len);
        if (skb_copy_bits(skb, 0, nla_data(attr), payload_len))
                goto nla_put_failure;

out:
        genlmsg_end(msg, hdr);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

#define NET_DM_MAX_PACKET_SIZE (0xffff - NLA_HDRLEN - NLA_ALIGNTO)

static void net_dm_packet_report(struct sk_buff *skb)
{
        struct sk_buff *msg;
        size_t payload_len;
        int rc;

        /* Make sure we start copying the packet from the MAC header */
        if (skb->data > skb_mac_header(skb))
                skb_push(skb, skb->data - skb_mac_header(skb));
        else
                skb_pull(skb, skb_mac_header(skb) - skb->data);

        /* Ensure packet fits inside a single netlink attribute */
        payload_len = min_t(size_t, skb->len, NET_DM_MAX_PACKET_SIZE);
        if (net_dm_trunc_len)
                payload_len = min_t(size_t, net_dm_trunc_len, payload_len);

        msg = nlmsg_new(net_dm_packet_report_size(payload_len), GFP_KERNEL);
        if (!msg)
                goto out;

        rc = net_dm_packet_report_fill(msg, skb, payload_len);
        if (rc) {
                nlmsg_free(msg);
                goto out;
        }

        genlmsg_multicast(&net_drop_monitor_family, msg, 0, 0, GFP_KERNEL);

out:
        consume_skb(skb);
}

static void net_dm_packet_work(struct work_struct *work)
{
        struct per_cpu_dm_data *data;
        struct sk_buff_head list;
        struct sk_buff *skb;
        unsigned long flags;

        data = container_of(work, struct per_cpu_dm_data, dm_alert_work);

        __skb_queue_head_init(&list);

        spin_lock_irqsave(&data->drop_queue.lock, flags);
        skb_queue_splice_tail_init(&data->drop_queue, &list);
        spin_unlock_irqrestore(&data->drop_queue.lock, flags);

        while ((skb = __skb_dequeue(&list)))
                net_dm_packet_report(skb);
}

static size_t
net_dm_flow_action_cookie_size(const struct devlink_trap_metadata *hw_metadata)
{
        return hw_metadata->fa_cookie ?
               nla_total_size(hw_metadata->fa_cookie->cookie_len) : 0;
}

static size_t
net_dm_hw_packet_report_size(size_t payload_len,
                             const struct devlink_trap_metadata *hw_metadata)
{
        size_t size;

        size = nlmsg_msg_size(GENL_HDRLEN + net_drop_monitor_family.hdrsize);

        return NLMSG_ALIGN(size) +
               /* NET_DM_ATTR_ORIGIN */
               nla_total_size(sizeof(u16)) +
               /* NET_DM_ATTR_HW_TRAP_GROUP_NAME */
               nla_total_size(strlen(hw_metadata->trap_group_name) + 1) +
               /* NET_DM_ATTR_HW_TRAP_NAME */
               nla_total_size(strlen(hw_metadata->trap_name) + 1) +
               /* NET_DM_ATTR_IN_PORT */
               net_dm_in_port_size() +
               /* NET_DM_ATTR_FLOW_ACTION_COOKIE */
               net_dm_flow_action_cookie_size(hw_metadata) +
               /* NET_DM_ATTR_TIMESTAMP */
               nla_total_size(sizeof(u64)) +
               /* NET_DM_ATTR_ORIG_LEN */
               nla_total_size(sizeof(u32)) +
               /* NET_DM_ATTR_PROTO */
               nla_total_size(sizeof(u16)) +
               /* NET_DM_ATTR_PAYLOAD */
               nla_total_size(payload_len);
}

static int net_dm_hw_packet_report_fill(struct sk_buff *msg,
                                        struct sk_buff *skb, size_t payload_len)
{
        struct devlink_trap_metadata *hw_metadata;
        struct nlattr *attr;
        void *hdr;

        hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;

        hdr = genlmsg_put(msg, 0, 0, &net_drop_monitor_family, 0,
                          NET_DM_CMD_PACKET_ALERT);
        if (!hdr)
                return -EMSGSIZE;

        if (nla_put_u16(msg, NET_DM_ATTR_ORIGIN, NET_DM_ORIGIN_HW))
                goto nla_put_failure;

        if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_GROUP_NAME,
                           hw_metadata->trap_group_name))
                goto nla_put_failure;

        if (nla_put_string(msg, NET_DM_ATTR_HW_TRAP_NAME,
                           hw_metadata->trap_name))
                goto nla_put_failure;

        if (hw_metadata->input_dev) {
                struct net_device *dev = hw_metadata->input_dev;
                int rc;

                rc = net_dm_packet_report_in_port_put(msg, dev->ifindex,
                                                      dev->name);
                if (rc)
                        goto nla_put_failure;
        }

        if (hw_metadata->fa_cookie &&
            nla_put(msg, NET_DM_ATTR_FLOW_ACTION_COOKIE,
                    hw_metadata->fa_cookie->cookie_len,
                    hw_metadata->fa_cookie->cookie))
                goto nla_put_failure;

        if (nla_put_u64_64bit(msg, NET_DM_ATTR_TIMESTAMP,
                              ktime_to_ns(skb->tstamp), NET_DM_ATTR_PAD))
                goto nla_put_failure;

        if (nla_put_u32(msg, NET_DM_ATTR_ORIG_LEN, skb->len))
                goto nla_put_failure;

        if (!payload_len)
                goto out;

        if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol)))
                goto nla_put_failure;

        attr = skb_put(msg, nla_total_size(payload_len));
        attr->nla_type = NET_DM_ATTR_PAYLOAD;
        attr->nla_len = nla_attr_size(payload_len);
        if (skb_copy_bits(skb, 0, nla_data(attr), payload_len))
                goto nla_put_failure;

out:
        genlmsg_end(msg, hdr);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static struct devlink_trap_metadata *
net_dm_hw_metadata_copy(const struct devlink_trap_metadata *metadata)
{
        const struct flow_action_cookie *fa_cookie;
        struct devlink_trap_metadata *hw_metadata;
        const char *trap_group_name;
        const char *trap_name;

        hw_metadata = kzalloc(sizeof(*hw_metadata), GFP_ATOMIC);
        if (!hw_metadata)
                return NULL;

        trap_group_name = kstrdup(metadata->trap_group_name, GFP_ATOMIC);
        if (!trap_group_name)
                goto free_hw_metadata;
        hw_metadata->trap_group_name = trap_group_name;

        trap_name = kstrdup(metadata->trap_name, GFP_ATOMIC);
        if (!trap_name)
                goto free_trap_group;
        hw_metadata->trap_name = trap_name;

        if (metadata->fa_cookie) {
                size_t cookie_size = sizeof(*fa_cookie) +
                                     metadata->fa_cookie->cookie_len;

                fa_cookie = kmemdup(metadata->fa_cookie, cookie_size,
                                    GFP_ATOMIC);
                if (!fa_cookie)
                        goto free_trap_name;
                hw_metadata->fa_cookie = fa_cookie;
        }

        hw_metadata->input_dev = metadata->input_dev;
        netdev_hold(hw_metadata->input_dev, &hw_metadata->dev_tracker,
                    GFP_ATOMIC);

        return hw_metadata;

free_trap_name:
        kfree(trap_name);
free_trap_group:
        kfree(trap_group_name);
free_hw_metadata:
        kfree(hw_metadata);
        return NULL;
}

static void
net_dm_hw_metadata_free(struct devlink_trap_metadata *hw_metadata)
{
        netdev_put(hw_metadata->input_dev, &hw_metadata->dev_tracker);
        kfree(hw_metadata->fa_cookie);
        kfree(hw_metadata->trap_name);
        kfree(hw_metadata->trap_group_name);
        kfree(hw_metadata);
}

static void net_dm_hw_packet_report(struct sk_buff *skb)
{
        struct devlink_trap_metadata *hw_metadata;
        struct sk_buff *msg;
        size_t payload_len;
        int rc;

        if (skb->data > skb_mac_header(skb))
                skb_push(skb, skb->data - skb_mac_header(skb));
        else
                skb_pull(skb, skb_mac_header(skb) - skb->data);

        payload_len = min_t(size_t, skb->len, NET_DM_MAX_PACKET_SIZE);
        if (net_dm_trunc_len)
                payload_len = min_t(size_t, net_dm_trunc_len, payload_len);

        hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
        msg = nlmsg_new(net_dm_hw_packet_report_size(payload_len, hw_metadata),
                        GFP_KERNEL);
        if (!msg)
                goto out;

        rc = net_dm_hw_packet_report_fill(msg, skb, payload_len);
        if (rc) {
                nlmsg_free(msg);
                goto out;
        }

        genlmsg_multicast(&net_drop_monitor_family, msg, 0, 0, GFP_KERNEL);

out:
        net_dm_hw_metadata_free(NET_DM_SKB_CB(skb)->hw_metadata);
        consume_skb(skb);
}

static void net_dm_hw_packet_work(struct work_struct *work)
{
        struct per_cpu_dm_data *hw_data;
        struct sk_buff_head list;
        struct sk_buff *skb;
        unsigned long flags;

        hw_data = container_of(work, struct per_cpu_dm_data, dm_alert_work);

        __skb_queue_head_init(&list);

        spin_lock_irqsave(&hw_data->drop_queue.lock, flags);
        skb_queue_splice_tail_init(&hw_data->drop_queue, &list);
        spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);

        while ((skb = __skb_dequeue(&list)))
                net_dm_hw_packet_report(skb);
}

static void
net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink,
                            struct sk_buff *skb,
                            const struct devlink_trap_metadata *metadata)
{
        struct devlink_trap_metadata *n_hw_metadata;
        ktime_t tstamp = ktime_get_real();
        struct per_cpu_dm_data *hw_data;
        struct sk_buff *nskb;
        unsigned long flags;

        if (metadata->trap_type == DEVLINK_TRAP_TYPE_CONTROL)
                return;

        if (!skb_mac_header_was_set(skb))
                return;

        nskb = skb_clone(skb, GFP_ATOMIC);
        if (!nskb)
                return;

        n_hw_metadata = net_dm_hw_metadata_copy(metadata);
        if (!n_hw_metadata)
                goto free;

        NET_DM_SKB_CB(nskb)->hw_metadata = n_hw_metadata;
        nskb->tstamp = tstamp;

        hw_data = this_cpu_ptr(&dm_hw_cpu_data);

        spin_lock_irqsave(&hw_data->drop_queue.lock, flags);
        if (skb_queue_len(&hw_data->drop_queue) < net_dm_queue_len)
                __skb_queue_tail(&hw_data->drop_queue, nskb);
        else
                goto unlock_free;
        spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);

        schedule_work(&hw_data->dm_alert_work);

        return;

unlock_free:
        spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags);
        u64_stats_update_begin(&hw_data->stats.syncp);
        u64_stats_inc(&hw_data->stats.dropped);
        u64_stats_update_end(&hw_data->stats.syncp);
        net_dm_hw_metadata_free(n_hw_metadata);
free:
        consume_skb(nskb);
}

static const struct net_dm_alert_ops net_dm_alert_packet_ops = {
        .kfree_skb_probe        = net_dm_packet_trace_kfree_skb_hit,
        .napi_poll_probe        = net_dm_packet_trace_napi_poll_hit,
        .work_item_func                = net_dm_packet_work,
        .hw_work_item_func        = net_dm_hw_packet_work,
        .hw_trap_probe                = net_dm_hw_trap_packet_probe,
};

static const struct net_dm_alert_ops *net_dm_alert_ops_arr[] = {
        [NET_DM_ALERT_MODE_SUMMARY]        = &net_dm_alert_summary_ops,
        [NET_DM_ALERT_MODE_PACKET]        = &net_dm_alert_packet_ops,
};

#if IS_ENABLED(CONFIG_NET_DEVLINK)
static int net_dm_hw_probe_register(const struct net_dm_alert_ops *ops)
{
        return register_trace_devlink_trap_report(ops->hw_trap_probe, NULL);
}

static void net_dm_hw_probe_unregister(const struct net_dm_alert_ops *ops)
{
        unregister_trace_devlink_trap_report(ops->hw_trap_probe, NULL);
        tracepoint_synchronize_unregister();
}
#else
static int net_dm_hw_probe_register(const struct net_dm_alert_ops *ops)
{
        return -EOPNOTSUPP;
}

static void net_dm_hw_probe_unregister(const struct net_dm_alert_ops *ops)
{
}
#endif

static int net_dm_hw_monitor_start(struct netlink_ext_ack *extack)
{
        const struct net_dm_alert_ops *ops;
        int cpu, rc;

        if (monitor_hw) {
                NL_SET_ERR_MSG_MOD(extack, "Hardware monitoring already enabled");
                return -EAGAIN;
        }

        ops = net_dm_alert_ops_arr[net_dm_alert_mode];

        if (!try_module_get(THIS_MODULE)) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to take reference on module");
                return -ENODEV;
        }

        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
                struct net_dm_hw_entries *hw_entries;

                INIT_WORK(&hw_data->dm_alert_work, ops->hw_work_item_func);
                timer_setup(&hw_data->send_timer, sched_send_work, 0);
                hw_entries = net_dm_hw_reset_per_cpu_data(hw_data);
                kfree(hw_entries);
        }

        rc = net_dm_hw_probe_register(ops);
        if (rc) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to connect probe to devlink_trap_probe() tracepoint");
                goto err_module_put;
        }

        monitor_hw = true;

        return 0;

err_module_put:
        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
                struct sk_buff *skb;

                del_timer_sync(&hw_data->send_timer);
                cancel_work_sync(&hw_data->dm_alert_work);
                while ((skb = __skb_dequeue(&hw_data->drop_queue))) {
                        struct devlink_trap_metadata *hw_metadata;

                        hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
                        net_dm_hw_metadata_free(hw_metadata);
                        consume_skb(skb);
                }
        }
        module_put(THIS_MODULE);
        return rc;
}

static void net_dm_hw_monitor_stop(struct netlink_ext_ack *extack)
{
        const struct net_dm_alert_ops *ops;
        int cpu;

        if (!monitor_hw) {
                NL_SET_ERR_MSG_MOD(extack, "Hardware monitoring already disabled");
                return;
        }

        ops = net_dm_alert_ops_arr[net_dm_alert_mode];

        monitor_hw = false;

        net_dm_hw_probe_unregister(ops);

        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
                struct sk_buff *skb;

                del_timer_sync(&hw_data->send_timer);
                cancel_work_sync(&hw_data->dm_alert_work);
                while ((skb = __skb_dequeue(&hw_data->drop_queue))) {
                        struct devlink_trap_metadata *hw_metadata;

                        hw_metadata = NET_DM_SKB_CB(skb)->hw_metadata;
                        net_dm_hw_metadata_free(hw_metadata);
                        consume_skb(skb);
                }
        }

        module_put(THIS_MODULE);
}

static int net_dm_trace_on_set(struct netlink_ext_ack *extack)
{
        const struct net_dm_alert_ops *ops;
        int cpu, rc;

        ops = net_dm_alert_ops_arr[net_dm_alert_mode];

        if (!try_module_get(THIS_MODULE)) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to take reference on module");
                return -ENODEV;
        }

        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu);
                struct sk_buff *skb;

                INIT_WORK(&data->dm_alert_work, ops->work_item_func);
                timer_setup(&data->send_timer, sched_send_work, 0);
                /* Allocate a new per-CPU skb for the summary alert message and
                 * free the old one which might contain stale data from
                 * previous tracing.
                 */
                skb = reset_per_cpu_data(data);
                consume_skb(skb);
        }

        rc = register_trace_kfree_skb(ops->kfree_skb_probe, NULL);
        if (rc) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to connect probe to kfree_skb() tracepoint");
                goto err_module_put;
        }

        rc = register_trace_napi_poll(ops->napi_poll_probe, NULL);
        if (rc) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to connect probe to napi_poll() tracepoint");
                goto err_unregister_trace;
        }

        return 0;

err_unregister_trace:
        unregister_trace_kfree_skb(ops->kfree_skb_probe, NULL);
err_module_put:
        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu);
                struct sk_buff *skb;

                del_timer_sync(&data->send_timer);
                cancel_work_sync(&data->dm_alert_work);
                while ((skb = __skb_dequeue(&data->drop_queue)))
                        consume_skb(skb);
        }
        module_put(THIS_MODULE);
        return rc;
}

static void net_dm_trace_off_set(void)
{
        const struct net_dm_alert_ops *ops;
        int cpu;

        ops = net_dm_alert_ops_arr[net_dm_alert_mode];

        unregister_trace_napi_poll(ops->napi_poll_probe, NULL);
        unregister_trace_kfree_skb(ops->kfree_skb_probe, NULL);

        tracepoint_synchronize_unregister();

        /* Make sure we do not send notifications to user space after request
         * to stop tracing returns.
         */
        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu);
                struct sk_buff *skb;

                del_timer_sync(&data->send_timer);
                cancel_work_sync(&data->dm_alert_work);
                while ((skb = __skb_dequeue(&data->drop_queue)))
                        consume_skb(skb);
        }

        module_put(THIS_MODULE);
}

static int set_all_monitor_traces(int state, struct netlink_ext_ack *extack)
{
        int rc = 0;

        if (state == trace_state) {
                NL_SET_ERR_MSG_MOD(extack, "Trace state already set to requested state");
                return -EAGAIN;
        }

        switch (state) {
        case TRACE_ON:
                rc = net_dm_trace_on_set(extack);
                break;
        case TRACE_OFF:
                net_dm_trace_off_set();
                break;
        default:
                rc = 1;
                break;
        }

        if (!rc)
                trace_state = state;
        else
                rc = -EINPROGRESS;

        return rc;
}

static bool net_dm_is_monitoring(void)
{
        return trace_state == TRACE_ON || monitor_hw;
}

static int net_dm_alert_mode_get_from_info(struct genl_info *info,
                                           enum net_dm_alert_mode *p_alert_mode)
{
        u8 val;

        val = nla_get_u8(info->attrs[NET_DM_ATTR_ALERT_MODE]);

        switch (val) {
        case NET_DM_ALERT_MODE_SUMMARY:
        case NET_DM_ALERT_MODE_PACKET:
                *p_alert_mode = val;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static int net_dm_alert_mode_set(struct genl_info *info)
{
        struct netlink_ext_ack *extack = info->extack;
        enum net_dm_alert_mode alert_mode;
        int rc;

        if (!info->attrs[NET_DM_ATTR_ALERT_MODE])
                return 0;

        rc = net_dm_alert_mode_get_from_info(info, &alert_mode);
        if (rc) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid alert mode");
                return -EINVAL;
        }

        net_dm_alert_mode = alert_mode;

        return 0;
}

static void net_dm_trunc_len_set(struct genl_info *info)
{
        if (!info->attrs[NET_DM_ATTR_TRUNC_LEN])
                return;

        net_dm_trunc_len = nla_get_u32(info->attrs[NET_DM_ATTR_TRUNC_LEN]);
}

static void net_dm_queue_len_set(struct genl_info *info)
{
        if (!info->attrs[NET_DM_ATTR_QUEUE_LEN])
                return;

        net_dm_queue_len = nla_get_u32(info->attrs[NET_DM_ATTR_QUEUE_LEN]);
}

static int net_dm_cmd_config(struct sk_buff *skb,
                        struct genl_info *info)
{
        struct netlink_ext_ack *extack = info->extack;
        int rc;

        if (net_dm_is_monitoring()) {
                NL_SET_ERR_MSG_MOD(extack, "Cannot configure drop monitor during monitoring");
                return -EBUSY;
        }

        rc = net_dm_alert_mode_set(info);
        if (rc)
                return rc;

        net_dm_trunc_len_set(info);

        net_dm_queue_len_set(info);

        return 0;
}

static int net_dm_monitor_start(bool set_sw, bool set_hw,
                                struct netlink_ext_ack *extack)
{
        bool sw_set = false;
        int rc;

        if (set_sw) {
                rc = set_all_monitor_traces(TRACE_ON, extack);
                if (rc)
                        return rc;
                sw_set = true;
        }

        if (set_hw) {
                rc = net_dm_hw_monitor_start(extack);
                if (rc)
                        goto err_monitor_hw;
        }

        return 0;

err_monitor_hw:
        if (sw_set)
                set_all_monitor_traces(TRACE_OFF, extack);
        return rc;
}

static void net_dm_monitor_stop(bool set_sw, bool set_hw,
                                struct netlink_ext_ack *extack)
{
        if (set_hw)
                net_dm_hw_monitor_stop(extack);
        if (set_sw)
                set_all_monitor_traces(TRACE_OFF, extack);
}

static int net_dm_cmd_trace(struct sk_buff *skb,
                        struct genl_info *info)
{
        bool set_sw = !!info->attrs[NET_DM_ATTR_SW_DROPS];
        bool set_hw = !!info->attrs[NET_DM_ATTR_HW_DROPS];
        struct netlink_ext_ack *extack = info->extack;

        /* To maintain backward compatibility, we start / stop monitoring of
         * software drops if no flag is specified.
         */
        if (!set_sw && !set_hw)
                set_sw = true;

        switch (info->genlhdr->cmd) {
        case NET_DM_CMD_START:
                return net_dm_monitor_start(set_sw, set_hw, extack);
        case NET_DM_CMD_STOP:
                net_dm_monitor_stop(set_sw, set_hw, extack);
                return 0;
        }

        return -EOPNOTSUPP;
}

static int net_dm_config_fill(struct sk_buff *msg, struct genl_info *info)
{
        void *hdr;

        hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
                          &net_drop_monitor_family, 0, NET_DM_CMD_CONFIG_NEW);
        if (!hdr)
                return -EMSGSIZE;

        if (nla_put_u8(msg, NET_DM_ATTR_ALERT_MODE, net_dm_alert_mode))
                goto nla_put_failure;

        if (nla_put_u32(msg, NET_DM_ATTR_TRUNC_LEN, net_dm_trunc_len))
                goto nla_put_failure;

        if (nla_put_u32(msg, NET_DM_ATTR_QUEUE_LEN, net_dm_queue_len))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static int net_dm_cmd_config_get(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        int rc;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        rc = net_dm_config_fill(msg, info);
        if (rc)
                goto free_msg;

        return genlmsg_reply(msg, info);

free_msg:
        nlmsg_free(msg);
        return rc;
}

static void net_dm_stats_read(struct net_dm_stats *stats)
{
        int cpu;

        memset(stats, 0, sizeof(*stats));
        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *data = &per_cpu(dm_cpu_data, cpu);
                struct net_dm_stats *cpu_stats = &data->stats;
                unsigned int start;
                u64 dropped;

                do {
                        start = u64_stats_fetch_begin(&cpu_stats->syncp);
                        dropped = u64_stats_read(&cpu_stats->dropped);
                } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));

                u64_stats_add(&stats->dropped, dropped);
        }
}

static int net_dm_stats_put(struct sk_buff *msg)
{
        struct net_dm_stats stats;
        struct nlattr *attr;

        net_dm_stats_read(&stats);

        attr = nla_nest_start(msg, NET_DM_ATTR_STATS);
        if (!attr)
                return -EMSGSIZE;

        if (nla_put_u64_64bit(msg, NET_DM_ATTR_STATS_DROPPED,
                              u64_stats_read(&stats.dropped), NET_DM_ATTR_PAD))
                goto nla_put_failure;

        nla_nest_end(msg, attr);

        return 0;

nla_put_failure:
        nla_nest_cancel(msg, attr);
        return -EMSGSIZE;
}

static void net_dm_hw_stats_read(struct net_dm_stats *stats)
{
        int cpu;

        memset(stats, 0, sizeof(*stats));
        for_each_possible_cpu(cpu) {
                struct per_cpu_dm_data *hw_data = &per_cpu(dm_hw_cpu_data, cpu);
                struct net_dm_stats *cpu_stats = &hw_data->stats;
                unsigned int start;
                u64 dropped;

                do {
                        start = u64_stats_fetch_begin(&cpu_stats->syncp);
                        dropped = u64_stats_read(&cpu_stats->dropped);
                } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));

                u64_stats_add(&stats->dropped, dropped);
        }
}

static int net_dm_hw_stats_put(struct sk_buff *msg)
{
        struct net_dm_stats stats;
        struct nlattr *attr;

        net_dm_hw_stats_read(&stats);

        attr = nla_nest_start(msg, NET_DM_ATTR_HW_STATS);
        if (!attr)
                return -EMSGSIZE;

        if (nla_put_u64_64bit(msg, NET_DM_ATTR_STATS_DROPPED,
                              u64_stats_read(&stats.dropped), NET_DM_ATTR_PAD))
                goto nla_put_failure;

        nla_nest_end(msg, attr);

        return 0;

nla_put_failure:
        nla_nest_cancel(msg, attr);
        return -EMSGSIZE;
}

static int net_dm_stats_fill(struct sk_buff *msg, struct genl_info *info)
{
        void *hdr;
        int rc;

        hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
                          &net_drop_monitor_family, 0, NET_DM_CMD_STATS_NEW);
        if (!hdr)
                return -EMSGSIZE;

        rc = net_dm_stats_put(msg);
        if (rc)
                goto nla_put_failure;

        rc = net_dm_hw_stats_put(msg);
        if (rc)
                goto nla_put_failure;

        genlmsg_end(msg, hdr);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static int net_dm_cmd_stats_get(struct sk_buff *skb, struct genl_info *info)
{
        struct sk_buff *msg;
        int rc;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        rc = net_dm_stats_fill(msg, info);
        if (rc)
                goto free_msg;

        return genlmsg_reply(msg, info);

free_msg:
        nlmsg_free(msg);
        return rc;
}

static int dropmon_net_event(struct notifier_block *ev_block,
                             unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct dm_hw_stat_delta *stat;

        switch (event) {
        case NETDEV_REGISTER:
                if (WARN_ON_ONCE(rtnl_dereference(dev->dm_private)))
                        break;
                stat = kzalloc(sizeof(*stat), GFP_KERNEL);
                if (!stat)
                        break;

                stat->last_rx = jiffies;
                rcu_assign_pointer(dev->dm_private, stat);

                break;
        case NETDEV_UNREGISTER:
                stat = rtnl_dereference(dev->dm_private);
                if (stat) {
                        rcu_assign_pointer(dev->dm_private, NULL);
                        kfree_rcu(stat, rcu);
                }
                break;
        }
        return NOTIFY_DONE;
}

static const struct nla_policy net_dm_nl_policy[NET_DM_ATTR_MAX + 1] = {
        [NET_DM_ATTR_UNSPEC] = { .strict_start_type = NET_DM_ATTR_UNSPEC + 1 },
        [NET_DM_ATTR_ALERT_MODE] = { .type = NLA_U8 },
        [NET_DM_ATTR_TRUNC_LEN] = { .type = NLA_U32 },
        [NET_DM_ATTR_QUEUE_LEN] = { .type = NLA_U32 },
        [NET_DM_ATTR_SW_DROPS]        = {. type = NLA_FLAG },
        [NET_DM_ATTR_HW_DROPS]        = {. type = NLA_FLAG },
};

static const struct genl_small_ops dropmon_ops[] = {
        {
                .cmd = NET_DM_CMD_CONFIG,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = net_dm_cmd_config,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = NET_DM_CMD_START,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = net_dm_cmd_trace,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = NET_DM_CMD_STOP,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit = net_dm_cmd_trace,
                .flags = GENL_ADMIN_PERM,
        },
        {
                .cmd = NET_DM_CMD_CONFIG_GET,
                .doit = net_dm_cmd_config_get,
        },
        {
                .cmd = NET_DM_CMD_STATS_GET,
                .doit = net_dm_cmd_stats_get,
        },
};

static int net_dm_nl_pre_doit(const struct genl_split_ops *ops,
                              struct sk_buff *skb, struct genl_info *info)
{
        mutex_lock(&net_dm_mutex);

        return 0;
}

static void net_dm_nl_post_doit(const struct genl_split_ops *ops,
                                struct sk_buff *skb, struct genl_info *info)
{
        mutex_unlock(&net_dm_mutex);
}

static struct genl_family net_drop_monitor_family __ro_after_init = {
        .hdrsize        = 0,
        .name           = "NET_DM",
        .version        = 2,
        .maxattr        = NET_DM_ATTR_MAX,
        .policy                = net_dm_nl_policy,
        .pre_doit        = net_dm_nl_pre_doit,
        .post_doit        = net_dm_nl_post_doit,
        .module                = THIS_MODULE,
        .small_ops        = dropmon_ops,
        .n_small_ops        = ARRAY_SIZE(dropmon_ops),
        .resv_start_op        = NET_DM_CMD_STATS_GET + 1,
        .mcgrps                = dropmon_mcgrps,
        .n_mcgrps        = ARRAY_SIZE(dropmon_mcgrps),
};

static struct notifier_block dropmon_net_notifier = {
        .notifier_call = dropmon_net_event
};

static void __net_dm_cpu_data_init(struct per_cpu_dm_data *data)
{
        raw_spin_lock_init(&data->lock);
        skb_queue_head_init(&data->drop_queue);
        u64_stats_init(&data->stats.syncp);
}

static void __net_dm_cpu_data_fini(struct per_cpu_dm_data *data)
{
        WARN_ON(!skb_queue_empty(&data->drop_queue));
}

static void net_dm_cpu_data_init(int cpu)
{
        struct per_cpu_dm_data *data;

        data = &per_cpu(dm_cpu_data, cpu);
        __net_dm_cpu_data_init(data);
}

static void net_dm_cpu_data_fini(int cpu)
{
        struct per_cpu_dm_data *data;

        data = &per_cpu(dm_cpu_data, cpu);
        /* At this point, we should have exclusive access
         * to this struct and can free the skb inside it.
         */
        consume_skb(data->skb);
        __net_dm_cpu_data_fini(data);
}

static void net_dm_hw_cpu_data_init(int cpu)
{
        struct per_cpu_dm_data *hw_data;

        hw_data = &per_cpu(dm_hw_cpu_data, cpu);
        __net_dm_cpu_data_init(hw_data);
}

static void net_dm_hw_cpu_data_fini(int cpu)
{
        struct per_cpu_dm_data *hw_data;

        hw_data = &per_cpu(dm_hw_cpu_data, cpu);
        kfree(hw_data->hw_entries);
        __net_dm_cpu_data_fini(hw_data);
}

static int __init init_net_drop_monitor(void)
{
        int cpu, rc;

        pr_info("Initializing network drop monitor service\n");

        if (sizeof(void *) > 8) {
                pr_err("Unable to store program counters on this arch, Drop monitor failed\n");
                return -ENOSPC;
        }

        rc = genl_register_family(&net_drop_monitor_family);
        if (rc) {
                pr_err("Could not create drop monitor netlink family\n");
                return rc;
        }
        WARN_ON(net_drop_monitor_family.mcgrp_offset != NET_DM_GRP_ALERT);

        rc = register_netdevice_notifier(&dropmon_net_notifier);
        if (rc < 0) {
                pr_crit("Failed to register netdevice notifier\n");
                goto out_unreg;
        }

        rc = 0;

        for_each_possible_cpu(cpu) {
                net_dm_cpu_data_init(cpu);
                net_dm_hw_cpu_data_init(cpu);
        }

        goto out;

out_unreg:
        genl_unregister_family(&net_drop_monitor_family);
out:
        return rc;
}

static void exit_net_drop_monitor(void)
{
        int cpu;

        BUG_ON(unregister_netdevice_notifier(&dropmon_net_notifier));

        /*
         * Because of the module_get/put we do in the trace state change path
         * we are guaranteed not to have any current users when we get here
         */

        for_each_possible_cpu(cpu) {
                net_dm_hw_cpu_data_fini(cpu);
                net_dm_cpu_data_fini(cpu);
        }

        BUG_ON(genl_unregister_family(&net_drop_monitor_family));
}

module_init(init_net_drop_monitor);
module_exit(exit_net_drop_monitor);

MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Neil Horman <nhorman@tuxdriver.com>");
MODULE_ALIAS_GENL_FAMILY("NET_DM");
MODULE_DESCRIPTION("Monitoring code for network dropped packet alerts");





















   52 































   70 


   70 

   70 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#include <linux/notifier.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <net/net_namespace.h>
#include <net/fib_notifier.h>
#include <net/netns/ipv6.h>
#include <net/ip6_fib.h>

int call_fib6_notifier(struct notifier_block *nb,
                       enum fib_event_type event_type,
                       struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifier(nb, event_type, info);
}

int call_fib6_notifiers(struct net *net, enum fib_event_type event_type,
                        struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifiers(net, event_type, info);
}

static unsigned int fib6_seq_read(const struct net *net)
{
        return fib6_tables_seq_read(net) + fib6_rules_seq_read(net);
}

static int fib6_dump(struct net *net, struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        int err;

        err = fib6_rules_dump(net, nb, extack);
        if (err)
                return err;

        return fib6_tables_dump(net, nb, extack);
}

static const struct fib_notifier_ops fib6_notifier_ops_template = {
        .family                = AF_INET6,
        .fib_seq_read        = fib6_seq_read,
        .fib_dump        = fib6_dump,
        .owner                = THIS_MODULE,
};

int __net_init fib6_notifier_init(struct net *net)
{
        struct fib_notifier_ops *ops;

        ops = fib_notifier_ops_register(&fib6_notifier_ops_template, net);
        if (IS_ERR(ops))
                return PTR_ERR(ops);
        net->ipv6.notifier_ops = ops;

        return 0;
}

void __net_exit fib6_notifier_exit(struct net *net)
{
        fib_notifier_ops_unregister(net->ipv6.notifier_ops);
}
















































































































































































































































   18 



   19 

   18 
   18 






























































   51 


   50 





   51 



   51 
   49 

   51 

   19 

   35 


   36 


   17 


























































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains functions which manage clock event devices.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 */

#include <linux/clockchips.h>
#include <linux/hrtimer.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/device.h>

#include "tick-internal.h"

/* The registered clock event devices */
static LIST_HEAD(clockevent_devices);
static LIST_HEAD(clockevents_released);
/* Protection for the above */
static DEFINE_RAW_SPINLOCK(clockevents_lock);
/* Protection for unbind operations */
static DEFINE_MUTEX(clockevents_mutex);

struct ce_unbind {
        struct clock_event_device *ce;
        int res;
};

static u64 cev_delta2ns(unsigned long latch, struct clock_event_device *evt,
                        bool ismax)
{
        u64 clc = (u64) latch << evt->shift;
        u64 rnd;

        if (WARN_ON(!evt->mult))
                evt->mult = 1;
        rnd = (u64) evt->mult - 1;

        /*
         * Upper bound sanity check. If the backwards conversion is
         * not equal latch, we know that the above shift overflowed.
         */
        if ((clc >> evt->shift) != (u64)latch)
                clc = ~0ULL;

        /*
         * Scaled math oddities:
         *
         * For mult <= (1 << shift) we can safely add mult - 1 to
         * prevent integer rounding loss. So the backwards conversion
         * from nsec to device ticks will be correct.
         *
         * For mult > (1 << shift), i.e. device frequency is > 1GHz we
         * need to be careful. Adding mult - 1 will result in a value
         * which when converted back to device ticks can be larger
         * than latch by up to (mult - 1) >> shift. For the min_delta
         * calculation we still want to apply this in order to stay
         * above the minimum device ticks limit. For the upper limit
         * we would end up with a latch value larger than the upper
         * limit of the device, so we omit the add to stay below the
         * device upper boundary.
         *
         * Also omit the add if it would overflow the u64 boundary.
         */
        if ((~0ULL - clc > rnd) &&
            (!ismax || evt->mult <= (1ULL << evt->shift)))
                clc += rnd;

        do_div(clc, evt->mult);

        /* Deltas less than 1usec are pointless noise */
        return clc > 1000 ? clc : 1000;
}

/**
 * clockevent_delta2ns - Convert a latch value (device ticks) to nanoseconds
 * @latch:        value to convert
 * @evt:        pointer to clock event device descriptor
 *
 * Math helper, returns latch value converted to nanoseconds (bound checked)
 */
u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt)
{
        return cev_delta2ns(latch, evt, false);
}
EXPORT_SYMBOL_GPL(clockevent_delta2ns);

static int __clockevents_switch_state(struct clock_event_device *dev,
                                      enum clock_event_state state)
{
        if (dev->features & CLOCK_EVT_FEAT_DUMMY)
                return 0;

        /* Transition with new state-specific callbacks */
        switch (state) {
        case CLOCK_EVT_STATE_DETACHED:
                /* The clockevent device is getting replaced. Shut it down. */

        case CLOCK_EVT_STATE_SHUTDOWN:
                if (dev->set_state_shutdown)
                        return dev->set_state_shutdown(dev);
                return 0;

        case CLOCK_EVT_STATE_PERIODIC:
                /* Core internal bug */
                if (!(dev->features & CLOCK_EVT_FEAT_PERIODIC))
                        return -ENOSYS;
                if (dev->set_state_periodic)
                        return dev->set_state_periodic(dev);
                return 0;

        case CLOCK_EVT_STATE_ONESHOT:
                /* Core internal bug */
                if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                        return -ENOSYS;
                if (dev->set_state_oneshot)
                        return dev->set_state_oneshot(dev);
                return 0;

        case CLOCK_EVT_STATE_ONESHOT_STOPPED:
                /* Core internal bug */
                if (WARN_ONCE(!clockevent_state_oneshot(dev),
                              "Current state: %d\n",
                              clockevent_get_state(dev)))
                        return -EINVAL;

                if (dev->set_state_oneshot_stopped)
                        return dev->set_state_oneshot_stopped(dev);
                else
                        return -ENOSYS;

        default:
                return -ENOSYS;
        }
}

/**
 * clockevents_switch_state - set the operating state of a clock event device
 * @dev:        device to modify
 * @state:        new state
 *
 * Must be called with interrupts disabled !
 */
void clockevents_switch_state(struct clock_event_device *dev,
                              enum clock_event_state state)
{
        if (clockevent_get_state(dev) != state) {
                if (__clockevents_switch_state(dev, state))
                        return;

                clockevent_set_state(dev, state);

                /*
                 * A nsec2cyc multiplicator of 0 is invalid and we'd crash
                 * on it, so fix it up and emit a warning:
                 */
                if (clockevent_state_oneshot(dev)) {
                        if (WARN_ON(!dev->mult))
                                dev->mult = 1;
                }
        }
}

/**
 * clockevents_shutdown - shutdown the device and clear next_event
 * @dev:        device to shutdown
 */
void clockevents_shutdown(struct clock_event_device *dev)
{
        clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
        dev->next_event = KTIME_MAX;
}

/**
 * clockevents_tick_resume -        Resume the tick device before using it again
 * @dev:                        device to resume
 */
int clockevents_tick_resume(struct clock_event_device *dev)
{
        int ret = 0;

        if (dev->tick_resume)
                ret = dev->tick_resume(dev);

        return ret;
}

#ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST

/* Limit min_delta to a jiffy */
#define MIN_DELTA_LIMIT                (NSEC_PER_SEC / HZ)

/**
 * clockevents_increase_min_delta - raise minimum delta of a clock event device
 * @dev:       device to increase the minimum delta
 *
 * Returns 0 on success, -ETIME when the minimum delta reached the limit.
 */
static int clockevents_increase_min_delta(struct clock_event_device *dev)
{
        /* Nothing to do if we already reached the limit */
        if (dev->min_delta_ns >= MIN_DELTA_LIMIT) {
                printk_deferred(KERN_WARNING
                                "CE: Reprogramming failure. Giving up\n");
                dev->next_event = KTIME_MAX;
                return -ETIME;
        }

        if (dev->min_delta_ns < 5000)
                dev->min_delta_ns = 5000;
        else
                dev->min_delta_ns += dev->min_delta_ns >> 1;

        if (dev->min_delta_ns > MIN_DELTA_LIMIT)
                dev->min_delta_ns = MIN_DELTA_LIMIT;

        printk_deferred(KERN_WARNING
                        "CE: %s increased min_delta_ns to %llu nsec\n",
                        dev->name ? dev->name : "?",
                        (unsigned long long) dev->min_delta_ns);
        return 0;
}

/**
 * clockevents_program_min_delta - Set clock event device to the minimum delay.
 * @dev:        device to program
 *
 * Returns 0 on success, -ETIME when the retry loop failed.
 */
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
        unsigned long long clc;
        int64_t delta;
        int i;

        for (i = 0;;) {
                delta = dev->min_delta_ns;
                dev->next_event = ktime_add_ns(ktime_get(), delta);

                if (clockevent_state_shutdown(dev))
                        return 0;

                dev->retries++;
                clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
                if (dev->set_next_event((unsigned long) clc, dev) == 0)
                        return 0;

                if (++i > 2) {
                        /*
                         * We tried 3 times to program the device with the
                         * given min_delta_ns. Try to increase the minimum
                         * delta, if that fails as well get out of here.
                         */
                        if (clockevents_increase_min_delta(dev))
                                return -ETIME;
                        i = 0;
                }
        }
}

#else  /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */

/**
 * clockevents_program_min_delta - Set clock event device to the minimum delay.
 * @dev:        device to program
 *
 * Returns 0 on success, -ETIME when the retry loop failed.
 */
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
        unsigned long long clc;
        int64_t delta = 0;
        int i;

        for (i = 0; i < 10; i++) {
                delta += dev->min_delta_ns;
                dev->next_event = ktime_add_ns(ktime_get(), delta);

                if (clockevent_state_shutdown(dev))
                        return 0;

                dev->retries++;
                clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
                if (dev->set_next_event((unsigned long) clc, dev) == 0)
                        return 0;
        }
        return -ETIME;
}

#endif /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */

/**
 * clockevents_program_event - Reprogram the clock event device.
 * @dev:        device to program
 * @expires:        absolute expiry time (monotonic clock)
 * @force:        program minimum delay if expires can not be set
 *
 * Returns 0 on success, -ETIME when the event is in the past.
 */
int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
                              bool force)
{
        unsigned long long clc;
        int64_t delta;
        int rc;

        if (WARN_ON_ONCE(expires < 0))
                return -ETIME;

        dev->next_event = expires;

        if (clockevent_state_shutdown(dev))
                return 0;

        /* We must be in ONESHOT state here */
        WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n",
                  clockevent_get_state(dev));

        /* Shortcut for clockevent devices that can deal with ktime. */
        if (dev->features & CLOCK_EVT_FEAT_KTIME)
                return dev->set_next_ktime(expires, dev);

        delta = ktime_to_ns(ktime_sub(expires, ktime_get()));
        if (delta <= 0)
                return force ? clockevents_program_min_delta(dev) : -ETIME;

        delta = min(delta, (int64_t) dev->max_delta_ns);
        delta = max(delta, (int64_t) dev->min_delta_ns);

        clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
        rc = dev->set_next_event((unsigned long) clc, dev);

        return (rc && force) ? clockevents_program_min_delta(dev) : rc;
}

/*
 * Called after a clockevent has been added which might
 * have replaced a current regular or broadcast device. A
 * released normal device might be a suitable replacement
 * for the current broadcast device. Similarly a released
 * broadcast device might be a suitable replacement for a
 * normal device.
 */
static void clockevents_notify_released(void)
{
        struct clock_event_device *dev;

        /*
         * Keep iterating as long as tick_check_new_device()
         * replaces a device.
         */
        while (!list_empty(&clockevents_released)) {
                dev = list_entry(clockevents_released.next,
                                 struct clock_event_device, list);
                list_move(&dev->list, &clockevent_devices);
                tick_check_new_device(dev);
        }
}

/*
 * Try to install a replacement clock event device
 */
static int clockevents_replace(struct clock_event_device *ced)
{
        struct clock_event_device *dev, *newdev = NULL;

        list_for_each_entry(dev, &clockevent_devices, list) {
                if (dev == ced || !clockevent_state_detached(dev))
                        continue;

                if (!tick_check_replacement(newdev, dev))
                        continue;

                if (!try_module_get(dev->owner))
                        continue;

                if (newdev)
                        module_put(newdev->owner);
                newdev = dev;
        }
        if (newdev) {
                tick_install_replacement(newdev);
                list_del_init(&ced->list);
        }
        return newdev ? 0 : -EBUSY;
}

/*
 * Called with clockevents_mutex and clockevents_lock held
 */
static int __clockevents_try_unbind(struct clock_event_device *ced, int cpu)
{
        /* Fast track. Device is unused */
        if (clockevent_state_detached(ced)) {
                list_del_init(&ced->list);
                return 0;
        }

        return ced == per_cpu(tick_cpu_device, cpu).evtdev ? -EAGAIN : -EBUSY;
}

/*
 * SMP function call to unbind a device
 */
static void __clockevents_unbind(void *arg)
{
        struct ce_unbind *cu = arg;
        int res;

        raw_spin_lock(&clockevents_lock);
        res = __clockevents_try_unbind(cu->ce, smp_processor_id());
        if (res == -EAGAIN)
                res = clockevents_replace(cu->ce);
        cu->res = res;
        raw_spin_unlock(&clockevents_lock);
}

/*
 * Issues smp function call to unbind a per cpu device. Called with
 * clockevents_mutex held.
 */
static int clockevents_unbind(struct clock_event_device *ced, int cpu)
{
        struct ce_unbind cu = { .ce = ced, .res = -ENODEV };

        smp_call_function_single(cpu, __clockevents_unbind, &cu, 1);
        return cu.res;
}

/*
 * Unbind a clockevents device.
 */
int clockevents_unbind_device(struct clock_event_device *ced, int cpu)
{
        int ret;

        mutex_lock(&clockevents_mutex);
        ret = clockevents_unbind(ced, cpu);
        mutex_unlock(&clockevents_mutex);
        return ret;
}
EXPORT_SYMBOL_GPL(clockevents_unbind_device);

/**
 * clockevents_register_device - register a clock event device
 * @dev:        device to register
 */
void clockevents_register_device(struct clock_event_device *dev)
{
        unsigned long flags;

        /* Initialize state to DETACHED */
        clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);

        if (!dev->cpumask) {
                WARN_ON(num_possible_cpus() > 1);
                dev->cpumask = cpumask_of(smp_processor_id());
        }

        if (dev->cpumask == cpu_all_mask) {
                WARN(1, "%s cpumask == cpu_all_mask, using cpu_possible_mask instead\n",
                     dev->name);
                dev->cpumask = cpu_possible_mask;
        }

        raw_spin_lock_irqsave(&clockevents_lock, flags);

        list_add(&dev->list, &clockevent_devices);
        tick_check_new_device(dev);
        clockevents_notify_released();

        raw_spin_unlock_irqrestore(&clockevents_lock, flags);
}
EXPORT_SYMBOL_GPL(clockevents_register_device);

static void clockevents_config(struct clock_event_device *dev, u32 freq)
{
        u64 sec;

        if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                return;

        /*
         * Calculate the maximum number of seconds we can sleep. Limit
         * to 10 minutes for hardware which can program more than
         * 32bit ticks so we still get reasonable conversion values.
         */
        sec = dev->max_delta_ticks;
        do_div(sec, freq);
        if (!sec)
                sec = 1;
        else if (sec > 600 && dev->max_delta_ticks > UINT_MAX)
                sec = 600;

        clockevents_calc_mult_shift(dev, freq, sec);
        dev->min_delta_ns = cev_delta2ns(dev->min_delta_ticks, dev, false);
        dev->max_delta_ns = cev_delta2ns(dev->max_delta_ticks, dev, true);
}

/**
 * clockevents_config_and_register - Configure and register a clock event device
 * @dev:        device to register
 * @freq:        The clock frequency
 * @min_delta:        The minimum clock ticks to program in oneshot mode
 * @max_delta:        The maximum clock ticks to program in oneshot mode
 *
 * min/max_delta can be 0 for devices which do not support oneshot mode.
 */
void clockevents_config_and_register(struct clock_event_device *dev,
                                     u32 freq, unsigned long min_delta,
                                     unsigned long max_delta)
{
        dev->min_delta_ticks = min_delta;
        dev->max_delta_ticks = max_delta;
        clockevents_config(dev, freq);
        clockevents_register_device(dev);
}
EXPORT_SYMBOL_GPL(clockevents_config_and_register);

int __clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
        clockevents_config(dev, freq);

        if (clockevent_state_oneshot(dev))
                return clockevents_program_event(dev, dev->next_event, false);

        if (clockevent_state_periodic(dev))
                return __clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);

        return 0;
}

/**
 * clockevents_update_freq - Update frequency and reprogram a clock event device.
 * @dev:        device to modify
 * @freq:        new device frequency
 *
 * Reconfigure and reprogram a clock event device in oneshot
 * mode. Must be called on the cpu for which the device delivers per
 * cpu timer events. If called for the broadcast device the core takes
 * care of serialization.
 *
 * Returns 0 on success, -ETIME when the event is in the past.
 */
int clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
        unsigned long flags;
        int ret;

        local_irq_save(flags);
        ret = tick_broadcast_update_freq(dev, freq);
        if (ret == -ENODEV)
                ret = __clockevents_update_freq(dev, freq);
        local_irq_restore(flags);
        return ret;
}

/*
 * Noop handler when we shut down an event device
 */
void clockevents_handle_noop(struct clock_event_device *dev)
{
}

/**
 * clockevents_exchange_device - release and request clock devices
 * @old:        device to release (can be NULL)
 * @new:        device to request (can be NULL)
 *
 * Called from various tick functions with clockevents_lock held and
 * interrupts disabled.
 */
void clockevents_exchange_device(struct clock_event_device *old,
                                 struct clock_event_device *new)
{
        /*
         * Caller releases a clock event device. We queue it into the
         * released list and do a notify add later.
         */
        if (old) {
                module_put(old->owner);
                clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
                list_move(&old->list, &clockevents_released);
        }

        if (new) {
                BUG_ON(!clockevent_state_detached(new));
                clockevents_shutdown(new);
        }
}

/**
 * clockevents_suspend - suspend clock devices
 */
void clockevents_suspend(void)
{
        struct clock_event_device *dev;

        list_for_each_entry_reverse(dev, &clockevent_devices, list)
                if (dev->suspend && !clockevent_state_detached(dev))
                        dev->suspend(dev);
}

/**
 * clockevents_resume - resume clock devices
 */
void clockevents_resume(void)
{
        struct clock_event_device *dev;

        list_for_each_entry(dev, &clockevent_devices, list)
                if (dev->resume && !clockevent_state_detached(dev))
                        dev->resume(dev);
}

#ifdef CONFIG_HOTPLUG_CPU

/**
 * tick_offline_cpu - Shutdown all clock events related
 *                    to this CPU and take it out of the
 *                    broadcast mechanism.
 * @cpu:        The outgoing CPU
 *
 * Called by the dying CPU during teardown.
 */
void tick_offline_cpu(unsigned int cpu)
{
        struct clock_event_device *dev, *tmp;

        raw_spin_lock(&clockevents_lock);

        tick_broadcast_offline(cpu);
        tick_shutdown(cpu);

        /*
         * Unregister the clock event devices which were
         * released above.
         */
        list_for_each_entry_safe(dev, tmp, &clockevents_released, list)
                list_del(&dev->list);

        /*
         * Now check whether the CPU has left unused per cpu devices
         */
        list_for_each_entry_safe(dev, tmp, &clockevent_devices, list) {
                if (cpumask_test_cpu(cpu, dev->cpumask) &&
                    cpumask_weight(dev->cpumask) == 1 &&
                    !tick_is_broadcast_device(dev)) {
                        BUG_ON(!clockevent_state_detached(dev));
                        list_del(&dev->list);
                }
        }

        raw_spin_unlock(&clockevents_lock);
}
#endif

#ifdef CONFIG_SYSFS
static const struct bus_type clockevents_subsys = {
        .name                = "clockevents",
        .dev_name       = "clockevent",
};

static DEFINE_PER_CPU(struct device, tick_percpu_dev);
static struct tick_device *tick_get_tick_dev(struct device *dev);

static ssize_t current_device_show(struct device *dev,
                                   struct device_attribute *attr,
                                   char *buf)
{
        struct tick_device *td;
        ssize_t count = 0;

        raw_spin_lock_irq(&clockevents_lock);
        td = tick_get_tick_dev(dev);
        if (td && td->evtdev)
                count = sysfs_emit(buf, "%s\n", td->evtdev->name);
        raw_spin_unlock_irq(&clockevents_lock);
        return count;
}
static DEVICE_ATTR_RO(current_device);

/* We don't support the abomination of removable broadcast devices */
static ssize_t unbind_device_store(struct device *dev,
                                   struct device_attribute *attr,
                                   const char *buf, size_t count)
{
        char name[CS_NAME_LEN];
        ssize_t ret = sysfs_get_uname(buf, name, count);
        struct clock_event_device *ce = NULL, *iter;

        if (ret < 0)
                return ret;

        ret = -ENODEV;
        mutex_lock(&clockevents_mutex);
        raw_spin_lock_irq(&clockevents_lock);
        list_for_each_entry(iter, &clockevent_devices, list) {
                if (!strcmp(iter->name, name)) {
                        ret = __clockevents_try_unbind(iter, dev->id);
                        ce = iter;
                        break;
                }
        }
        raw_spin_unlock_irq(&clockevents_lock);
        /*
         * We hold clockevents_mutex, so ce can't go away
         */
        if (ret == -EAGAIN)
                ret = clockevents_unbind(ce, dev->id);
        mutex_unlock(&clockevents_mutex);
        return ret ? ret : count;
}
static DEVICE_ATTR_WO(unbind_device);

#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
static struct device tick_bc_dev = {
        .init_name        = "broadcast",
        .id                = 0,
        .bus                = &clockevents_subsys,
};

static struct tick_device *tick_get_tick_dev(struct device *dev)
{
        return dev == &tick_bc_dev ? tick_get_broadcast_device() :
                &per_cpu(tick_cpu_device, dev->id);
}

static __init int tick_broadcast_init_sysfs(void)
{
        int err = device_register(&tick_bc_dev);

        if (!err)
                err = device_create_file(&tick_bc_dev, &dev_attr_current_device);
        return err;
}
#else
static struct tick_device *tick_get_tick_dev(struct device *dev)
{
        return &per_cpu(tick_cpu_device, dev->id);
}
static inline int tick_broadcast_init_sysfs(void) { return 0; }
#endif

static int __init tick_init_sysfs(void)
{
        int cpu;

        for_each_possible_cpu(cpu) {
                struct device *dev = &per_cpu(tick_percpu_dev, cpu);
                int err;

                dev->id = cpu;
                dev->bus = &clockevents_subsys;
                err = device_register(dev);
                if (!err)
                        err = device_create_file(dev, &dev_attr_current_device);
                if (!err)
                        err = device_create_file(dev, &dev_attr_unbind_device);
                if (err)
                        return err;
        }
        return tick_broadcast_init_sysfs();
}

static int __init clockevents_init_sysfs(void)
{
        int err = subsys_system_register(&clockevents_subsys, NULL);

        if (!err)
                err = tick_init_sysfs();
        return err;
}
device_initcall(clockevents_init_sysfs);
#endif /* SYSFS */





























    1 
  143 















    4 









    4 


    1 

    3 





    3 


    3 

    2 

    3 
    1 


    3 

    3 

    4 


























































   43 




   35 





















   35 



    1 
























    8 
    7 






































   10 







    5 




    3 


    5 


















    3 


    3 





    4 




    1 

    2 


    3 







    4 


























    5 

    5 














    5 

















    5 




   10 








    9 


    8 



    6 






    6 
    1 

    5 


    5 



















    3 


    3 
    3 








    3 


    2 


    2 




    2 



    2 



    1 



    1 




    1 



    1 








    1 

   83 












   54 
   54 
   53 




























   81 






   75 






































    3 
    2 

    1 

    1 



























   56 


   10 


    4 




    3 







    1 























  144 
  144 
  144 


   35 
   27 
   35 
   27 




























  145 
  144 

  143 

  142 



    3 





  144 

    3 


    1 














   43 
   42 
   43 
   43 

    1 



    4 














    5 


    4 



    1 










    2 
























   15 




   16 




  143 















   71 

   65 

   61 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
// SPDX-License-Identifier: GPL-2.0
#include <linux/kmod.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/net_tstamp.h>
#include <linux/phylib_stubs.h>
#include <linux/wireless.h>
#include <linux/if_bridge.h>
#include <net/dsa_stubs.h>
#include <net/wext.h>

#include "dev.h"

/*
 *        Map an interface index to its name (SIOCGIFNAME)
 */

/*
 *        We need this ioctl for efficient implementation of the
 *        if_indextoname() function required by the IPv6 API.  Without
 *        it, we would have to search all the interfaces to find a
 *        match.  --pb
 */

static int dev_ifname(struct net *net, struct ifreq *ifr)
{
        ifr->ifr_name[IFNAMSIZ-1] = 0;
        return netdev_get_name(net, ifr->ifr_name, ifr->ifr_ifindex);
}

/*
 *        Perform a SIOCGIFCONF call. This structure will change
 *        size eventually, and there is nothing I can do about it.
 *        Thus we will need a 'compatibility mode'.
 */
int dev_ifconf(struct net *net, struct ifconf __user *uifc)
{
        struct net_device *dev;
        void __user *pos;
        size_t size;
        int len, total = 0, done;

        /* both the ifconf and the ifreq structures are slightly different */
        if (in_compat_syscall()) {
                struct compat_ifconf ifc32;

                if (copy_from_user(&ifc32, uifc, sizeof(struct compat_ifconf)))
                        return -EFAULT;

                pos = compat_ptr(ifc32.ifcbuf);
                len = ifc32.ifc_len;
                size = sizeof(struct compat_ifreq);
        } else {
                struct ifconf ifc;

                if (copy_from_user(&ifc, uifc, sizeof(struct ifconf)))
                        return -EFAULT;

                pos = ifc.ifc_buf;
                len = ifc.ifc_len;
                size = sizeof(struct ifreq);
        }

        /* Loop over the interfaces, and write an info block for each. */
        rtnl_net_lock(net);
        for_each_netdev(net, dev) {
                if (!pos)
                        done = inet_gifconf(dev, NULL, 0, size);
                else
                        done = inet_gifconf(dev, pos + total,
                                            len - total, size);
                if (done < 0) {
                        rtnl_net_unlock(net);
                        return -EFAULT;
                }
                total += done;
        }
        rtnl_net_unlock(net);

        return put_user(total, &uifc->ifc_len);
}

static int dev_getifmap(struct net_device *dev, struct ifreq *ifr)
{
        struct ifmap *ifmap = &ifr->ifr_map;

        if (in_compat_syscall()) {
                struct compat_ifmap *cifmap = (struct compat_ifmap *)ifmap;

                cifmap->mem_start = dev->mem_start;
                cifmap->mem_end   = dev->mem_end;
                cifmap->base_addr = dev->base_addr;
                cifmap->irq       = dev->irq;
                cifmap->dma       = dev->dma;
                cifmap->port      = dev->if_port;

                return 0;
        }

        ifmap->mem_start  = dev->mem_start;
        ifmap->mem_end    = dev->mem_end;
        ifmap->base_addr  = dev->base_addr;
        ifmap->irq        = dev->irq;
        ifmap->dma        = dev->dma;
        ifmap->port       = dev->if_port;

        return 0;
}

static int dev_setifmap(struct net_device *dev, struct ifreq *ifr)
{
        struct compat_ifmap *cifmap = (struct compat_ifmap *)&ifr->ifr_map;

        if (!dev->netdev_ops->ndo_set_config)
                return -EOPNOTSUPP;

        if (in_compat_syscall()) {
                struct ifmap ifmap = {
                        .mem_start  = cifmap->mem_start,
                        .mem_end    = cifmap->mem_end,
                        .base_addr  = cifmap->base_addr,
                        .irq        = cifmap->irq,
                        .dma        = cifmap->dma,
                        .port       = cifmap->port,
                };

                return dev->netdev_ops->ndo_set_config(dev, &ifmap);
        }

        return dev->netdev_ops->ndo_set_config(dev, &ifr->ifr_map);
}

/*
 *        Perform the SIOCxIFxxx calls, inside rcu_read_lock()
 */
static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
{
        int err;
        struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);

        if (!dev)
                return -ENODEV;

        switch (cmd) {
        case SIOCGIFFLAGS:        /* Get interface flags */
                ifr->ifr_flags = (short) dev_get_flags(dev);
                return 0;

        case SIOCGIFMETRIC:        /* Get the metric on the interface
                                   (currently unused) */
                ifr->ifr_metric = 0;
                return 0;

        case SIOCGIFMTU:        /* Get the MTU of a device */
                ifr->ifr_mtu = dev->mtu;
                return 0;

        case SIOCGIFSLAVE:
                err = -EINVAL;
                break;

        case SIOCGIFMAP:
                return dev_getifmap(dev, ifr);

        case SIOCGIFINDEX:
                ifr->ifr_ifindex = dev->ifindex;
                return 0;

        case SIOCGIFTXQLEN:
                ifr->ifr_qlen = dev->tx_queue_len;
                return 0;

        default:
                /* dev_ioctl() should ensure this case
                 * is never reached
                 */
                WARN_ON(1);
                err = -ENOTTY;
                break;

        }
        return err;
}

static int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg)
{
        enum hwtstamp_tx_types tx_type;
        enum hwtstamp_rx_filters rx_filter;
        int tx_type_valid = 0;
        int rx_filter_valid = 0;

        if (cfg->flags & ~HWTSTAMP_FLAG_MASK)
                return -EINVAL;

        tx_type = cfg->tx_type;
        rx_filter = cfg->rx_filter;

        switch (tx_type) {
        case HWTSTAMP_TX_OFF:
        case HWTSTAMP_TX_ON:
        case HWTSTAMP_TX_ONESTEP_SYNC:
        case HWTSTAMP_TX_ONESTEP_P2P:
                tx_type_valid = 1;
                break;
        case __HWTSTAMP_TX_CNT:
                /* not a real value */
                break;
        }

        switch (rx_filter) {
        case HWTSTAMP_FILTER_NONE:
        case HWTSTAMP_FILTER_ALL:
        case HWTSTAMP_FILTER_SOME:
        case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
        case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
        case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
        case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
        case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
        case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
        case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
        case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
        case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
        case HWTSTAMP_FILTER_PTP_V2_EVENT:
        case HWTSTAMP_FILTER_PTP_V2_SYNC:
        case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
        case HWTSTAMP_FILTER_NTP_ALL:
                rx_filter_valid = 1;
                break;
        case __HWTSTAMP_FILTER_CNT:
                /* not a real value */
                break;
        }

        if (!tx_type_valid || !rx_filter_valid)
                return -ERANGE;

        return 0;
}

static int dev_eth_ioctl(struct net_device *dev,
                         struct ifreq *ifr, unsigned int cmd)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (!ops->ndo_eth_ioctl)
                return -EOPNOTSUPP;

        if (!netif_device_present(dev))
                return -ENODEV;

        return ops->ndo_eth_ioctl(dev, ifr, cmd);
}

/**
 * dev_get_hwtstamp_phylib() - Get hardware timestamping settings of NIC
 *        or of attached phylib PHY
 * @dev: Network device
 * @cfg: Timestamping configuration structure
 *
 * Helper for calling the default hardware provider timestamping.
 *
 * Note: phy_mii_ioctl() only handles SIOCSHWTSTAMP (not SIOCGHWTSTAMP), and
 * there only exists a phydev->mii_ts->hwtstamp() method. So this will return
 * -EOPNOTSUPP for phylib for now, which is still more accurate than letting
 * the netdev handle the GET request.
 */
static int dev_get_hwtstamp_phylib(struct net_device *dev,
                                   struct kernel_hwtstamp_config *cfg)
{
        if (phy_is_default_hwtstamp(dev->phydev))
                return phy_hwtstamp_get(dev->phydev, cfg);

        return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg);
}

static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct kernel_hwtstamp_config kernel_cfg = {};
        struct hwtstamp_config cfg;
        int err;

        if (!ops->ndo_hwtstamp_get)
                return dev_eth_ioctl(dev, ifr, SIOCGHWTSTAMP); /* legacy */

        if (!netif_device_present(dev))
                return -ENODEV;

        kernel_cfg.ifr = ifr;
        err = dev_get_hwtstamp_phylib(dev, &kernel_cfg);
        if (err)
                return err;

        /* If the request was resolved through an unconverted driver, omit
         * the copy_to_user(), since the implementation has already done that
         */
        if (!kernel_cfg.copied_to_user) {
                hwtstamp_config_from_kernel(&cfg, &kernel_cfg);

                if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)))
                        return -EFAULT;
        }

        return 0;
}

/**
 * dev_set_hwtstamp_phylib() - Change hardware timestamping of NIC
 *        or of attached phylib PHY
 * @dev: Network device
 * @cfg: Timestamping configuration structure
 * @extack: Netlink extended ack message structure, for error reporting
 *
 * Helper for enforcing a common policy that phylib timestamping, if available,
 * should take precedence in front of hardware timestamping provided by the
 * netdev. If the netdev driver needs to perform specific actions even for PHY
 * timestamping to work properly (a switch port must trap the timestamped
 * frames and not forward them), it must set dev->see_all_hwtstamp_requests.
 */
int dev_set_hwtstamp_phylib(struct net_device *dev,
                            struct kernel_hwtstamp_config *cfg,
                            struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        bool phy_ts = phy_is_default_hwtstamp(dev->phydev);
        struct kernel_hwtstamp_config old_cfg = {};
        bool changed = false;
        int err;

        cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV;

        if (phy_ts && dev->see_all_hwtstamp_requests) {
                err = ops->ndo_hwtstamp_get(dev, &old_cfg);
                if (err)
                        return err;
        }

        if (!phy_ts || dev->see_all_hwtstamp_requests) {
                err = ops->ndo_hwtstamp_set(dev, cfg, extack);
                if (err) {
                        if (extack->_msg)
                                netdev_err(dev, "%s\n", extack->_msg);
                        return err;
                }
        }

        if (phy_ts && dev->see_all_hwtstamp_requests)
                changed = kernel_hwtstamp_config_changed(&old_cfg, cfg);

        if (phy_ts) {
                err = phy_hwtstamp_set(dev->phydev, cfg, extack);
                if (err) {
                        if (changed)
                                ops->ndo_hwtstamp_set(dev, &old_cfg, NULL);
                        return err;
                }
        }

        return 0;
}

static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct kernel_hwtstamp_config kernel_cfg = {};
        struct netlink_ext_ack extack = {};
        struct hwtstamp_config cfg;
        int err;

        if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
                return -EFAULT;

        hwtstamp_config_to_kernel(&kernel_cfg, &cfg);
        kernel_cfg.ifr = ifr;

        err = net_hwtstamp_validate(&kernel_cfg);
        if (err)
                return err;

        err = dsa_conduit_hwtstamp_validate(dev, &kernel_cfg, &extack);
        if (err) {
                if (extack._msg)
                        netdev_err(dev, "%s\n", extack._msg);
                return err;
        }

        if (!ops->ndo_hwtstamp_set)
                return dev_eth_ioctl(dev, ifr, SIOCSHWTSTAMP); /* legacy */

        if (!netif_device_present(dev))
                return -ENODEV;

        err = dev_set_hwtstamp_phylib(dev, &kernel_cfg, &extack);
        if (err)
                return err;

        /* The driver may have modified the configuration, so copy the
         * updated version of it back to user space
         */
        if (!kernel_cfg.copied_to_user) {
                hwtstamp_config_from_kernel(&cfg, &kernel_cfg);

                if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)))
                        return -EFAULT;
        }

        return 0;
}

static int generic_hwtstamp_ioctl_lower(struct net_device *dev, int cmd,
                                        struct kernel_hwtstamp_config *kernel_cfg)
{
        struct ifreq ifrr;
        int err;

        strscpy_pad(ifrr.ifr_name, dev->name, IFNAMSIZ);
        ifrr.ifr_ifru = kernel_cfg->ifr->ifr_ifru;

        err = dev_eth_ioctl(dev, &ifrr, cmd);
        if (err)
                return err;

        kernel_cfg->ifr->ifr_ifru = ifrr.ifr_ifru;
        kernel_cfg->copied_to_user = true;

        return 0;
}

int generic_hwtstamp_get_lower(struct net_device *dev,
                               struct kernel_hwtstamp_config *kernel_cfg)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (!netif_device_present(dev))
                return -ENODEV;

        if (ops->ndo_hwtstamp_get)
                return dev_get_hwtstamp_phylib(dev, kernel_cfg);

        /* Legacy path: unconverted lower driver */
        return generic_hwtstamp_ioctl_lower(dev, SIOCGHWTSTAMP, kernel_cfg);
}
EXPORT_SYMBOL(generic_hwtstamp_get_lower);

int generic_hwtstamp_set_lower(struct net_device *dev,
                               struct kernel_hwtstamp_config *kernel_cfg,
                               struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (!netif_device_present(dev))
                return -ENODEV;

        if (ops->ndo_hwtstamp_set)
                return dev_set_hwtstamp_phylib(dev, kernel_cfg, extack);

        /* Legacy path: unconverted lower driver */
        return generic_hwtstamp_ioctl_lower(dev, SIOCSHWTSTAMP, kernel_cfg);
}
EXPORT_SYMBOL(generic_hwtstamp_set_lower);

static int dev_siocbond(struct net_device *dev,
                        struct ifreq *ifr, unsigned int cmd)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (ops->ndo_siocbond) {
                if (netif_device_present(dev))
                        return ops->ndo_siocbond(dev, ifr, cmd);
                else
                        return -ENODEV;
        }

        return -EOPNOTSUPP;
}

static int dev_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
                              void __user *data, unsigned int cmd)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (ops->ndo_siocdevprivate) {
                if (netif_device_present(dev))
                        return ops->ndo_siocdevprivate(dev, ifr, data, cmd);
                else
                        return -ENODEV;
        }

        return -EOPNOTSUPP;
}

static int dev_siocwandev(struct net_device *dev, struct if_settings *ifs)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (ops->ndo_siocwandev) {
                if (netif_device_present(dev))
                        return ops->ndo_siocwandev(dev, ifs);
                else
                        return -ENODEV;
        }

        return -EOPNOTSUPP;
}

/*
 *        Perform the SIOCxIFxxx calls, inside rtnl_lock()
 */
static int dev_ifsioc(struct net *net, struct ifreq *ifr, void __user *data,
                      unsigned int cmd)
{
        int err;
        struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
        const struct net_device_ops *ops;
        netdevice_tracker dev_tracker;

        if (!dev)
                return -ENODEV;

        ops = dev->netdev_ops;

        switch (cmd) {
        case SIOCSIFFLAGS:        /* Set interface flags */
                return dev_change_flags(dev, ifr->ifr_flags, NULL);

        case SIOCSIFMETRIC:        /* Set the metric on the interface
                                   (currently unused) */
                return -EOPNOTSUPP;

        case SIOCSIFMTU:        /* Set the MTU of a device */
                return dev_set_mtu(dev, ifr->ifr_mtu);

        case SIOCSIFHWADDR:
                if (dev->addr_len > sizeof(struct sockaddr))
                        return -EINVAL;
                return dev_set_mac_address_user(dev, &ifr->ifr_hwaddr, NULL);

        case SIOCSIFHWBROADCAST:
                if (ifr->ifr_hwaddr.sa_family != dev->type)
                        return -EINVAL;
                memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
                       min(sizeof(ifr->ifr_hwaddr.sa_data_min),
                           (size_t)dev->addr_len));
                call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
                return 0;

        case SIOCSIFMAP:
                return dev_setifmap(dev, ifr);

        case SIOCADDMULTI:
                if (!ops->ndo_set_rx_mode ||
                    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
                        return -EINVAL;
                if (!netif_device_present(dev))
                        return -ENODEV;
                return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);

        case SIOCDELMULTI:
                if (!ops->ndo_set_rx_mode ||
                    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
                        return -EINVAL;
                if (!netif_device_present(dev))
                        return -ENODEV;
                return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);

        case SIOCSIFTXQLEN:
                if (ifr->ifr_qlen < 0)
                        return -EINVAL;
                return dev_change_tx_queue_len(dev, ifr->ifr_qlen);

        case SIOCSIFNAME:
                ifr->ifr_newname[IFNAMSIZ-1] = '\0';
                return dev_change_name(dev, ifr->ifr_newname);

        case SIOCWANDEV:
                return dev_siocwandev(dev, &ifr->ifr_settings);

        case SIOCBRADDIF:
        case SIOCBRDELIF:
                if (!netif_device_present(dev))
                        return -ENODEV;
                if (!netif_is_bridge_master(dev))
                        return -EOPNOTSUPP;
                netdev_hold(dev, &dev_tracker, GFP_KERNEL);
                rtnl_unlock();
                err = br_ioctl_call(net, netdev_priv(dev), cmd, ifr, NULL);
                netdev_put(dev, &dev_tracker);
                rtnl_lock();
                return err;

        case SIOCDEVPRIVATE ... SIOCDEVPRIVATE + 15:
                return dev_siocdevprivate(dev, ifr, data, cmd);

        case SIOCSHWTSTAMP:
                return dev_set_hwtstamp(dev, ifr);

        case SIOCGHWTSTAMP:
                return dev_get_hwtstamp(dev, ifr);

        case SIOCGMIIPHY:
        case SIOCGMIIREG:
        case SIOCSMIIREG:
                return dev_eth_ioctl(dev, ifr, cmd);

        case SIOCBONDENSLAVE:
        case SIOCBONDRELEASE:
        case SIOCBONDSETHWADDR:
        case SIOCBONDSLAVEINFOQUERY:
        case SIOCBONDINFOQUERY:
        case SIOCBONDCHANGEACTIVE:
                return dev_siocbond(dev, ifr, cmd);

        /* Unknown ioctl */
        default:
                err = -EINVAL;
        }
        return err;
}

/**
 *        dev_load         - load a network module
 *        @net: the applicable net namespace
 *        @name: name of interface
 *
 *        If a network interface is not present and the process has suitable
 *        privileges this function loads the module. If module loading is not
 *        available in this kernel then it becomes a nop.
 */

void dev_load(struct net *net, const char *name)
{
        struct net_device *dev;
        int no_module;

        rcu_read_lock();
        dev = dev_get_by_name_rcu(net, name);
        rcu_read_unlock();

        no_module = !dev;
        if (no_module && capable(CAP_NET_ADMIN))
                no_module = request_module("netdev-%s", name);
        if (no_module && capable(CAP_SYS_MODULE))
                request_module("%s", name);
}
EXPORT_SYMBOL(dev_load);

/*
 *        This function handles all "interface"-type I/O control requests. The actual
 *        'doing' part of this is dev_ifsioc above.
 */

/**
 *        dev_ioctl        -        network device ioctl
 *        @net: the applicable net namespace
 *        @cmd: command to issue
 *        @ifr: pointer to a struct ifreq in user space
 *        @data: data exchanged with userspace
 *        @need_copyout: whether or not copy_to_user() should be called
 *
 *        Issue ioctl functions to devices. This is normally called by the
 *        user space syscall interfaces but can sometimes be useful for
 *        other purposes. The return value is the return from the syscall if
 *        positive or a negative errno code on error.
 */

int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
              void __user *data, bool *need_copyout)
{
        int ret;
        char *colon;

        if (need_copyout)
                *need_copyout = true;
        if (cmd == SIOCGIFNAME)
                return dev_ifname(net, ifr);

        ifr->ifr_name[IFNAMSIZ-1] = 0;

        colon = strchr(ifr->ifr_name, ':');
        if (colon)
                *colon = 0;

        /*
         *        See which interface the caller is talking about.
         */

        switch (cmd) {
        case SIOCGIFHWADDR:
                dev_load(net, ifr->ifr_name);
                ret = dev_get_mac_address(&ifr->ifr_hwaddr, net, ifr->ifr_name);
                if (colon)
                        *colon = ':';
                return ret;
        /*
         *        These ioctl calls:
         *        - can be done by all.
         *        - atomic and do not require locking.
         *        - return a value
         */
        case SIOCGIFFLAGS:
        case SIOCGIFMETRIC:
        case SIOCGIFMTU:
        case SIOCGIFSLAVE:
        case SIOCGIFMAP:
        case SIOCGIFINDEX:
        case SIOCGIFTXQLEN:
                dev_load(net, ifr->ifr_name);
                rcu_read_lock();
                ret = dev_ifsioc_locked(net, ifr, cmd);
                rcu_read_unlock();
                if (colon)
                        *colon = ':';
                return ret;

        case SIOCETHTOOL:
                dev_load(net, ifr->ifr_name);
                ret = dev_ethtool(net, ifr, data);
                if (colon)
                        *colon = ':';
                return ret;

        /*
         *        These ioctl calls:
         *        - require superuser power.
         *        - require strict serialization.
         *        - return a value
         */
        case SIOCGMIIPHY:
        case SIOCGMIIREG:
        case SIOCSIFNAME:
                dev_load(net, ifr->ifr_name);
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                rtnl_lock();
                ret = dev_ifsioc(net, ifr, data, cmd);
                rtnl_unlock();
                if (colon)
                        *colon = ':';
                return ret;

        /*
         *        These ioctl calls:
         *        - require superuser power.
         *        - require strict serialization.
         *        - do not return a value
         */
        case SIOCSIFMAP:
        case SIOCSIFTXQLEN:
                if (!capable(CAP_NET_ADMIN))
                        return -EPERM;
                fallthrough;
        /*
         *        These ioctl calls:
         *        - require local superuser power.
         *        - require strict serialization.
         *        - do not return a value
         */
        case SIOCSIFFLAGS:
        case SIOCSIFMETRIC:
        case SIOCSIFMTU:
        case SIOCSIFHWADDR:
        case SIOCSIFSLAVE:
        case SIOCADDMULTI:
        case SIOCDELMULTI:
        case SIOCSIFHWBROADCAST:
        case SIOCSMIIREG:
        case SIOCBONDENSLAVE:
        case SIOCBONDRELEASE:
        case SIOCBONDSETHWADDR:
        case SIOCBONDCHANGEACTIVE:
        case SIOCBRADDIF:
        case SIOCBRDELIF:
        case SIOCSHWTSTAMP:
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        return -EPERM;
                fallthrough;
        case SIOCBONDSLAVEINFOQUERY:
        case SIOCBONDINFOQUERY:
                dev_load(net, ifr->ifr_name);
                rtnl_lock();
                ret = dev_ifsioc(net, ifr, data, cmd);
                rtnl_unlock();
                if (need_copyout)
                        *need_copyout = false;
                return ret;

        case SIOCGIFMEM:
                /* Get the per device memory space. We can add this but
                 * currently do not support it */
        case SIOCSIFMEM:
                /* Set the per device memory buffer space.
                 * Not applicable in our case */
        case SIOCSIFLINK:
                return -ENOTTY;

        /*
         *        Unknown or private ioctl.
         */
        default:
                if (cmd == SIOCWANDEV ||
                    cmd == SIOCGHWTSTAMP ||
                    (cmd >= SIOCDEVPRIVATE &&
                     cmd <= SIOCDEVPRIVATE + 15)) {
                        dev_load(net, ifr->ifr_name);
                        rtnl_lock();
                        ret = dev_ifsioc(net, ifr, data, cmd);
                        rtnl_unlock();
                        return ret;
                }
                return -ENOTTY;
        }
}





















































    4 
    4 




    4 














    4 























    4 







    4 























































































































































































































































































































































































































































































































































































































































































    4 



    4 






    4 


    4 


    4 
















    4 
    4 
    4 









    4 



    4 




    4 

    4 





    4 

































































    4 






















    4 




    4 






































































    4 


    3 


































































































































































































































































































































































    4 


































































    4 
















    4 


    3 
    4 


    4 


    4 
    4 





    4 





































    4 


    4 












































    4 
























































































































































    4 






    4 












































    4 





    4 










    4 
    4 


    4 
    4 

    4 
















    4 






    4 
    4 



    4 













    4 







    4 
    4 
    4 















    4 






    4 



































































    4 









    4 































    4 



    4 
    4 

    4 











    4 









    4 














































    4 

    4 



    4 
    4 
    4 
    4 



    4 
    4 
    4 
    4 















































































































   25 






    9 
    9 





















































































    4 






    4 







    4 



    4 

    4 

































































































































    4 
    4 
    4 





    4 







    4 










    4 






































































































    4 







    4 



    4 



    4 


    4 









































    4 










    4 
    4 








    4 


    4 









    4 























    4 
    4 











    4 
















    4 




    4 
























































    4 
    4 



    4 





















































































    4 















    4 

































































































































































































































































    4 
























































    4 







    4 


    4 






    4 








    4 



    4 







    4 
















    4 







    4 


    4 







    4 




























































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 










































    4 

    4 










    4 
    4 
    4 
    4 
    4 
    4 

    4 
    4 















    4 




    4 




    4 











    4 
    4 

    4 

    4 


































    4 



    4 


    4 





    4 



    4 






































































































































    4 









    4 



    4 










    4 

    4 
    4 
    4 

    4 

    4 

    4 


    4 






































































































































































































































































    2 











































































































































    4 























































































































































    4 














    4 


    4 
    4 
    4 


    4 


    4 





















    4 



















    4 





































































































    4 





















    4 




    4 



    4 


    4 




    4 



    4 


    4 






























    4 

















    4 
    4 



    4 


    4 

































































































































































    4 

    4 
    4 
    3 

    4 


    4 


    4 










































    8 

    8 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007-2010        Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
 * Copyright (C) 2018-2024 Intel Corporation
 */

#include <linux/jiffies.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/rcupdate.h>
#include <linux/export.h>
#include <linux/kcov.h>
#include <linux/bitops.h>
#include <kunit/visibility.h>
#include <net/mac80211.h>
#include <net/ieee80211_radiotap.h>
#include <linux/unaligned.h>

#include "ieee80211_i.h"
#include "driver-ops.h"
#include "led.h"
#include "mesh.h"
#include "wep.h"
#include "wpa.h"
#include "tkip.h"
#include "wme.h"
#include "rate.h"

/*
 * monitor mode reception
 *
 * This function cleans up the SKB, i.e. it removes all the stuff
 * only useful for monitoring.
 */
static struct sk_buff *ieee80211_clean_skb(struct sk_buff *skb,
                                           unsigned int present_fcs_len,
                                           unsigned int rtap_space)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_hdr *hdr;
        unsigned int hdrlen;
        __le16 fc;

        if (present_fcs_len)
                __pskb_trim(skb, skb->len - present_fcs_len);
        pskb_pull(skb, rtap_space);

        /* After pulling radiotap header, clear all flags that indicate
         * info in skb->data.
         */
        status->flag &= ~(RX_FLAG_RADIOTAP_TLV_AT_END |
                          RX_FLAG_RADIOTAP_LSIG |
                          RX_FLAG_RADIOTAP_HE_MU |
                          RX_FLAG_RADIOTAP_HE);

        hdr = (void *)skb->data;
        fc = hdr->frame_control;

        /*
         * Remove the HT-Control field (if present) on management
         * frames after we've sent the frame to monitoring. We
         * (currently) don't need it, and don't properly parse
         * frames with it present, due to the assumption of a
         * fixed management header length.
         */
        if (likely(!ieee80211_is_mgmt(fc) || !ieee80211_has_order(fc)))
                return skb;

        hdrlen = ieee80211_hdrlen(fc);
        hdr->frame_control &= ~cpu_to_le16(IEEE80211_FCTL_ORDER);

        if (!pskb_may_pull(skb, hdrlen)) {
                dev_kfree_skb(skb);
                return NULL;
        }

        memmove(skb->data + IEEE80211_HT_CTL_LEN, skb->data,
                hdrlen - IEEE80211_HT_CTL_LEN);
        pskb_pull(skb, IEEE80211_HT_CTL_LEN);

        return skb;
}

static inline bool should_drop_frame(struct sk_buff *skb, int present_fcs_len,
                                     unsigned int rtap_space)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_hdr *hdr;

        hdr = (void *)(skb->data + rtap_space);

        if (status->flag & (RX_FLAG_FAILED_FCS_CRC |
                            RX_FLAG_FAILED_PLCP_CRC |
                            RX_FLAG_ONLY_MONITOR |
                            RX_FLAG_NO_PSDU))
                return true;

        if (unlikely(skb->len < 16 + present_fcs_len + rtap_space))
                return true;

        if (ieee80211_is_ctl(hdr->frame_control) &&
            !ieee80211_is_pspoll(hdr->frame_control) &&
            !ieee80211_is_back_req(hdr->frame_control))
                return true;

        return false;
}

static int
ieee80211_rx_radiotap_hdrlen(struct ieee80211_local *local,
                             struct ieee80211_rx_status *status,
                             struct sk_buff *skb)
{
        int len;

        /* always present fields */
        len = sizeof(struct ieee80211_radiotap_header) + 8;

        /* allocate extra bitmaps */
        if (status->chains)
                len += 4 * hweight8(status->chains);

        if (ieee80211_have_rx_timestamp(status)) {
                len = ALIGN(len, 8);
                len += 8;
        }
        if (ieee80211_hw_check(&local->hw, SIGNAL_DBM))
                len += 1;

        /* antenna field, if we don't have per-chain info */
        if (!status->chains)
                len += 1;

        /* padding for RX_FLAGS if necessary */
        len = ALIGN(len, 2);

        if (status->encoding == RX_ENC_HT) /* HT info */
                len += 3;

        if (status->flag & RX_FLAG_AMPDU_DETAILS) {
                len = ALIGN(len, 4);
                len += 8;
        }

        if (status->encoding == RX_ENC_VHT) {
                len = ALIGN(len, 2);
                len += 12;
        }

        if (local->hw.radiotap_timestamp.units_pos >= 0) {
                len = ALIGN(len, 8);
                len += 12;
        }

        if (status->encoding == RX_ENC_HE &&
            status->flag & RX_FLAG_RADIOTAP_HE) {
                len = ALIGN(len, 2);
                len += 12;
                BUILD_BUG_ON(sizeof(struct ieee80211_radiotap_he) != 12);
        }

        if (status->encoding == RX_ENC_HE &&
            status->flag & RX_FLAG_RADIOTAP_HE_MU) {
                len = ALIGN(len, 2);
                len += 12;
                BUILD_BUG_ON(sizeof(struct ieee80211_radiotap_he_mu) != 12);
        }

        if (status->flag & RX_FLAG_NO_PSDU)
                len += 1;

        if (status->flag & RX_FLAG_RADIOTAP_LSIG) {
                len = ALIGN(len, 2);
                len += 4;
                BUILD_BUG_ON(sizeof(struct ieee80211_radiotap_lsig) != 4);
        }

        if (status->chains) {
                /* antenna and antenna signal fields */
                len += 2 * hweight8(status->chains);
        }

        if (status->flag & RX_FLAG_RADIOTAP_TLV_AT_END) {
                int tlv_offset = 0;

                /*
                 * The position to look at depends on the existence (or non-
                 * existence) of other elements, so take that into account...
                 */
                if (status->flag & RX_FLAG_RADIOTAP_HE)
                        tlv_offset +=
                                sizeof(struct ieee80211_radiotap_he);
                if (status->flag & RX_FLAG_RADIOTAP_HE_MU)
                        tlv_offset +=
                                sizeof(struct ieee80211_radiotap_he_mu);
                if (status->flag & RX_FLAG_RADIOTAP_LSIG)
                        tlv_offset +=
                                sizeof(struct ieee80211_radiotap_lsig);

                /* ensure 4 byte alignment for TLV */
                len = ALIGN(len, 4);

                /* TLVs until the mac header */
                len += skb_mac_header(skb) - &skb->data[tlv_offset];
        }

        return len;
}

static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
                                           int link_id,
                                           struct sta_info *sta,
                                           struct sk_buff *skb)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);

        if (link_id >= 0) {
                status->link_valid = 1;
                status->link_id = link_id;
        } else {
                status->link_valid = 0;
        }

        skb_queue_tail(&sdata->skb_queue, skb);
        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
        if (sta)
                sta->deflink.rx_stats.packets++;
}

static void ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
                                         int link_id,
                                         struct sta_info *sta,
                                         struct sk_buff *skb)
{
        skb->protocol = 0;
        __ieee80211_queue_skb_to_iface(sdata, link_id, sta, skb);
}

static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata,
                                         struct sk_buff *skb,
                                         int rtap_space)
{
        struct {
                struct ieee80211_hdr_3addr hdr;
                u8 category;
                u8 action_code;
        } __packed __aligned(2) action;

        if (!sdata)
                return;

        BUILD_BUG_ON(sizeof(action) != IEEE80211_MIN_ACTION_SIZE + 1);

        if (skb->len < rtap_space + sizeof(action) +
                       VHT_MUMIMO_GROUPS_DATA_LEN)
                return;

        if (!is_valid_ether_addr(sdata->u.mntr.mu_follow_addr))
                return;

        skb_copy_bits(skb, rtap_space, &action, sizeof(action));

        if (!ieee80211_is_action(action.hdr.frame_control))
                return;

        if (action.category != WLAN_CATEGORY_VHT)
                return;

        if (action.action_code != WLAN_VHT_ACTION_GROUPID_MGMT)
                return;

        if (!ether_addr_equal(action.hdr.addr1, sdata->u.mntr.mu_follow_addr))
                return;

        skb = skb_copy(skb, GFP_ATOMIC);
        if (!skb)
                return;

        ieee80211_queue_skb_to_iface(sdata, -1, NULL, skb);
}

/*
 * ieee80211_add_rx_radiotap_header - add radiotap header
 *
 * add a radiotap header containing all the fields which the hardware provided.
 */
static void
ieee80211_add_rx_radiotap_header(struct ieee80211_local *local,
                                 struct sk_buff *skb,
                                 struct ieee80211_rate *rate,
                                 int rtap_len, bool has_fcs)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_radiotap_header *rthdr;
        unsigned char *pos;
        __le32 *it_present;
        u32 it_present_val;
        u16 rx_flags = 0;
        u16 channel_flags = 0;
        u32 tlvs_len = 0;
        int mpdulen, chain;
        unsigned long chains = status->chains;
        struct ieee80211_radiotap_he he = {};
        struct ieee80211_radiotap_he_mu he_mu = {};
        struct ieee80211_radiotap_lsig lsig = {};

        if (status->flag & RX_FLAG_RADIOTAP_HE) {
                he = *(struct ieee80211_radiotap_he *)skb->data;
                skb_pull(skb, sizeof(he));
                WARN_ON_ONCE(status->encoding != RX_ENC_HE);
        }

        if (status->flag & RX_FLAG_RADIOTAP_HE_MU) {
                he_mu = *(struct ieee80211_radiotap_he_mu *)skb->data;
                skb_pull(skb, sizeof(he_mu));
        }

        if (status->flag & RX_FLAG_RADIOTAP_LSIG) {
                lsig = *(struct ieee80211_radiotap_lsig *)skb->data;
                skb_pull(skb, sizeof(lsig));
        }

        if (status->flag & RX_FLAG_RADIOTAP_TLV_AT_END) {
                /* data is pointer at tlv all other info was pulled off */
                tlvs_len = skb_mac_header(skb) - skb->data;
        }

        mpdulen = skb->len;
        if (!(has_fcs && ieee80211_hw_check(&local->hw, RX_INCLUDES_FCS)))
                mpdulen += FCS_LEN;

        rthdr = skb_push(skb, rtap_len - tlvs_len);
        memset(rthdr, 0, rtap_len - tlvs_len);
        it_present = &rthdr->it_present;

        /* radiotap header, set always present flags */
        rthdr->it_len = cpu_to_le16(rtap_len);
        it_present_val = BIT(IEEE80211_RADIOTAP_FLAGS) |
                         BIT(IEEE80211_RADIOTAP_CHANNEL) |
                         BIT(IEEE80211_RADIOTAP_RX_FLAGS);

        if (!status->chains)
                it_present_val |= BIT(IEEE80211_RADIOTAP_ANTENNA);

        for_each_set_bit(chain, &chains, IEEE80211_MAX_CHAINS) {
                it_present_val |=
                        BIT(IEEE80211_RADIOTAP_EXT) |
                        BIT(IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE);
                put_unaligned_le32(it_present_val, it_present);
                it_present++;
                it_present_val = BIT(IEEE80211_RADIOTAP_ANTENNA) |
                                 BIT(IEEE80211_RADIOTAP_DBM_ANTSIGNAL);
        }

        if (status->flag & RX_FLAG_RADIOTAP_TLV_AT_END)
                it_present_val |= BIT(IEEE80211_RADIOTAP_TLV);

        put_unaligned_le32(it_present_val, it_present);

        /* This references through an offset into it_optional[] rather
         * than via it_present otherwise later uses of pos will cause
         * the compiler to think we have walked past the end of the
         * struct member.
         */
        pos = (void *)&rthdr->it_optional[it_present + 1 - rthdr->it_optional];

        /* the order of the following fields is important */

        /* IEEE80211_RADIOTAP_TSFT */
        if (ieee80211_have_rx_timestamp(status)) {
                /* padding */
                while ((pos - (u8 *)rthdr) & 7)
                        *pos++ = 0;
                put_unaligned_le64(
                        ieee80211_calculate_rx_timestamp(local, status,
                                                         mpdulen, 0),
                        pos);
                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_TSFT));
                pos += 8;
        }

        /* IEEE80211_RADIOTAP_FLAGS */
        if (has_fcs && ieee80211_hw_check(&local->hw, RX_INCLUDES_FCS))
                *pos |= IEEE80211_RADIOTAP_F_FCS;
        if (status->flag & (RX_FLAG_FAILED_FCS_CRC | RX_FLAG_FAILED_PLCP_CRC))
                *pos |= IEEE80211_RADIOTAP_F_BADFCS;
        if (status->enc_flags & RX_ENC_FLAG_SHORTPRE)
                *pos |= IEEE80211_RADIOTAP_F_SHORTPRE;
        pos++;

        /* IEEE80211_RADIOTAP_RATE */
        if (!rate || status->encoding != RX_ENC_LEGACY) {
                /*
                 * Without rate information don't add it. If we have,
                 * MCS information is a separate field in radiotap,
                 * added below. The byte here is needed as padding
                 * for the channel though, so initialise it to 0.
                 */
                *pos = 0;
        } else {
                int shift = 0;
                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_RATE));
                if (status->bw == RATE_INFO_BW_10)
                        shift = 1;
                else if (status->bw == RATE_INFO_BW_5)
                        shift = 2;
                *pos = DIV_ROUND_UP(rate->bitrate, 5 * (1 << shift));
        }
        pos++;

        /* IEEE80211_RADIOTAP_CHANNEL */
        /* TODO: frequency offset in KHz */
        put_unaligned_le16(status->freq, pos);
        pos += 2;
        if (status->bw == RATE_INFO_BW_10)
                channel_flags |= IEEE80211_CHAN_HALF;
        else if (status->bw == RATE_INFO_BW_5)
                channel_flags |= IEEE80211_CHAN_QUARTER;

        if (status->band == NL80211_BAND_5GHZ ||
            status->band == NL80211_BAND_6GHZ)
                channel_flags |= IEEE80211_CHAN_OFDM | IEEE80211_CHAN_5GHZ;
        else if (status->encoding != RX_ENC_LEGACY)
                channel_flags |= IEEE80211_CHAN_DYN | IEEE80211_CHAN_2GHZ;
        else if (rate && rate->flags & IEEE80211_RATE_ERP_G)
                channel_flags |= IEEE80211_CHAN_OFDM | IEEE80211_CHAN_2GHZ;
        else if (rate)
                channel_flags |= IEEE80211_CHAN_CCK | IEEE80211_CHAN_2GHZ;
        else
                channel_flags |= IEEE80211_CHAN_2GHZ;
        put_unaligned_le16(channel_flags, pos);
        pos += 2;

        /* IEEE80211_RADIOTAP_DBM_ANTSIGNAL */
        if (ieee80211_hw_check(&local->hw, SIGNAL_DBM) &&
            !(status->flag & RX_FLAG_NO_SIGNAL_VAL)) {
                *pos = status->signal;
                rthdr->it_present |=
                        cpu_to_le32(BIT(IEEE80211_RADIOTAP_DBM_ANTSIGNAL));
                pos++;
        }

        /* IEEE80211_RADIOTAP_LOCK_QUALITY is missing */

        if (!status->chains) {
                /* IEEE80211_RADIOTAP_ANTENNA */
                *pos = status->antenna;
                pos++;
        }

        /* IEEE80211_RADIOTAP_DB_ANTNOISE is not used */

        /* IEEE80211_RADIOTAP_RX_FLAGS */
        /* ensure 2 byte alignment for the 2 byte field as required */
        if ((pos - (u8 *)rthdr) & 1)
                *pos++ = 0;
        if (status->flag & RX_FLAG_FAILED_PLCP_CRC)
                rx_flags |= IEEE80211_RADIOTAP_F_RX_BADPLCP;
        put_unaligned_le16(rx_flags, pos);
        pos += 2;

        if (status->encoding == RX_ENC_HT) {
                unsigned int stbc;

                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_MCS));
                *pos = local->hw.radiotap_mcs_details;
                if (status->enc_flags & RX_ENC_FLAG_HT_GF)
                        *pos |= IEEE80211_RADIOTAP_MCS_HAVE_FMT;
                if (status->enc_flags & RX_ENC_FLAG_LDPC)
                        *pos |= IEEE80211_RADIOTAP_MCS_HAVE_FEC;
                pos++;
                *pos = 0;
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        *pos |= IEEE80211_RADIOTAP_MCS_SGI;
                if (status->bw == RATE_INFO_BW_40)
                        *pos |= IEEE80211_RADIOTAP_MCS_BW_40;
                if (status->enc_flags & RX_ENC_FLAG_HT_GF)
                        *pos |= IEEE80211_RADIOTAP_MCS_FMT_GF;
                if (status->enc_flags & RX_ENC_FLAG_LDPC)
                        *pos |= IEEE80211_RADIOTAP_MCS_FEC_LDPC;
                stbc = (status->enc_flags & RX_ENC_FLAG_STBC_MASK) >> RX_ENC_FLAG_STBC_SHIFT;
                *pos |= stbc << IEEE80211_RADIOTAP_MCS_STBC_SHIFT;
                pos++;
                *pos++ = status->rate_idx;
        }

        if (status->flag & RX_FLAG_AMPDU_DETAILS) {
                u16 flags = 0;

                /* ensure 4 byte alignment */
                while ((pos - (u8 *)rthdr) & 3)
                        pos++;
                rthdr->it_present |=
                        cpu_to_le32(BIT(IEEE80211_RADIOTAP_AMPDU_STATUS));
                put_unaligned_le32(status->ampdu_reference, pos);
                pos += 4;
                if (status->flag & RX_FLAG_AMPDU_LAST_KNOWN)
                        flags |= IEEE80211_RADIOTAP_AMPDU_LAST_KNOWN;
                if (status->flag & RX_FLAG_AMPDU_IS_LAST)
                        flags |= IEEE80211_RADIOTAP_AMPDU_IS_LAST;
                if (status->flag & RX_FLAG_AMPDU_DELIM_CRC_ERROR)
                        flags |= IEEE80211_RADIOTAP_AMPDU_DELIM_CRC_ERR;
                if (status->flag & RX_FLAG_AMPDU_EOF_BIT_KNOWN)
                        flags |= IEEE80211_RADIOTAP_AMPDU_EOF_KNOWN;
                if (status->flag & RX_FLAG_AMPDU_EOF_BIT)
                        flags |= IEEE80211_RADIOTAP_AMPDU_EOF;
                put_unaligned_le16(flags, pos);
                pos += 2;
                *pos++ = 0;
                *pos++ = 0;
        }

        if (status->encoding == RX_ENC_VHT) {
                u16 known = local->hw.radiotap_vht_details;

                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_VHT));
                put_unaligned_le16(known, pos);
                pos += 2;
                /* flags */
                if (status->enc_flags & RX_ENC_FLAG_SHORT_GI)
                        *pos |= IEEE80211_RADIOTAP_VHT_FLAG_SGI;
                /* in VHT, STBC is binary */
                if (status->enc_flags & RX_ENC_FLAG_STBC_MASK)
                        *pos |= IEEE80211_RADIOTAP_VHT_FLAG_STBC;
                if (status->enc_flags & RX_ENC_FLAG_BF)
                        *pos |= IEEE80211_RADIOTAP_VHT_FLAG_BEAMFORMED;
                pos++;
                /* bandwidth */
                switch (status->bw) {
                case RATE_INFO_BW_80:
                        *pos++ = 4;
                        break;
                case RATE_INFO_BW_160:
                        *pos++ = 11;
                        break;
                case RATE_INFO_BW_40:
                        *pos++ = 1;
                        break;
                default:
                        *pos++ = 0;
                }
                /* MCS/NSS */
                *pos = (status->rate_idx << 4) | status->nss;
                pos += 4;
                /* coding field */
                if (status->enc_flags & RX_ENC_FLAG_LDPC)
                        *pos |= IEEE80211_RADIOTAP_CODING_LDPC_USER0;
                pos++;
                /* group ID */
                pos++;
                /* partial_aid */
                pos += 2;
        }

        if (local->hw.radiotap_timestamp.units_pos >= 0) {
                u16 accuracy = 0;
                u8 flags;
                u64 ts;

                rthdr->it_present |=
                        cpu_to_le32(BIT(IEEE80211_RADIOTAP_TIMESTAMP));

                /* ensure 8 byte alignment */
                while ((pos - (u8 *)rthdr) & 7)
                        pos++;

                if (status->flag & RX_FLAG_MACTIME_IS_RTAP_TS64) {
                        flags = IEEE80211_RADIOTAP_TIMESTAMP_FLAG_64BIT;
                        ts = status->mactime;
                } else {
                        flags = IEEE80211_RADIOTAP_TIMESTAMP_FLAG_32BIT;
                        ts = status->device_timestamp;
                }

                put_unaligned_le64(ts, pos);
                pos += sizeof(u64);

                if (local->hw.radiotap_timestamp.accuracy >= 0) {
                        accuracy = local->hw.radiotap_timestamp.accuracy;
                        flags |= IEEE80211_RADIOTAP_TIMESTAMP_FLAG_ACCURACY;
                }
                put_unaligned_le16(accuracy, pos);
                pos += sizeof(u16);

                *pos++ = local->hw.radiotap_timestamp.units_pos;
                *pos++ = flags;
        }

        if (status->encoding == RX_ENC_HE &&
            status->flag & RX_FLAG_RADIOTAP_HE) {
#define HE_PREP(f, val)        le16_encode_bits(val, IEEE80211_RADIOTAP_HE_##f)

                if (status->enc_flags & RX_ENC_FLAG_STBC_MASK) {
                        he.data6 |= HE_PREP(DATA6_NSTS,
                                            FIELD_GET(RX_ENC_FLAG_STBC_MASK,
                                                      status->enc_flags));
                        he.data3 |= HE_PREP(DATA3_STBC, 1);
                } else {
                        he.data6 |= HE_PREP(DATA6_NSTS, status->nss);
                }

#define CHECK_GI(s) \
        BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA5_GI_##s != \
                     (int)NL80211_RATE_INFO_HE_GI_##s)

                CHECK_GI(0_8);
                CHECK_GI(1_6);
                CHECK_GI(3_2);

                he.data3 |= HE_PREP(DATA3_DATA_MCS, status->rate_idx);
                he.data3 |= HE_PREP(DATA3_DATA_DCM, status->he_dcm);
                he.data3 |= HE_PREP(DATA3_CODING,
                                    !!(status->enc_flags & RX_ENC_FLAG_LDPC));

                he.data5 |= HE_PREP(DATA5_GI, status->he_gi);

                switch (status->bw) {
                case RATE_INFO_BW_20:
                        he.data5 |= HE_PREP(DATA5_DATA_BW_RU_ALLOC,
                                            IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_20MHZ);
                        break;
                case RATE_INFO_BW_40:
                        he.data5 |= HE_PREP(DATA5_DATA_BW_RU_ALLOC,
                                            IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_40MHZ);
                        break;
                case RATE_INFO_BW_80:
                        he.data5 |= HE_PREP(DATA5_DATA_BW_RU_ALLOC,
                                            IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_80MHZ);
                        break;
                case RATE_INFO_BW_160:
                        he.data5 |= HE_PREP(DATA5_DATA_BW_RU_ALLOC,
                                            IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_160MHZ);
                        break;
                case RATE_INFO_BW_HE_RU:
#define CHECK_RU_ALLOC(s) \
        BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA5_DATA_BW_RU_ALLOC_##s##T != \
                     NL80211_RATE_INFO_HE_RU_ALLOC_##s + 4)

                        CHECK_RU_ALLOC(26);
                        CHECK_RU_ALLOC(52);
                        CHECK_RU_ALLOC(106);
                        CHECK_RU_ALLOC(242);
                        CHECK_RU_ALLOC(484);
                        CHECK_RU_ALLOC(996);
                        CHECK_RU_ALLOC(2x996);

                        he.data5 |= HE_PREP(DATA5_DATA_BW_RU_ALLOC,
                                            status->he_ru + 4);
                        break;
                default:
                        WARN_ONCE(1, "Invalid SU BW %d\n", status->bw);
                }

                /* ensure 2 byte alignment */
                while ((pos - (u8 *)rthdr) & 1)
                        pos++;
                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_HE));
                memcpy(pos, &he, sizeof(he));
                pos += sizeof(he);
        }

        if (status->encoding == RX_ENC_HE &&
            status->flag & RX_FLAG_RADIOTAP_HE_MU) {
                /* ensure 2 byte alignment */
                while ((pos - (u8 *)rthdr) & 1)
                        pos++;
                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_HE_MU));
                memcpy(pos, &he_mu, sizeof(he_mu));
                pos += sizeof(he_mu);
        }

        if (status->flag & RX_FLAG_NO_PSDU) {
                rthdr->it_present |=
                        cpu_to_le32(BIT(IEEE80211_RADIOTAP_ZERO_LEN_PSDU));
                *pos++ = status->zero_length_psdu_type;
        }

        if (status->flag & RX_FLAG_RADIOTAP_LSIG) {
                /* ensure 2 byte alignment */
                while ((pos - (u8 *)rthdr) & 1)
                        pos++;
                rthdr->it_present |= cpu_to_le32(BIT(IEEE80211_RADIOTAP_LSIG));
                memcpy(pos, &lsig, sizeof(lsig));
                pos += sizeof(lsig);
        }

        for_each_set_bit(chain, &chains, IEEE80211_MAX_CHAINS) {
                *pos++ = status->chain_signal[chain];
                *pos++ = chain;
        }
}

static struct sk_buff *
ieee80211_make_monitor_skb(struct ieee80211_local *local,
                           struct sk_buff **origskb,
                           struct ieee80211_rate *rate,
                           int rtap_space, bool use_origskb)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(*origskb);
        int rt_hdrlen, needed_headroom;
        struct sk_buff *skb;

        /* room for the radiotap header based on driver features */
        rt_hdrlen = ieee80211_rx_radiotap_hdrlen(local, status, *origskb);
        needed_headroom = rt_hdrlen - rtap_space;

        if (use_origskb) {
                /* only need to expand headroom if necessary */
                skb = *origskb;
                *origskb = NULL;

                /*
                 * This shouldn't trigger often because most devices have an
                 * RX header they pull before we get here, and that should
                 * be big enough for our radiotap information. We should
                 * probably export the length to drivers so that we can have
                 * them allocate enough headroom to start with.
                 */
                if (skb_headroom(skb) < needed_headroom &&
                    pskb_expand_head(skb, needed_headroom, 0, GFP_ATOMIC)) {
                        dev_kfree_skb(skb);
                        return NULL;
                }
        } else {
                /*
                 * Need to make a copy and possibly remove radiotap header
                 * and FCS from the original.
                 */
                skb = skb_copy_expand(*origskb, needed_headroom + NET_SKB_PAD,
                                      0, GFP_ATOMIC);

                if (!skb)
                        return NULL;
        }

        /* prepend radiotap information */
        ieee80211_add_rx_radiotap_header(local, skb, rate, rt_hdrlen, true);

        skb_reset_mac_header(skb);
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        skb->pkt_type = PACKET_OTHERHOST;
        skb->protocol = htons(ETH_P_802_2);

        return skb;
}

/*
 * This function copies a received frame to all monitor interfaces and
 * returns a cleaned-up SKB that no longer includes the FCS nor the
 * radiotap header the driver might have added.
 */
static struct sk_buff *
ieee80211_rx_monitor(struct ieee80211_local *local, struct sk_buff *origskb,
                     struct ieee80211_rate *rate)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(origskb);
        struct ieee80211_sub_if_data *sdata, *prev_sdata = NULL;
        struct sk_buff *skb, *monskb = NULL;
        int present_fcs_len = 0;
        unsigned int rtap_space = 0;
        struct ieee80211_sub_if_data *monitor_sdata =
                rcu_dereference(local->monitor_sdata);
        bool only_monitor = false;
        unsigned int min_head_len;

        if (WARN_ON_ONCE(status->flag & RX_FLAG_RADIOTAP_TLV_AT_END &&
                         !skb_mac_header_was_set(origskb))) {
                /* with this skb no way to know where frame payload starts */
                dev_kfree_skb(origskb);
                return NULL;
        }

        if (status->flag & RX_FLAG_RADIOTAP_HE)
                rtap_space += sizeof(struct ieee80211_radiotap_he);

        if (status->flag & RX_FLAG_RADIOTAP_HE_MU)
                rtap_space += sizeof(struct ieee80211_radiotap_he_mu);

        if (status->flag & RX_FLAG_RADIOTAP_LSIG)
                rtap_space += sizeof(struct ieee80211_radiotap_lsig);

        if (status->flag & RX_FLAG_RADIOTAP_TLV_AT_END)
                rtap_space += skb_mac_header(origskb) - &origskb->data[rtap_space];

        min_head_len = rtap_space;

        /*
         * First, we may need to make a copy of the skb because
         *  (1) we need to modify it for radiotap (if not present), and
         *  (2) the other RX handlers will modify the skb we got.
         *
         * We don't need to, of course, if we aren't going to return
         * the SKB because it has a bad FCS/PLCP checksum.
         */

        if (!(status->flag & RX_FLAG_NO_PSDU)) {
                if (ieee80211_hw_check(&local->hw, RX_INCLUDES_FCS)) {
                        if (unlikely(origskb->len <= FCS_LEN + rtap_space)) {
                                /* driver bug */
                                WARN_ON(1);
                                dev_kfree_skb(origskb);
                                return NULL;
                        }
                        present_fcs_len = FCS_LEN;
                }

                /* also consider the hdr->frame_control */
                min_head_len += 2;
        }

        /* ensure that the expected data elements are in skb head */
        if (!pskb_may_pull(origskb, min_head_len)) {
                dev_kfree_skb(origskb);
                return NULL;
        }

        only_monitor = should_drop_frame(origskb, present_fcs_len, rtap_space);

        if (!local->monitors || (status->flag & RX_FLAG_SKIP_MONITOR)) {
                if (only_monitor) {
                        dev_kfree_skb(origskb);
                        return NULL;
                }

                return ieee80211_clean_skb(origskb, present_fcs_len,
                                           rtap_space);
        }

        ieee80211_handle_mu_mimo_mon(monitor_sdata, origskb, rtap_space);

        list_for_each_entry_rcu(sdata, &local->mon_list, u.mntr.list) {
                struct cfg80211_chan_def *chandef;

                chandef = &sdata->vif.bss_conf.chanreq.oper;
                if (chandef->chan &&
                    chandef->chan->center_freq != status->freq)
                        continue;

                if (!prev_sdata) {
                        prev_sdata = sdata;
                        continue;
                }

                if (ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR))
                        ieee80211_handle_mu_mimo_mon(sdata, origskb, rtap_space);

                if (!monskb)
                        monskb = ieee80211_make_monitor_skb(local, &origskb,
                                                            rate, rtap_space,
                                                            false);
                if (!monskb)
                        continue;

                skb = skb_clone(monskb, GFP_ATOMIC);
                if (!skb)
                        continue;

                skb->dev = prev_sdata->dev;
                dev_sw_netstats_rx_add(skb->dev, skb->len);
                netif_receive_skb(skb);
                prev_sdata = sdata;
        }

        if (prev_sdata) {
                if (monskb)
                        skb = monskb;
                else
                        skb = ieee80211_make_monitor_skb(local, &origskb,
                                                         rate, rtap_space,
                                                         only_monitor);
                if (skb) {
                        skb->dev = prev_sdata->dev;
                        dev_sw_netstats_rx_add(skb->dev, skb->len);
                        netif_receive_skb(skb);
                }
        }

        if (!origskb)
                return NULL;

        return ieee80211_clean_skb(origskb, present_fcs_len, rtap_space);
}

static void ieee80211_parse_qos(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);
        int tid, seqno_idx, security_idx;

        /* does the frame have a qos control field? */
        if (ieee80211_is_data_qos(hdr->frame_control)) {
                u8 *qc = ieee80211_get_qos_ctl(hdr);
                /* frame has qos control */
                tid = *qc & IEEE80211_QOS_CTL_TID_MASK;
                if (*qc & IEEE80211_QOS_CTL_A_MSDU_PRESENT)
                        status->rx_flags |= IEEE80211_RX_AMSDU;

                seqno_idx = tid;
                security_idx = tid;
        } else {
                /*
                 * IEEE 802.11-2007, 7.1.3.4.1 ("Sequence Number field"):
                 *
                 *        Sequence numbers for management frames, QoS data
                 *        frames with a broadcast/multicast address in the
                 *        Address 1 field, and all non-QoS data frames sent
                 *        by QoS STAs are assigned using an additional single
                 *        modulo-4096 counter, [...]
                 *
                 * We also use that counter for non-QoS STAs.
                 */
                seqno_idx = IEEE80211_NUM_TIDS;
                security_idx = 0;
                if (ieee80211_is_mgmt(hdr->frame_control))
                        security_idx = IEEE80211_NUM_TIDS;
                tid = 0;
        }

        rx->seqno_idx = seqno_idx;
        rx->security_idx = security_idx;
        /* Set skb->priority to 1d tag if highest order bit of TID is not set.
         * For now, set skb->priority to 0 for other cases. */
        rx->skb->priority = (tid > 7) ? 0 : tid;
}

/**
 * DOC: Packet alignment
 *
 * Drivers always need to pass packets that are aligned to two-byte boundaries
 * to the stack.
 *
 * Additionally, they should, if possible, align the payload data in a way that
 * guarantees that the contained IP header is aligned to a four-byte
 * boundary. In the case of regular frames, this simply means aligning the
 * payload to a four-byte boundary (because either the IP header is directly
 * contained, or IV/RFC1042 headers that have a length divisible by four are
 * in front of it).  If the payload data is not properly aligned and the
 * architecture doesn't support efficient unaligned operations, mac80211
 * will align the data.
 *
 * With A-MSDU frames, however, the payload data address must yield two modulo
 * four because there are 14-byte 802.3 headers within the A-MSDU frames that
 * push the IP header further back to a multiple of four again. Thankfully, the
 * specs were sane enough this time around to require padding each A-MSDU
 * subframe to a length that is a multiple of four.
 *
 * Padding like Atheros hardware adds which is between the 802.11 header and
 * the payload is not supported; the driver is required to move the 802.11
 * header to be directly in front of the payload in that case.
 */
static void ieee80211_verify_alignment(struct ieee80211_rx_data *rx)
{
#ifdef CONFIG_MAC80211_VERBOSE_DEBUG
        WARN_ON_ONCE((unsigned long)rx->skb->data & 1);
#endif
}


/* rx handlers */

static int ieee80211_is_unicast_robust_mgmt_frame(struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;

        if (is_multicast_ether_addr(hdr->addr1))
                return 0;

        return ieee80211_is_robust_mgmt_frame(skb);
}


static int ieee80211_is_multicast_robust_mgmt_frame(struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;

        if (!is_multicast_ether_addr(hdr->addr1))
                return 0;

        return ieee80211_is_robust_mgmt_frame(skb);
}


/* Get the BIP key index from MMIE; return -1 if this is not a BIP frame */
static int ieee80211_get_mmie_keyidx(struct sk_buff *skb)
{
        struct ieee80211_mgmt *hdr = (struct ieee80211_mgmt *) skb->data;
        struct ieee80211_mmie *mmie;
        struct ieee80211_mmie_16 *mmie16;

        if (skb->len < 24 + sizeof(*mmie) || !is_multicast_ether_addr(hdr->da))
                return -1;

        if (!ieee80211_is_robust_mgmt_frame(skb) &&
            !ieee80211_is_beacon(hdr->frame_control))
                return -1; /* not a robust management frame */

        mmie = (struct ieee80211_mmie *)
                (skb->data + skb->len - sizeof(*mmie));
        if (mmie->element_id == WLAN_EID_MMIE &&
            mmie->length == sizeof(*mmie) - 2)
                return le16_to_cpu(mmie->key_id);

        mmie16 = (struct ieee80211_mmie_16 *)
                (skb->data + skb->len - sizeof(*mmie16));
        if (skb->len >= 24 + sizeof(*mmie16) &&
            mmie16->element_id == WLAN_EID_MMIE &&
            mmie16->length == sizeof(*mmie16) - 2)
                return le16_to_cpu(mmie16->key_id);

        return -1;
}

static int ieee80211_get_keyid(struct sk_buff *skb)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
        __le16 fc = hdr->frame_control;
        int hdrlen = ieee80211_hdrlen(fc);
        u8 keyid;

        /* WEP, TKIP, CCMP and GCMP */
        if (unlikely(skb->len < hdrlen + IEEE80211_WEP_IV_LEN))
                return -EINVAL;

        skb_copy_bits(skb, hdrlen + 3, &keyid, 1);

        keyid >>= 6;

        return keyid;
}

static ieee80211_rx_result ieee80211_rx_mesh_check(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        char *dev_addr = rx->sdata->vif.addr;

        if (ieee80211_is_data(hdr->frame_control)) {
                if (is_multicast_ether_addr(hdr->addr1)) {
                        if (ieee80211_has_tods(hdr->frame_control) ||
                            !ieee80211_has_fromds(hdr->frame_control))
                                return RX_DROP_MONITOR;
                        if (ether_addr_equal(hdr->addr3, dev_addr))
                                return RX_DROP_MONITOR;
                } else {
                        if (!ieee80211_has_a4(hdr->frame_control))
                                return RX_DROP_MONITOR;
                        if (ether_addr_equal(hdr->addr4, dev_addr))
                                return RX_DROP_MONITOR;
                }
        }

        /* If there is not an established peer link and this is not a peer link
         * establisment frame, beacon or probe, drop the frame.
         */

        if (!rx->sta || sta_plink_state(rx->sta) != NL80211_PLINK_ESTAB) {
                struct ieee80211_mgmt *mgmt;

                if (!ieee80211_is_mgmt(hdr->frame_control))
                        return RX_DROP_MONITOR;

                if (ieee80211_is_action(hdr->frame_control)) {
                        u8 category;

                        /* make sure category field is present */
                        if (rx->skb->len < IEEE80211_MIN_ACTION_SIZE)
                                return RX_DROP_MONITOR;

                        mgmt = (struct ieee80211_mgmt *)hdr;
                        category = mgmt->u.action.category;
                        if (category != WLAN_CATEGORY_MESH_ACTION &&
                            category != WLAN_CATEGORY_SELF_PROTECTED)
                                return RX_DROP_MONITOR;
                        return RX_CONTINUE;
                }

                if (ieee80211_is_probe_req(hdr->frame_control) ||
                    ieee80211_is_probe_resp(hdr->frame_control) ||
                    ieee80211_is_beacon(hdr->frame_control) ||
                    ieee80211_is_auth(hdr->frame_control))
                        return RX_CONTINUE;

                return RX_DROP_MONITOR;
        }

        return RX_CONTINUE;
}

static inline bool ieee80211_rx_reorder_ready(struct tid_ampdu_rx *tid_agg_rx,
                                              int index)
{
        struct sk_buff_head *frames = &tid_agg_rx->reorder_buf[index];
        struct sk_buff *tail = skb_peek_tail(frames);
        struct ieee80211_rx_status *status;

        if (tid_agg_rx->reorder_buf_filtered &&
            tid_agg_rx->reorder_buf_filtered & BIT_ULL(index))
                return true;

        if (!tail)
                return false;

        status = IEEE80211_SKB_RXCB(tail);
        if (status->flag & RX_FLAG_AMSDU_MORE)
                return false;

        return true;
}

static void ieee80211_release_reorder_frame(struct ieee80211_sub_if_data *sdata,
                                            struct tid_ampdu_rx *tid_agg_rx,
                                            int index,
                                            struct sk_buff_head *frames)
{
        struct sk_buff_head *skb_list = &tid_agg_rx->reorder_buf[index];
        struct sk_buff *skb;
        struct ieee80211_rx_status *status;

        lockdep_assert_held(&tid_agg_rx->reorder_lock);

        if (skb_queue_empty(skb_list))
                goto no_frame;

        if (!ieee80211_rx_reorder_ready(tid_agg_rx, index)) {
                __skb_queue_purge(skb_list);
                goto no_frame;
        }

        /* release frames from the reorder ring buffer */
        tid_agg_rx->stored_mpdu_num--;
        while ((skb = __skb_dequeue(skb_list))) {
                status = IEEE80211_SKB_RXCB(skb);
                status->rx_flags |= IEEE80211_RX_DEFERRED_RELEASE;
                __skb_queue_tail(frames, skb);
        }

no_frame:
        if (tid_agg_rx->reorder_buf_filtered)
                tid_agg_rx->reorder_buf_filtered &= ~BIT_ULL(index);
        tid_agg_rx->head_seq_num = ieee80211_sn_inc(tid_agg_rx->head_seq_num);
}

static void ieee80211_release_reorder_frames(struct ieee80211_sub_if_data *sdata,
                                             struct tid_ampdu_rx *tid_agg_rx,
                                             u16 head_seq_num,
                                             struct sk_buff_head *frames)
{
        int index;

        lockdep_assert_held(&tid_agg_rx->reorder_lock);

        while (ieee80211_sn_less(tid_agg_rx->head_seq_num, head_seq_num)) {
                index = tid_agg_rx->head_seq_num % tid_agg_rx->buf_size;
                ieee80211_release_reorder_frame(sdata, tid_agg_rx, index,
                                                frames);
        }
}

/*
 * Timeout (in jiffies) for skb's that are waiting in the RX reorder buffer. If
 * the skb was added to the buffer longer than this time ago, the earlier
 * frames that have not yet been received are assumed to be lost and the skb
 * can be released for processing. This may also release other skb's from the
 * reorder buffer if there are no additional gaps between the frames.
 *
 * Callers must hold tid_agg_rx->reorder_lock.
 */
#define HT_RX_REORDER_BUF_TIMEOUT (HZ / 10)

static void ieee80211_sta_reorder_release(struct ieee80211_sub_if_data *sdata,
                                          struct tid_ampdu_rx *tid_agg_rx,
                                          struct sk_buff_head *frames)
{
        int index, i, j;

        lockdep_assert_held(&tid_agg_rx->reorder_lock);

        /* release the buffer until next missing frame */
        index = tid_agg_rx->head_seq_num % tid_agg_rx->buf_size;
        if (!ieee80211_rx_reorder_ready(tid_agg_rx, index) &&
            tid_agg_rx->stored_mpdu_num) {
                /*
                 * No buffers ready to be released, but check whether any
                 * frames in the reorder buffer have timed out.
                 */
                int skipped = 1;
                for (j = (index + 1) % tid_agg_rx->buf_size; j != index;
                     j = (j + 1) % tid_agg_rx->buf_size) {
                        if (!ieee80211_rx_reorder_ready(tid_agg_rx, j)) {
                                skipped++;
                                continue;
                        }
                        if (skipped &&
                            !time_after(jiffies, tid_agg_rx->reorder_time[j] +
                                        HT_RX_REORDER_BUF_TIMEOUT))
                                goto set_release_timer;

                        /* don't leave incomplete A-MSDUs around */
                        for (i = (index + 1) % tid_agg_rx->buf_size; i != j;
                             i = (i + 1) % tid_agg_rx->buf_size)
                                __skb_queue_purge(&tid_agg_rx->reorder_buf[i]);

                        ht_dbg_ratelimited(sdata,
                                           "release an RX reorder frame due to timeout on earlier frames\n");
                        ieee80211_release_reorder_frame(sdata, tid_agg_rx, j,
                                                        frames);

                        /*
                         * Increment the head seq# also for the skipped slots.
                         */
                        tid_agg_rx->head_seq_num =
                                (tid_agg_rx->head_seq_num +
                                 skipped) & IEEE80211_SN_MASK;
                        skipped = 0;
                }
        } else while (ieee80211_rx_reorder_ready(tid_agg_rx, index)) {
                ieee80211_release_reorder_frame(sdata, tid_agg_rx, index,
                                                frames);
                index =        tid_agg_rx->head_seq_num % tid_agg_rx->buf_size;
        }

        if (tid_agg_rx->stored_mpdu_num) {
                j = index = tid_agg_rx->head_seq_num % tid_agg_rx->buf_size;

                for (; j != (index - 1) % tid_agg_rx->buf_size;
                     j = (j + 1) % tid_agg_rx->buf_size) {
                        if (ieee80211_rx_reorder_ready(tid_agg_rx, j))
                                break;
                }

 set_release_timer:

                if (!tid_agg_rx->removed)
                        mod_timer(&tid_agg_rx->reorder_timer,
                                  tid_agg_rx->reorder_time[j] + 1 +
                                  HT_RX_REORDER_BUF_TIMEOUT);
        } else {
                del_timer(&tid_agg_rx->reorder_timer);
        }
}

/*
 * As this function belongs to the RX path it must be under
 * rcu_read_lock protection. It returns false if the frame
 * can be processed immediately, true if it was consumed.
 */
static bool ieee80211_sta_manage_reorder_buf(struct ieee80211_sub_if_data *sdata,
                                             struct tid_ampdu_rx *tid_agg_rx,
                                             struct sk_buff *skb,
                                             struct sk_buff_head *frames)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        u16 mpdu_seq_num = ieee80211_get_sn(hdr);
        u16 head_seq_num, buf_size;
        int index;
        bool ret = true;

        spin_lock(&tid_agg_rx->reorder_lock);

        /*
         * Offloaded BA sessions have no known starting sequence number so pick
         * one from first Rxed frame for this tid after BA was started.
         */
        if (unlikely(tid_agg_rx->auto_seq)) {
                tid_agg_rx->auto_seq = false;
                tid_agg_rx->ssn = mpdu_seq_num;
                tid_agg_rx->head_seq_num = mpdu_seq_num;
        }

        buf_size = tid_agg_rx->buf_size;
        head_seq_num = tid_agg_rx->head_seq_num;

        /*
         * If the current MPDU's SN is smaller than the SSN, it shouldn't
         * be reordered.
         */
        if (unlikely(!tid_agg_rx->started)) {
                if (ieee80211_sn_less(mpdu_seq_num, head_seq_num)) {
                        ret = false;
                        goto out;
                }
                tid_agg_rx->started = true;
        }

        /* frame with out of date sequence number */
        if (ieee80211_sn_less(mpdu_seq_num, head_seq_num)) {
                dev_kfree_skb(skb);
                goto out;
        }

        /*
         * If frame the sequence number exceeds our buffering window
         * size release some previous frames to make room for this one.
         */
        if (!ieee80211_sn_less(mpdu_seq_num, head_seq_num + buf_size)) {
                head_seq_num = ieee80211_sn_inc(
                                ieee80211_sn_sub(mpdu_seq_num, buf_size));
                /* release stored frames up to new head to stack */
                ieee80211_release_reorder_frames(sdata, tid_agg_rx,
                                                 head_seq_num, frames);
        }

        /* Now the new frame is always in the range of the reordering buffer */

        index = mpdu_seq_num % tid_agg_rx->buf_size;

        /* check if we already stored this frame */
        if (ieee80211_rx_reorder_ready(tid_agg_rx, index)) {
                dev_kfree_skb(skb);
                goto out;
        }

        /*
         * If the current MPDU is in the right order and nothing else
         * is stored we can process it directly, no need to buffer it.
         * If it is first but there's something stored, we may be able
         * to release frames after this one.
         */
        if (mpdu_seq_num == tid_agg_rx->head_seq_num &&
            tid_agg_rx->stored_mpdu_num == 0) {
                if (!(status->flag & RX_FLAG_AMSDU_MORE))
                        tid_agg_rx->head_seq_num =
                                ieee80211_sn_inc(tid_agg_rx->head_seq_num);
                ret = false;
                goto out;
        }

        /* put the frame in the reordering buffer */
        __skb_queue_tail(&tid_agg_rx->reorder_buf[index], skb);
        if (!(status->flag & RX_FLAG_AMSDU_MORE)) {
                tid_agg_rx->reorder_time[index] = jiffies;
                tid_agg_rx->stored_mpdu_num++;
                ieee80211_sta_reorder_release(sdata, tid_agg_rx, frames);
        }

 out:
        spin_unlock(&tid_agg_rx->reorder_lock);
        return ret;
}

/*
 * Reorder MPDUs from A-MPDUs, keeping them on a buffer. Returns
 * true if the MPDU was buffered, false if it should be processed.
 */
static void ieee80211_rx_reorder_ampdu(struct ieee80211_rx_data *rx,
                                       struct sk_buff_head *frames)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
        struct sta_info *sta = rx->sta;
        struct tid_ampdu_rx *tid_agg_rx;
        u16 sc;
        u8 tid, ack_policy;

        if (!ieee80211_is_data_qos(hdr->frame_control) ||
            is_multicast_ether_addr(hdr->addr1))
                goto dont_reorder;

        /*
         * filter the QoS data rx stream according to
         * STA/TID and check if this STA/TID is on aggregation
         */

        if (!sta)
                goto dont_reorder;

        ack_policy = *ieee80211_get_qos_ctl(hdr) &
                     IEEE80211_QOS_CTL_ACK_POLICY_MASK;
        tid = ieee80211_get_tid(hdr);

        tid_agg_rx = rcu_dereference(sta->ampdu_mlme.tid_rx[tid]);
        if (!tid_agg_rx) {
                if (ack_policy == IEEE80211_QOS_CTL_ACK_POLICY_BLOCKACK &&
                    !test_bit(tid, rx->sta->ampdu_mlme.agg_session_valid) &&
                    !test_and_set_bit(tid, rx->sta->ampdu_mlme.unexpected_agg))
                        ieee80211_send_delba(rx->sdata, rx->sta->sta.addr, tid,
                                             WLAN_BACK_RECIPIENT,
                                             WLAN_REASON_QSTA_REQUIRE_SETUP);
                goto dont_reorder;
        }

        /* qos null data frames are excluded */
        if (unlikely(hdr->frame_control & cpu_to_le16(IEEE80211_STYPE_NULLFUNC)))
                goto dont_reorder;

        /* not part of a BA session */
        if (ack_policy == IEEE80211_QOS_CTL_ACK_POLICY_NOACK)
                goto dont_reorder;

        /* new, potentially un-ordered, ampdu frame - process it */

        /* reset session timer */
        if (tid_agg_rx->timeout)
                tid_agg_rx->last_rx = jiffies;

        /* if this mpdu is fragmented - terminate rx aggregation session */
        sc = le16_to_cpu(hdr->seq_ctrl);
        if (sc & IEEE80211_SCTL_FRAG) {
                ieee80211_queue_skb_to_iface(rx->sdata, rx->link_id, NULL, skb);
                return;
        }

        /*
         * No locking needed -- we will only ever process one
         * RX packet at a time, and thus own tid_agg_rx. All
         * other code manipulating it needs to (and does) make
         * sure that we cannot get to it any more before doing
         * anything with it.
         */
        if (ieee80211_sta_manage_reorder_buf(rx->sdata, tid_agg_rx, skb,
                                             frames))
                return;

 dont_reorder:
        __skb_queue_tail(frames, skb);
}

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);

        if (status->flag & RX_FLAG_DUP_VALIDATED)
                return RX_CONTINUE;

        /*
         * Drop duplicate 802.11 retransmissions
         * (IEEE 802.11-2012: 9.3.2.10 "Duplicate detection and recovery")
         */

        if (rx->skb->len < 24)
                return RX_CONTINUE;

        if (ieee80211_is_ctl(hdr->frame_control) ||
            ieee80211_is_any_nullfunc(hdr->frame_control))
                return RX_CONTINUE;

        if (!rx->sta)
                return RX_CONTINUE;

        if (unlikely(is_multicast_ether_addr(hdr->addr1))) {
                struct ieee80211_sub_if_data *sdata = rx->sdata;
                u16 sn = ieee80211_get_sn(hdr);

                if (!ieee80211_is_data_present(hdr->frame_control))
                        return RX_CONTINUE;

                if (!ieee80211_vif_is_mld(&sdata->vif) ||
                    sdata->vif.type != NL80211_IFTYPE_STATION)
                        return RX_CONTINUE;

                if (sdata->u.mgd.mcast_seq_last != IEEE80211_SN_MODULO &&
                    ieee80211_sn_less_eq(sn, sdata->u.mgd.mcast_seq_last))
                        return RX_DROP_U_DUP;

                sdata->u.mgd.mcast_seq_last = sn;
                return RX_CONTINUE;
        }

        if (unlikely(ieee80211_has_retry(hdr->frame_control) &&
                     rx->sta->last_seq_ctrl[rx->seqno_idx] == hdr->seq_ctrl)) {
                I802_DEBUG_INC(rx->local->dot11FrameDuplicateCount);
                rx->link_sta->rx_stats.num_duplicates++;
                return RX_DROP_U_DUP;
        } else if (!(status->flag & RX_FLAG_AMSDU_MORE)) {
                rx->sta->last_seq_ctrl[rx->seqno_idx] = hdr->seq_ctrl;
        }

        return RX_CONTINUE;
}

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_check(struct ieee80211_rx_data *rx)
{
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;

        /* Drop disallowed frame classes based on STA auth/assoc state;
         * IEEE 802.11, Chap 5.5.
         *
         * mac80211 filters only based on association state, i.e. it drops
         * Class 3 frames from not associated stations. hostapd sends
         * deauth/disassoc frames when needed. In addition, hostapd is
         * responsible for filtering on both auth and assoc states.
         */

        if (ieee80211_vif_is_mesh(&rx->sdata->vif))
                return ieee80211_rx_mesh_check(rx);

        if (unlikely((ieee80211_is_data(hdr->frame_control) ||
                      ieee80211_is_pspoll(hdr->frame_control)) &&
                     rx->sdata->vif.type != NL80211_IFTYPE_ADHOC &&
                     rx->sdata->vif.type != NL80211_IFTYPE_OCB &&
                     (!rx->sta || !test_sta_flag(rx->sta, WLAN_STA_ASSOC)))) {
                /*
                 * accept port control frames from the AP even when it's not
                 * yet marked ASSOC to prevent a race where we don't set the
                 * assoc bit quickly enough before it sends the first frame
                 */
                if (rx->sta && rx->sdata->vif.type == NL80211_IFTYPE_STATION &&
                    ieee80211_is_data_present(hdr->frame_control)) {
                        unsigned int hdrlen;
                        __be16 ethertype;

                        hdrlen = ieee80211_hdrlen(hdr->frame_control);

                        if (rx->skb->len < hdrlen + 8)
                                return RX_DROP_MONITOR;

                        skb_copy_bits(rx->skb, hdrlen + 6, &ethertype, 2);
                        if (ethertype == rx->sdata->control_port_protocol)
                                return RX_CONTINUE;
                }

                if (rx->sdata->vif.type == NL80211_IFTYPE_AP &&
                    cfg80211_rx_spurious_frame(rx->sdata->dev,
                                               hdr->addr2,
                                               GFP_ATOMIC))
                        return RX_DROP_U_SPURIOUS;

                return RX_DROP_MONITOR;
        }

        return RX_CONTINUE;
}


static ieee80211_rx_result debug_noinline
ieee80211_rx_h_check_more_data(struct ieee80211_rx_data *rx)
{
        struct ieee80211_local *local;
        struct ieee80211_hdr *hdr;
        struct sk_buff *skb;

        local = rx->local;
        skb = rx->skb;
        hdr = (struct ieee80211_hdr *) skb->data;

        if (!local->pspolling)
                return RX_CONTINUE;

        if (!ieee80211_has_fromds(hdr->frame_control))
                /* this is not from AP */
                return RX_CONTINUE;

        if (!ieee80211_is_data(hdr->frame_control))
                return RX_CONTINUE;

        if (!ieee80211_has_moredata(hdr->frame_control)) {
                /* AP has no more frames buffered for us */
                local->pspolling = false;
                return RX_CONTINUE;
        }

        /* more data bit is set, let's request a new frame from the AP */
        ieee80211_send_pspoll(local, rx->sdata);

        return RX_CONTINUE;
}

static void sta_ps_start(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ps_data *ps;
        int tid;

        if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
            sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                ps = &sdata->bss->ps;
        else
                return;

        atomic_inc(&ps->num_sta_ps);
        set_sta_flag(sta, WLAN_STA_PS_STA);
        if (!ieee80211_hw_check(&local->hw, AP_LINK_PS))
                drv_sta_notify(local, sdata, STA_NOTIFY_SLEEP, &sta->sta);
        ps_dbg(sdata, "STA %pM aid %d enters power save mode\n",
               sta->sta.addr, sta->sta.aid);

        ieee80211_clear_fast_xmit(sta);

        for (tid = 0; tid < IEEE80211_NUM_TIDS; tid++) {
                struct ieee80211_txq *txq = sta->sta.txq[tid];
                struct txq_info *txqi = to_txq_info(txq);

                spin_lock(&local->active_txq_lock[txq->ac]);
                if (!list_empty(&txqi->schedule_order))
                        list_del_init(&txqi->schedule_order);
                spin_unlock(&local->active_txq_lock[txq->ac]);

                if (txq_has_queue(txq))
                        set_bit(tid, &sta->txq_buffered_tids);
                else
                        clear_bit(tid, &sta->txq_buffered_tids);
        }
}

static void sta_ps_end(struct sta_info *sta)
{
        ps_dbg(sta->sdata, "STA %pM aid %d exits power save mode\n",
               sta->sta.addr, sta->sta.aid);

        if (test_sta_flag(sta, WLAN_STA_PS_DRIVER)) {
                /*
                 * Clear the flag only if the other one is still set
                 * so that the TX path won't start TX'ing new frames
                 * directly ... In the case that the driver flag isn't
                 * set ieee80211_sta_ps_deliver_wakeup() will clear it.
                 */
                clear_sta_flag(sta, WLAN_STA_PS_STA);
                ps_dbg(sta->sdata, "STA %pM aid %d driver-ps-blocked\n",
                       sta->sta.addr, sta->sta.aid);
                return;
        }

        set_sta_flag(sta, WLAN_STA_PS_DELIVER);
        clear_sta_flag(sta, WLAN_STA_PS_STA);
        ieee80211_sta_ps_deliver_wakeup(sta);
}

int ieee80211_sta_ps_transition(struct ieee80211_sta *pubsta, bool start)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        bool in_ps;

        WARN_ON(!ieee80211_hw_check(&sta->local->hw, AP_LINK_PS));

        /* Don't let the same PS state be set twice */
        in_ps = test_sta_flag(sta, WLAN_STA_PS_STA);
        if ((start && in_ps) || (!start && !in_ps))
                return -EINVAL;

        if (start)
                sta_ps_start(sta);
        else
                sta_ps_end(sta);

        return 0;
}
EXPORT_SYMBOL(ieee80211_sta_ps_transition);

void ieee80211_sta_pspoll(struct ieee80211_sta *pubsta)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);

        if (test_sta_flag(sta, WLAN_STA_SP))
                return;

        if (!test_sta_flag(sta, WLAN_STA_PS_DRIVER))
                ieee80211_sta_ps_deliver_poll_response(sta);
        else
                set_sta_flag(sta, WLAN_STA_PSPOLL);
}
EXPORT_SYMBOL(ieee80211_sta_pspoll);

void ieee80211_sta_uapsd_trigger(struct ieee80211_sta *pubsta, u8 tid)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        int ac = ieee80211_ac_from_tid(tid);

        /*
         * If this AC is not trigger-enabled do nothing unless the
         * driver is calling us after it already checked.
         *
         * NB: This could/should check a separate bitmap of trigger-
         * enabled queues, but for now we only implement uAPSD w/o
         * TSPEC changes to the ACs, so they're always the same.
         */
        if (!(sta->sta.uapsd_queues & ieee80211_ac_to_qos_mask[ac]) &&
            tid != IEEE80211_NUM_TIDS)
                return;

        /* if we are in a service period, do nothing */
        if (test_sta_flag(sta, WLAN_STA_SP))
                return;

        if (!test_sta_flag(sta, WLAN_STA_PS_DRIVER))
                ieee80211_sta_ps_deliver_uapsd(sta);
        else
                set_sta_flag(sta, WLAN_STA_UAPSD);
}
EXPORT_SYMBOL(ieee80211_sta_uapsd_trigger);

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_uapsd_and_pspoll(struct ieee80211_rx_data *rx)
{
        struct ieee80211_sub_if_data *sdata = rx->sdata;
        struct ieee80211_hdr *hdr = (void *)rx->skb->data;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);

        if (!rx->sta)
                return RX_CONTINUE;

        if (sdata->vif.type != NL80211_IFTYPE_AP &&
            sdata->vif.type != NL80211_IFTYPE_AP_VLAN)
                return RX_CONTINUE;

        /*
         * The device handles station powersave, so don't do anything about
         * uAPSD and PS-Poll frames (the latter shouldn't even come up from
         * it to mac80211 since they're handled.)
         */
        if (ieee80211_hw_check(&sdata->local->hw, AP_LINK_PS))
                return RX_CONTINUE;

        /*
         * Don't do anything if the station isn't already asleep. In
         * the uAPSD case, the station will probably be marked asleep,
         * in the PS-Poll case the station must be confused ...
         */
        if (!test_sta_flag(rx->sta, WLAN_STA_PS_STA))
                return RX_CONTINUE;

        if (unlikely(ieee80211_is_pspoll(hdr->frame_control))) {
                ieee80211_sta_pspoll(&rx->sta->sta);

                /* Free PS Poll skb here instead of returning RX_DROP that would
                 * count as an dropped frame. */
                dev_kfree_skb(rx->skb);

                return RX_QUEUED;
        } else if (!ieee80211_has_morefrags(hdr->frame_control) &&
                   !(status->rx_flags & IEEE80211_RX_DEFERRED_RELEASE) &&
                   ieee80211_has_pm(hdr->frame_control) &&
                   (ieee80211_is_data_qos(hdr->frame_control) ||
                    ieee80211_is_qos_nullfunc(hdr->frame_control))) {
                u8 tid = ieee80211_get_tid(hdr);

                ieee80211_sta_uapsd_trigger(&rx->sta->sta, tid);
        }

        return RX_CONTINUE;
}

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx)
{
        struct sta_info *sta = rx->sta;
        struct link_sta_info *link_sta = rx->link_sta;
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
        int i;

        if (!sta || !link_sta)
                return RX_CONTINUE;

        /*
         * Update last_rx only for IBSS packets which are for the current
         * BSSID and for station already AUTHORIZED to avoid keeping the
         * current IBSS network alive in cases where other STAs start
         * using different BSSID. This will also give the station another
         * chance to restart the authentication/authorization in case
         * something went wrong the first time.
         */
        if (rx->sdata->vif.type == NL80211_IFTYPE_ADHOC) {
                u8 *bssid = ieee80211_get_bssid(hdr, rx->skb->len,
                                                NL80211_IFTYPE_ADHOC);
                if (ether_addr_equal(bssid, rx->sdata->u.ibss.bssid) &&
                    test_sta_flag(sta, WLAN_STA_AUTHORIZED)) {
                        link_sta->rx_stats.last_rx = jiffies;
                        if (ieee80211_is_data_present(hdr->frame_control) &&
                            !is_multicast_ether_addr(hdr->addr1))
                                link_sta->rx_stats.last_rate =
                                        sta_stats_encode_rate(status);
                }
        } else if (rx->sdata->vif.type == NL80211_IFTYPE_OCB) {
                link_sta->rx_stats.last_rx = jiffies;
        } else if (!ieee80211_is_s1g_beacon(hdr->frame_control) &&
                   !is_multicast_ether_addr(hdr->addr1)) {
                /*
                 * Mesh beacons will update last_rx when if they are found to
                 * match the current local configuration when processed.
                 */
                link_sta->rx_stats.last_rx = jiffies;
                if (ieee80211_is_data_present(hdr->frame_control))
                        link_sta->rx_stats.last_rate = sta_stats_encode_rate(status);
        }

        link_sta->rx_stats.fragments++;

        u64_stats_update_begin(&link_sta->rx_stats.syncp);
        link_sta->rx_stats.bytes += rx->skb->len;
        u64_stats_update_end(&link_sta->rx_stats.syncp);

        if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) {
                link_sta->rx_stats.last_signal = status->signal;
                ewma_signal_add(&link_sta->rx_stats_avg.signal,
                                -status->signal);
        }

        if (status->chains) {
                link_sta->rx_stats.chains = status->chains;
                for (i = 0; i < ARRAY_SIZE(status->chain_signal); i++) {
                        int signal = status->chain_signal[i];

                        if (!(status->chains & BIT(i)))
                                continue;

                        link_sta->rx_stats.chain_signal_last[i] = signal;
                        ewma_signal_add(&link_sta->rx_stats_avg.chain_signal[i],
                                        -signal);
                }
        }

        if (ieee80211_is_s1g_beacon(hdr->frame_control))
                return RX_CONTINUE;

        /*
         * Change STA power saving mode only at the end of a frame
         * exchange sequence, and only for a data or management
         * frame as specified in IEEE 802.11-2016 11.2.3.2
         */
        if (!ieee80211_hw_check(&sta->local->hw, AP_LINK_PS) &&
            !ieee80211_has_morefrags(hdr->frame_control) &&
            !is_multicast_ether_addr(hdr->addr1) &&
            (ieee80211_is_mgmt(hdr->frame_control) ||
             ieee80211_is_data(hdr->frame_control)) &&
            !(status->rx_flags & IEEE80211_RX_DEFERRED_RELEASE) &&
            (rx->sdata->vif.type == NL80211_IFTYPE_AP ||
             rx->sdata->vif.type == NL80211_IFTYPE_AP_VLAN)) {
                if (test_sta_flag(sta, WLAN_STA_PS_STA)) {
                        if (!ieee80211_has_pm(hdr->frame_control))
                                sta_ps_end(sta);
                } else {
                        if (ieee80211_has_pm(hdr->frame_control))
                                sta_ps_start(sta);
                }
        }

        /* mesh power save support */
        if (ieee80211_vif_is_mesh(&rx->sdata->vif))
                ieee80211_mps_rx_h_sta_process(sta, hdr);

        /*
         * Drop (qos-)data::nullfunc frames silently, since they
         * are used only to control station power saving mode.
         */
        if (ieee80211_is_any_nullfunc(hdr->frame_control)) {
                I802_DEBUG_INC(rx->local->rx_handlers_drop_nullfunc);

                /*
                 * If we receive a 4-addr nullfunc frame from a STA
                 * that was not moved to a 4-addr STA vlan yet send
                 * the event to userspace and for older hostapd drop
                 * the frame to the monitor interface.
                 */
                if (ieee80211_has_a4(hdr->frame_control) &&
                    (rx->sdata->vif.type == NL80211_IFTYPE_AP ||
                     (rx->sdata->vif.type == NL80211_IFTYPE_AP_VLAN &&
                      !rx->sdata->u.vlan.sta))) {
                        if (!test_and_set_sta_flag(sta, WLAN_STA_4ADDR_EVENT))
                                cfg80211_rx_unexpected_4addr_frame(
                                        rx->sdata->dev, sta->sta.addr,
                                        GFP_ATOMIC);
                        return RX_DROP_M_UNEXPECTED_4ADDR_FRAME;
                }
                /*
                 * Update counter and free packet here to avoid
                 * counting this as a dropped packed.
                 */
                link_sta->rx_stats.packets++;
                dev_kfree_skb(rx->skb);
                return RX_QUEUED;
        }

        return RX_CONTINUE;
} /* ieee80211_rx_h_sta_process */

static struct ieee80211_key *
ieee80211_rx_get_bigtk(struct ieee80211_rx_data *rx, int idx)
{
        struct ieee80211_key *key = NULL;
        int idx2;

        /* Make sure key gets set if either BIGTK key index is set so that
         * ieee80211_drop_unencrypted_mgmt() can properly drop both unprotected
         * Beacon frames and Beacon frames that claim to use another BIGTK key
         * index (i.e., a key that we do not have).
         */

        if (idx < 0) {
                idx = NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS;
                idx2 = idx + 1;
        } else {
                if (idx == NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS)
                        idx2 = idx + 1;
                else
                        idx2 = idx - 1;
        }

        if (rx->link_sta)
                key = rcu_dereference(rx->link_sta->gtk[idx]);
        if (!key)
                key = rcu_dereference(rx->link->gtk[idx]);
        if (!key && rx->link_sta)
                key = rcu_dereference(rx->link_sta->gtk[idx2]);
        if (!key)
                key = rcu_dereference(rx->link->gtk[idx2]);

        return key;
}

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
        int keyidx;
        ieee80211_rx_result result = RX_DROP_U_DECRYPT_FAIL;
        struct ieee80211_key *sta_ptk = NULL;
        struct ieee80211_key *ptk_idx = NULL;
        int mmie_keyidx = -1;
        __le16 fc;

        if (ieee80211_is_ext(hdr->frame_control))
                return RX_CONTINUE;

        /*
         * Key selection 101
         *
         * There are five types of keys:
         *  - GTK (group keys)
         *  - IGTK (group keys for management frames)
         *  - BIGTK (group keys for Beacon frames)
         *  - PTK (pairwise keys)
         *  - STK (station-to-station pairwise keys)
         *
         * When selecting a key, we have to distinguish between multicast
         * (including broadcast) and unicast frames, the latter can only
         * use PTKs and STKs while the former always use GTKs, IGTKs, and
         * BIGTKs. Unless, of course, actual WEP keys ("pre-RSNA") are used,
         * then unicast frames can also use key indices like GTKs. Hence, if we
         * don't have a PTK/STK we check the key index for a WEP key.
         *
         * Note that in a regular BSS, multicast frames are sent by the
         * AP only, associated stations unicast the frame to the AP first
         * which then multicasts it on their behalf.
         *
         * There is also a slight problem in IBSS mode: GTKs are negotiated
         * with each station, that is something we don't currently handle.
         * The spec seems to expect that one negotiates the same key with
         * every station but there's no such requirement; VLANs could be
         * possible.
         */

        /* start without a key */
        rx->key = NULL;
        fc = hdr->frame_control;

        if (rx->sta) {
                int keyid = rx->sta->ptk_idx;
                sta_ptk = rcu_dereference(rx->sta->ptk[keyid]);

                if (ieee80211_has_protected(fc) &&
                    !(status->flag & RX_FLAG_IV_STRIPPED)) {
                        keyid = ieee80211_get_keyid(rx->skb);

                        if (unlikely(keyid < 0))
                                return RX_DROP_U_NO_KEY_ID;

                        ptk_idx = rcu_dereference(rx->sta->ptk[keyid]);
                }
        }

        if (!ieee80211_has_protected(fc))
                mmie_keyidx = ieee80211_get_mmie_keyidx(rx->skb);

        if (!is_multicast_ether_addr(hdr->addr1) && sta_ptk) {
                rx->key = ptk_idx ? ptk_idx : sta_ptk;
                if ((status->flag & RX_FLAG_DECRYPTED) &&
                    (status->flag & RX_FLAG_IV_STRIPPED))
                        return RX_CONTINUE;
                /* Skip decryption if the frame is not protected. */
                if (!ieee80211_has_protected(fc))
                        return RX_CONTINUE;
        } else if (mmie_keyidx >= 0 && ieee80211_is_beacon(fc)) {
                /* Broadcast/multicast robust management frame / BIP */
                if ((status->flag & RX_FLAG_DECRYPTED) &&
                    (status->flag & RX_FLAG_IV_STRIPPED))
                        return RX_CONTINUE;

                if (mmie_keyidx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS ||
                    mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS +
                                   NUM_DEFAULT_BEACON_KEYS) {
                        if (rx->sdata->dev)
                                cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
                                                             skb->data,
                                                             skb->len);
                        return RX_DROP_M_BAD_BCN_KEYIDX;
                }

                rx->key = ieee80211_rx_get_bigtk(rx, mmie_keyidx);
                if (!rx->key)
                        return RX_CONTINUE; /* Beacon protection not in use */
        } else if (mmie_keyidx >= 0) {
                /* Broadcast/multicast robust management frame / BIP */
                if ((status->flag & RX_FLAG_DECRYPTED) &&
                    (status->flag & RX_FLAG_IV_STRIPPED))
                        return RX_CONTINUE;

                if (mmie_keyidx < NUM_DEFAULT_KEYS ||
                    mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS)
                        return RX_DROP_M_BAD_MGMT_KEYIDX; /* unexpected BIP keyidx */
                if (rx->link_sta) {
                        if (ieee80211_is_group_privacy_action(skb) &&
                            test_sta_flag(rx->sta, WLAN_STA_MFP))
                                return RX_DROP_MONITOR;

                        rx->key = rcu_dereference(rx->link_sta->gtk[mmie_keyidx]);
                }
                if (!rx->key)
                        rx->key = rcu_dereference(rx->link->gtk[mmie_keyidx]);
        } else if (!ieee80211_has_protected(fc)) {
                /*
                 * The frame was not protected, so skip decryption. However, we
                 * need to set rx->key if there is a key that could have been
                 * used so that the frame may be dropped if encryption would
                 * have been expected.
                 */
                struct ieee80211_key *key = NULL;
                int i;

                if (ieee80211_is_beacon(fc)) {
                        key = ieee80211_rx_get_bigtk(rx, -1);
                } else if (ieee80211_is_mgmt(fc) &&
                           is_multicast_ether_addr(hdr->addr1)) {
                        key = rcu_dereference(rx->link->default_mgmt_key);
                } else {
                        if (rx->link_sta) {
                                for (i = 0; i < NUM_DEFAULT_KEYS; i++) {
                                        key = rcu_dereference(rx->link_sta->gtk[i]);
                                        if (key)
                                                break;
                                }
                        }
                        if (!key) {
                                for (i = 0; i < NUM_DEFAULT_KEYS; i++) {
                                        key = rcu_dereference(rx->link->gtk[i]);
                                        if (key)
                                                break;
                                }
                        }
                }
                if (key)
                        rx->key = key;
                return RX_CONTINUE;
        } else {
                /*
                 * The device doesn't give us the IV so we won't be
                 * able to look up the key. That's ok though, we
                 * don't need to decrypt the frame, we just won't
                 * be able to keep statistics accurate.
                 * Except for key threshold notifications, should
                 * we somehow allow the driver to tell us which key
                 * the hardware used if this flag is set?
                 */
                if ((status->flag & RX_FLAG_DECRYPTED) &&
                    (status->flag & RX_FLAG_IV_STRIPPED))
                        return RX_CONTINUE;

                keyidx = ieee80211_get_keyid(rx->skb);

                if (unlikely(keyidx < 0))
                        return RX_DROP_U_NO_KEY_ID;

                /* check per-station GTK first, if multicast packet */
                if (is_multicast_ether_addr(hdr->addr1) && rx->link_sta)
                        rx->key = rcu_dereference(rx->link_sta->gtk[keyidx]);

                /* if not found, try default key */
                if (!rx->key) {
                        if (is_multicast_ether_addr(hdr->addr1))
                                rx->key = rcu_dereference(rx->link->gtk[keyidx]);
                        if (!rx->key)
                                rx->key = rcu_dereference(rx->sdata->keys[keyidx]);

                        /*
                         * RSNA-protected unicast frames should always be
                         * sent with pairwise or station-to-station keys,
                         * but for WEP we allow using a key index as well.
                         */
                        if (rx->key &&
                            rx->key->conf.cipher != WLAN_CIPHER_SUITE_WEP40 &&
                            rx->key->conf.cipher != WLAN_CIPHER_SUITE_WEP104 &&
                            !is_multicast_ether_addr(hdr->addr1))
                                rx->key = NULL;
                }
        }

        if (rx->key) {
                if (unlikely(rx->key->flags & KEY_FLAG_TAINTED))
                        return RX_DROP_MONITOR;

                /* TODO: add threshold stuff again */
        } else {
                return RX_DROP_MONITOR;
        }

        switch (rx->key->conf.cipher) {
        case WLAN_CIPHER_SUITE_WEP40:
        case WLAN_CIPHER_SUITE_WEP104:
                result = ieee80211_crypto_wep_decrypt(rx);
                break;
        case WLAN_CIPHER_SUITE_TKIP:
                result = ieee80211_crypto_tkip_decrypt(rx);
                break;
        case WLAN_CIPHER_SUITE_CCMP:
                result = ieee80211_crypto_ccmp_decrypt(
                        rx, IEEE80211_CCMP_MIC_LEN);
                break;
        case WLAN_CIPHER_SUITE_CCMP_256:
                result = ieee80211_crypto_ccmp_decrypt(
                        rx, IEEE80211_CCMP_256_MIC_LEN);
                break;
        case WLAN_CIPHER_SUITE_AES_CMAC:
                result = ieee80211_crypto_aes_cmac_decrypt(rx);
                break;
        case WLAN_CIPHER_SUITE_BIP_CMAC_256:
                result = ieee80211_crypto_aes_cmac_256_decrypt(rx);
                break;
        case WLAN_CIPHER_SUITE_BIP_GMAC_128:
        case WLAN_CIPHER_SUITE_BIP_GMAC_256:
                result = ieee80211_crypto_aes_gmac_decrypt(rx);
                break;
        case WLAN_CIPHER_SUITE_GCMP:
        case WLAN_CIPHER_SUITE_GCMP_256:
                result = ieee80211_crypto_gcmp_decrypt(rx);
                break;
        default:
                result = RX_DROP_U_BAD_CIPHER;
        }

        /* the hdr variable is invalid after the decrypt handlers */

        /* either the frame has been decrypted or will be dropped */
        status->flag |= RX_FLAG_DECRYPTED;

        if (unlikely(ieee80211_is_beacon(fc) && RX_RES_IS_UNUSABLE(result) &&
                     rx->sdata->dev))
                cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
                                             skb->data, skb->len);

        return result;
}

void ieee80211_init_frag_cache(struct ieee80211_fragment_cache *cache)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(cache->entries); i++)
                skb_queue_head_init(&cache->entries[i].skb_list);
}

void ieee80211_destroy_frag_cache(struct ieee80211_fragment_cache *cache)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(cache->entries); i++)
                __skb_queue_purge(&cache->entries[i].skb_list);
}

static inline struct ieee80211_fragment_entry *
ieee80211_reassemble_add(struct ieee80211_fragment_cache *cache,
                         unsigned int frag, unsigned int seq, int rx_queue,
                         struct sk_buff **skb)
{
        struct ieee80211_fragment_entry *entry;

        entry = &cache->entries[cache->next++];
        if (cache->next >= IEEE80211_FRAGMENT_MAX)
                cache->next = 0;

        __skb_queue_purge(&entry->skb_list);

        __skb_queue_tail(&entry->skb_list, *skb); /* no need for locking */
        *skb = NULL;
        entry->first_frag_time = jiffies;
        entry->seq = seq;
        entry->rx_queue = rx_queue;
        entry->last_frag = frag;
        entry->check_sequential_pn = false;
        entry->extra_len = 0;

        return entry;
}

static inline struct ieee80211_fragment_entry *
ieee80211_reassemble_find(struct ieee80211_fragment_cache *cache,
                          unsigned int frag, unsigned int seq,
                          int rx_queue, struct ieee80211_hdr *hdr)
{
        struct ieee80211_fragment_entry *entry;
        int i, idx;

        idx = cache->next;
        for (i = 0; i < IEEE80211_FRAGMENT_MAX; i++) {
                struct ieee80211_hdr *f_hdr;
                struct sk_buff *f_skb;

                idx--;
                if (idx < 0)
                        idx = IEEE80211_FRAGMENT_MAX - 1;

                entry = &cache->entries[idx];
                if (skb_queue_empty(&entry->skb_list) || entry->seq != seq ||
                    entry->rx_queue != rx_queue ||
                    entry->last_frag + 1 != frag)
                        continue;

                f_skb = __skb_peek(&entry->skb_list);
                f_hdr = (struct ieee80211_hdr *) f_skb->data;

                /*
                 * Check ftype and addresses are equal, else check next fragment
                 */
                if (((hdr->frame_control ^ f_hdr->frame_control) &
                     cpu_to_le16(IEEE80211_FCTL_FTYPE)) ||
                    !ether_addr_equal(hdr->addr1, f_hdr->addr1) ||
                    !ether_addr_equal(hdr->addr2, f_hdr->addr2))
                        continue;

                if (time_after(jiffies, entry->first_frag_time + 2 * HZ)) {
                        __skb_queue_purge(&entry->skb_list);
                        continue;
                }
                return entry;
        }

        return NULL;
}

static bool requires_sequential_pn(struct ieee80211_rx_data *rx, __le16 fc)
{
        return rx->key &&
                (rx->key->conf.cipher == WLAN_CIPHER_SUITE_CCMP ||
                 rx->key->conf.cipher == WLAN_CIPHER_SUITE_CCMP_256 ||
                 rx->key->conf.cipher == WLAN_CIPHER_SUITE_GCMP ||
                 rx->key->conf.cipher == WLAN_CIPHER_SUITE_GCMP_256) &&
                ieee80211_has_protected(fc);
}

static ieee80211_rx_result debug_noinline
ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx)
{
        struct ieee80211_fragment_cache *cache = &rx->sdata->frags;
        struct ieee80211_hdr *hdr;
        u16 sc;
        __le16 fc;
        unsigned int frag, seq;
        struct ieee80211_fragment_entry *entry;
        struct sk_buff *skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);

        hdr = (struct ieee80211_hdr *)rx->skb->data;
        fc = hdr->frame_control;

        if (ieee80211_is_ctl(fc) || ieee80211_is_ext(fc))
                return RX_CONTINUE;

        sc = le16_to_cpu(hdr->seq_ctrl);
        frag = sc & IEEE80211_SCTL_FRAG;

        if (rx->sta)
                cache = &rx->sta->frags;

        if (likely(!ieee80211_has_morefrags(fc) && frag == 0))
                goto out;

        if (is_multicast_ether_addr(hdr->addr1))
                return RX_DROP_MONITOR;

        I802_DEBUG_INC(rx->local->rx_handlers_fragments);

        if (skb_linearize(rx->skb))
                return RX_DROP_U_OOM;

        /*
         *  skb_linearize() might change the skb->data and
         *  previously cached variables (in this case, hdr) need to
         *  be refreshed with the new data.
         */
        hdr = (struct ieee80211_hdr *)rx->skb->data;
        seq = (sc & IEEE80211_SCTL_SEQ) >> 4;

        if (frag == 0) {
                /* This is the first fragment of a new frame. */
                entry = ieee80211_reassemble_add(cache, frag, seq,
                                                 rx->seqno_idx, &(rx->skb));
                if (requires_sequential_pn(rx, fc)) {
                        int queue = rx->security_idx;

                        /* Store CCMP/GCMP PN so that we can verify that the
                         * next fragment has a sequential PN value.
                         */
                        entry->check_sequential_pn = true;
                        entry->is_protected = true;
                        entry->key_color = rx->key->color;
                        memcpy(entry->last_pn,
                               rx->key->u.ccmp.rx_pn[queue],
                               IEEE80211_CCMP_PN_LEN);
                        BUILD_BUG_ON(offsetof(struct ieee80211_key,
                                              u.ccmp.rx_pn) !=
                                     offsetof(struct ieee80211_key,
                                              u.gcmp.rx_pn));
                        BUILD_BUG_ON(sizeof(rx->key->u.ccmp.rx_pn[queue]) !=
                                     sizeof(rx->key->u.gcmp.rx_pn[queue]));
                        BUILD_BUG_ON(IEEE80211_CCMP_PN_LEN !=
                                     IEEE80211_GCMP_PN_LEN);
                } else if (rx->key &&
                           (ieee80211_has_protected(fc) ||
                            (status->flag & RX_FLAG_DECRYPTED))) {
                        entry->is_protected = true;
                        entry->key_color = rx->key->color;
                }
                return RX_QUEUED;
        }

        /* This is a fragment for a frame that should already be pending in
         * fragment cache. Add this fragment to the end of the pending entry.
         */
        entry = ieee80211_reassemble_find(cache, frag, seq,
                                          rx->seqno_idx, hdr);
        if (!entry) {
                I802_DEBUG_INC(rx->local->rx_handlers_drop_defrag);
                return RX_DROP_MONITOR;
        }

        /* "The receiver shall discard MSDUs and MMPDUs whose constituent
         *  MPDU PN values are not incrementing in steps of 1."
         * see IEEE P802.11-REVmc/D5.0, 12.5.3.4.4, item d (for CCMP)
         * and IEEE P802.11-REVmc/D5.0, 12.5.5.4.4, item d (for GCMP)
         */
        if (entry->check_sequential_pn) {
                int i;
                u8 pn[IEEE80211_CCMP_PN_LEN], *rpn;

                if (!requires_sequential_pn(rx, fc))
                        return RX_DROP_U_NONSEQ_PN;

                /* Prevent mixed key and fragment cache attacks */
                if (entry->key_color != rx->key->color)
                        return RX_DROP_U_BAD_KEY_COLOR;

                memcpy(pn, entry->last_pn, IEEE80211_CCMP_PN_LEN);
                for (i = IEEE80211_CCMP_PN_LEN - 1; i >= 0; i--) {
                        pn[i]++;
                        if (pn[i])
                                break;
                }

                rpn = rx->ccm_gcm.pn;
                if (memcmp(pn, rpn, IEEE80211_CCMP_PN_LEN))
                        return RX_DROP_U_REPLAY;
                memcpy(entry->last_pn, pn, IEEE80211_CCMP_PN_LEN);
        } else if (entry->is_protected &&
                   (!rx->key ||
                    (!ieee80211_has_protected(fc) &&
                     !(status->flag & RX_FLAG_DECRYPTED)) ||
                    rx->key->color != entry->key_color)) {
                /* Drop this as a mixed key or fragment cache attack, even
                 * if for TKIP Michael MIC should protect us, and WEP is a
                 * lost cause anyway.
                 */
                return RX_DROP_U_EXPECT_DEFRAG_PROT;
        } else if (entry->is_protected && rx->key &&
                   entry->key_color != rx->key->color &&
                   (status->flag & RX_FLAG_DECRYPTED)) {
                return RX_DROP_U_BAD_KEY_COLOR;
        }

        skb_pull(rx->skb, ieee80211_hdrlen(fc));
        __skb_queue_tail(&entry->skb_list, rx->skb);
        entry->last_frag = frag;
        entry->extra_len += rx->skb->len;
        if (ieee80211_has_morefrags(fc)) {
                rx->skb = NULL;
                return RX_QUEUED;
        }

        rx->skb = __skb_dequeue(&entry->skb_list);
        if (skb_tailroom(rx->skb) < entry->extra_len) {
                I802_DEBUG_INC(rx->local->rx_expand_skb_head_defrag);
                if (unlikely(pskb_expand_head(rx->skb, 0, entry->extra_len,
                                              GFP_ATOMIC))) {
                        I802_DEBUG_INC(rx->local->rx_handlers_drop_defrag);
                        __skb_queue_purge(&entry->skb_list);
                        return RX_DROP_U_OOM;
                }
        }
        while ((skb = __skb_dequeue(&entry->skb_list))) {
                skb_put_data(rx->skb, skb->data, skb->len);
                dev_kfree_skb(skb);
        }

 out:
        ieee80211_led_rx(rx->local);
        if (rx->sta)
                rx->link_sta->rx_stats.packets++;
        return RX_CONTINUE;
}

static int ieee80211_802_1x_port_control(struct ieee80211_rx_data *rx)
{
        if (unlikely(!rx->sta || !test_sta_flag(rx->sta, WLAN_STA_AUTHORIZED)))
                return -EACCES;

        return 0;
}

static int ieee80211_drop_unencrypted(struct ieee80211_rx_data *rx, __le16 fc)
{
        struct sk_buff *skb = rx->skb;
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);

        /*
         * Pass through unencrypted frames if the hardware has
         * decrypted them already.
         */
        if (status->flag & RX_FLAG_DECRYPTED)
                return 0;

        /* Drop unencrypted frames if key is set. */
        if (unlikely(!ieee80211_has_protected(fc) &&
                     !ieee80211_is_any_nullfunc(fc) &&
                     ieee80211_is_data(fc) && rx->key))
                return -EACCES;

        return 0;
}

VISIBLE_IF_MAC80211_KUNIT ieee80211_rx_result
ieee80211_drop_unencrypted_mgmt(struct ieee80211_rx_data *rx)
{
        struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);
        struct ieee80211_mgmt *mgmt = (void *)rx->skb->data;
        __le16 fc = mgmt->frame_control;

        /*
         * Pass through unencrypted frames if the hardware has
         * decrypted them already.
         */
        if (status->flag & RX_FLAG_DECRYPTED)
                return RX_CONTINUE;

        /* drop unicast protected dual (that wasn't protected) */
        if (ieee80211_is_action(fc) &&
            mgmt->u.action.category == WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
                return RX_DROP_U_UNPROT_DUAL;

        if (rx->sta && test_sta_flag(rx->sta, WLAN_STA_MFP)) {
                if (unlikely(!ieee80211_has_protected(fc) &&
                             ieee80211_is_unicast_robust_mgmt_frame(rx->skb))) {
                        if (ieee80211_is_deauth(fc) ||
                            ieee80211_is_disassoc(fc)) {
                                /*
                                 * Permit unprotected deauth/disassoc frames
                                 * during 4-way-HS (key is installed after HS).
                                 */
                                if (!rx->key)
                                        return RX_CONTINUE;

                                cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
                                                             rx->skb->data,
                                                             rx->skb->len);
                        }
                        return RX_DROP_U_UNPROT_UCAST_MGMT;
                }
                /* BIP does not use Protected field, so need to check MMIE */
                if (unlikely(ieee80211_is_multicast_robust_mgmt_frame(rx->skb) &&
                             ieee80211_get_mmie_keyidx(rx->skb) < 0)) {
                        if (ieee80211_is_deauth(fc) ||
                            ieee80211_is_disassoc(fc))
                                cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
                                                             rx->skb->data,
                                                             rx->skb->len);
                        return RX_DROP_U_UNPROT_MCAST_MGMT;
                }
                if (unlikely(ieee80211_is_beacon(fc) && rx->key &&
                             ieee80211_get_mmie_keyidx(rx->skb) < 0)) {
                        cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev,
                                                     rx->skb->data,
                                                     rx->skb->len);
                        return RX_DROP_U_UNPROT_BEACON;
                }
                /*
                 * When using MFP, Action frames are not allowed prior to
                 * having configured keys.
                 */
                if (unlikely(ieee80211_is_action(fc) && !rx->key &&
                             ieee80211_is_robust_mgmt_frame(rx->skb)))
                        return RX_DROP_U_UNPROT_ACTION;

                /* drop unicast public action frames when using MPF */
                if (is_unicast_ether_addr(mgmt->da) &&
                    ieee80211_is_protected_dual_of_public_action(rx->skb))
                        return RX_DROP_U_UNPROT_UNICAST_PUB_ACTION;
        }

        /*
         * Drop robust action frames before assoc regardless of MFP state,
         * after assoc we also have decided on MFP or not.
         */
        if (ieee80211_is_action(fc) &&
            ieee80211_is_robust_mgmt_frame(rx->skb) &&
            (!rx->sta || !test_sta_flag(rx->sta, WLAN_STA_ASSOC)))
                return RX_DROP_U_UNPROT_ROBUST_ACTION;

        return RX_CONTINUE;
}
EXPORT_SYMBOL_IF_MAC80211_KUNIT(ieee80211_drop_unencrypted_mgmt);

static ieee80211_rx_result
__ieee80211_data_to_8023(struct ieee80211_rx_data *rx, bool *port_control)
{
        struct ieee80211_sub_if_data *sdata = rx->sdata;
        struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)rx->skb->data;
        bool check_port_control = false;
        struct ethhdr *ehdr;
        int ret;

        *port_control = false;
        if (ieee80211_has_a4(hdr->frame_control) &&
            sdata->vif.type == NL80211_IFTYPE_AP_VLAN && !sdata->u.vlan.sta)
                return RX_DROP_U_UNEXPECTED_VLAN_4ADDR;

        if (sdata->vif.type == NL80211_IFTYPE_STATION &&
            !!sdata->u.mgd.use_4addr != !!ieee80211_has_a4(hdr->frame_control)) {
                if (!sdata->u.mgd.use_4addr)
                        return RX_DROP_U_UNEXPECTED_STA_4ADDR;
                else if (!ether_addr_equal(hdr->addr1, sdata->vif.addr))
                        check_port_control = true;
        }

        if (is_multicast_ether_addr(hdr->addr1) &&
            sdata->vif.type == NL80211_IFTYPE_AP_VLAN && sdata->u.vlan.sta)
                return RX_DROP_U_UNEXPECTED_VLAN_MCAST;

        ret = ieee80211_data_to_8023(rx->skb, sdata->vif.addr, sdata->vif.type);
        if (ret < 0)
                return RX_DROP_U_INVALID_8023;

        ehdr = (struct ethhdr *) rx->skb->data;
        if (ehdr->h_proto == rx->sdata->control_port_protocol)
                *port_control = true;
        else if (check_port_control)
                return RX_DROP_U_NOT_PORT_CONTROL;

        return RX_CONTINUE;
}

bool ieee80211_is_our_addr(struct ieee80211_sub_if_data *sdata,
                           const u8 *addr, int *out_link_id)
{
        unsigned int link_id;

        /* non-MLO, or MLD address replaced by hardware */
        if (ether_addr_equal(sdata->vif.addr, addr))
                return true;

        if (!ieee80211_vif_is_mld(&sdata->vif))
                return false;

        for (link_id = 0; link_id < ARRAY_SIZE(sdata->vif.link_conf); link_id++) {
                struct ieee80211_bss_conf *conf;

                conf = rcu_dereference(sdata->vif.link_conf[link_id]);

                if (!conf)
                        continue;
                if (ether_addr_equal(conf->addr, addr)) {
                        if (out_link_id)
                                *out_link_id = link_id;
                        return true;
                }
        }

        return false;
}

/*
 * requires that rx->skb is a frame with ethernet header
 */
static bool ieee80211_frame_allowed(struct ieee80211_rx_data *rx, __le16 fc)
{
        static const u8 pae_group_addr[ETH_ALEN] __aligned(2)
                = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x03 };
        struct ethhdr *ehdr = (struct ethhdr *) rx->skb->data;

        /*
         * Allow EAPOL frames to us/the PAE group address regardless of
         * whether the frame was encrypted or not, and always disallow
         * all other destination addresses for them.
         */
        if (unlikely(ehdr->h_proto == rx->sdata->control_port_protocol))
                return ieee80211_is_our_addr(rx->sdata, ehdr->h_dest, NULL) ||
                       ether_addr_equal(ehdr->h_dest, pae_group_addr);

        if (ieee80211_802_1x_port_control(rx) ||
            ieee80211_drop_unencrypted(rx, fc))
                return false;

        return true;
}

static void ieee80211_deliver_skb_to_local_stack(struct sk_buff *skb,
                                                 struct ieee80211_rx_data *rx)
{
        struct ieee80211_sub_if_data *sdata = rx->sdata;
        struct net_device *dev = sdata->dev;

        if (unlikely((skb->protocol == sdata->control_port_protocol ||
                     (skb->protocol == cpu_to_be16(ETH_P_PREAUTH) &&
                      !sdata->control_port_no_preauth)) &&
                     sdata->control_port_over_nl80211)) {
                struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
                bool noencrypt = !(status->flag & RX_FLAG_DECRYPTED);

                cfg80211_rx_control_port(dev, skb, noencrypt, rx->link_id);
                dev_kfree_skb(skb);
        } else {
                struct ethhdr *ehdr = (void *)skb_mac_header(skb);

                memset(skb->cb, 0, sizeof(skb->cb));

                /*
                 * 802.1X over 802.11 requires that the authenticator address
                 * be used for EAPOL frames. However, 802.1X allows the use of
                 * the PAE group address instead. If the interface is part of
                 * a bridge and we pass the frame with the PAE group address,
                 * then the bridge will forward it to the network (even if the
                 * client was not associated yet), which isn't supposed to
                 * happen.
                 * To avoid that, rewrite the destination address to our own
                 * address, so that the authenticator (e.g. hostapd) will see
                 * the frame, but bridge won't forward it anywhere else. Note
                 * that due to earlier filtering, the only other address can
                 * be the PAE group address, unless the hardware allowed them
                 * through in 802.3 offloaded mode.
                 */
                if (unlikely(skb->protocol == sdata->control_port_protocol &&
                             !ether_addr_equal(ehdr->h_dest, sdata->vif.addr)))
                        ether_addr_copy(ehdr->h_dest, sdata->vif.addr);

                /* deliver to local stack */
                if (rx->list)
                        list_add_tail(&skb->list, rx->list);
                else
                        netif_receive_skb(skb);
        }
}

/*
 * requires that rx->skb is a frame with ethernet header
 */
static void
ieee80211_deliver_skb(struct ieee80211_rx_data *rx)
{
        struct ieee80211_sub_if_data *sdata = rx->sdata;
        struct net_device *dev = sdata->dev;
        struct sk_buff *skb, *xmit_skb;
        struct ethhdr *ehdr = (struct ethhdr *) rx->skb->data;
        struct sta_info *dsta;

        skb = rx->skb;
        xmit_skb = NULL;

        dev_sw_netstats_rx_add(dev, skb->len);

        if (rx->sta) {
                /* The seqno index has the same property as needed
                 * for the rx_msdu field, i.e. it is IEEE80211_NUM_TIDS
                 * for non-QoS-data frames. Here we know it's a data
                 * frame, so count MSDUs.
                 */
                u64_stats_update_begin(&rx->link_sta->rx_stats.syncp);
                rx->link_sta->rx_stats.msdu[rx->seqno_idx]++;
                u64_stats_update_end(&rx->link_sta->rx_stats.syncp);
        }

        if ((sdata->vif.type == NL80211_IFTYPE_AP ||
             sdata->vif.type == NL80211_IFTYPE_AP_VLAN) &&
            !(sdata->flags & IEEE80211_SDATA_DONT_BRIDGE_PACKETS) &&
            ehdr->h_proto != rx->sdata->control_port_protocol &&
            (sdata->vif.type != NL80211_IFTYPE_AP_VLAN || !sdata->u.vlan.sta)) {
                if (is_multicast_ether_addr(ehdr->h_dest) &&
                    ieee80211_vif_get_num_mcast_if(sdata) != 0) {
                        /*
                         * send multicast frames both to higher layers in
                         * local net stack and back to the wireless medium
                         */
                        xmit_skb = skb_copy(skb, GFP_ATOMIC);
                        if (!xmit_skb)
                                net_info_ratelimited("%s: failed to clone multicast frame\n",
                                                    dev->name);
                } else if (!is_multicast_ether_addr(ehdr->h_dest) &&
                           !ether_addr_equal(ehdr->h_dest, ehdr->h_source)) {
                        dsta = sta_info_get(sdata, ehdr->h_dest);
                        if (dsta) {
                                /*
                                 * The destination station is associated to
                                 * this AP (in this VLAN), so send the frame
                                 * directly to it and do not pass it to local
                                 * net stack.
                                 */
                                xmit_skb = skb;
                                skb = NULL;
                        }
                }
        }

#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
        if (skb) {
                /* 'align' will only take the values 0 or 2 here since all
                 * frames are required to be aligned to 2-byte boundaries
                 * when being passed to mac80211; the code here works just
                 * as well if that isn't true, but mac80211 assumes it can
                 * access fields as 2-byte aligned (e.g. for ether_addr_equal)
                 */
                int align;

                align = (unsigned long)(skb->data + sizeof(struct ethhdr)) & 3;
                if (align) {
                        if (WARN_ON(skb_headroom(skb) < 3)) {
                                dev_kfree_skb(skb);
                                skb = NULL;
                        } else {
                                u8 *data = skb->data;
                                size_t len = skb_headlen(skb);
                                skb->data -= align;
                                memmove(skb->data, data, len);
                                skb_set_tail_pointer(skb, len);
                        }
                }
        }
#endif

        if (skb) {
                skb->protocol = eth_type_trans(skb, dev);
                ieee80211_deliver_skb_to_local_stack(skb, rx);
        }

        if (xmit_skb) {
                /*
                 * Send to wireless media and increase priority by 256 to
                 * keep the received priority instead of reclassifying
                 * the frame (see cfg80211_classify8021d).
                 */
                xmit_skb->priority += 256;
                xmit_skb->protocol = htons(ETH_P_802_3);
                skb_reset_network_header(xmit_skb);
                skb_reset_mac_header(xmit_skb);
                dev_queue_xmit(xmit_skb);
        }
}

#ifdef CONFIG_MAC80211_MESH
static bool
ieee80211_rx_mesh_fast_forward(struct ieee80211_sub_if_data *sdata,
                               struct sk_buff *skb, int hdrlen)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_mesh_fast_tx_key key = {
                .type = MESH_FAST_TX_TYPE_FORWARDED
        };
        struct ieee80211_mesh_fast_tx *entry;
        struct ieee80211s_hdr *mesh_hdr;
        struct tid_ampdu_tx *tid_tx;
        struct sta_info *sta;
        struct ethhdr eth;
        u8 tid;

        mesh_hdr = (struct ieee80211s_hdr *)(skb->data + siz