Total coverage: 150350 (9%)of 1848919 
























    6 




























    2 



















    4 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_TC_GACT_H
#define __NET_TC_GACT_H

#include <net/act_api.h>
#include <linux/tc_act/tc_gact.h>

struct tcf_gact {
        struct tc_action        common;
#ifdef CONFIG_GACT_PROB
        u16                        tcfg_ptype;
        u16                        tcfg_pval;
        int                        tcfg_paction;
        atomic_t                packets;
#endif
};
#define to_gact(a) ((struct tcf_gact *)a)

static inline bool __is_tcf_gact_act(const struct tc_action *a, int act,
                                     bool is_ext)
{
#ifdef CONFIG_NET_CLS_ACT
        struct tcf_gact *gact;

        if (a->ops && a->ops->id != TCA_ID_GACT)
                return false;

        gact = to_gact(a);
        if ((!is_ext && gact->tcf_action == act) ||
            (is_ext && TC_ACT_EXT_CMP(gact->tcf_action, act)))
                return true;

#endif
        return false;
}

static inline bool is_tcf_gact_ok(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_OK, false);
}

static inline bool is_tcf_gact_shot(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_SHOT, false);
}

static inline bool is_tcf_gact_trap(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_TRAP, false);
}

static inline bool is_tcf_gact_goto_chain(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_GOTO_CHAIN, true);
}

static inline u32 tcf_gact_goto_chain_index(const struct tc_action *a)
{
        return READ_ONCE(a->tcfa_action) & TC_ACT_EXT_VAL_MASK;
}

static inline bool is_tcf_gact_continue(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_UNSPEC, false);
}

static inline bool is_tcf_gact_reclassify(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_RECLASSIFY, false);
}

static inline bool is_tcf_gact_pipe(const struct tc_action *a)
{
        return __is_tcf_gact_act(a, TC_ACT_PIPE, false);
}

#endif /* __NET_TC_GACT_H */






















    9 


    9 

    4 




    6 







    6 




























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2015, Heiner Kallweit <hkallweit1@gmail.com>
 */

#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>

#include "leds.h"

DEFINE_LED_TRIGGER(bt_power_led_trigger);

struct hci_basic_led_trigger {
        struct led_trigger        led_trigger;
        struct hci_dev                *hdev;
};

#define to_hci_basic_led_trigger(arg) container_of(arg, \
                        struct hci_basic_led_trigger, led_trigger)

void hci_leds_update_powered(struct hci_dev *hdev, bool enabled)
{
        if (hdev->power_led)
                led_trigger_event(hdev->power_led,
                                  enabled ? LED_FULL : LED_OFF);

        if (!enabled) {
                struct hci_dev *d;

                read_lock(&hci_dev_list_lock);

                list_for_each_entry(d, &hci_dev_list, list) {
                        if (test_bit(HCI_UP, &d->flags))
                                enabled = true;
                }

                read_unlock(&hci_dev_list_lock);
        }

        led_trigger_event(bt_power_led_trigger, enabled ? LED_FULL : LED_OFF);
}

static int power_activate(struct led_classdev *led_cdev)
{
        struct hci_basic_led_trigger *htrig;
        bool powered;

        htrig = to_hci_basic_led_trigger(led_cdev->trigger);
        powered = test_bit(HCI_UP, &htrig->hdev->flags);

        led_set_brightness(led_cdev, powered ? LED_FULL : LED_OFF);

        return 0;
}

static struct led_trigger *led_allocate_basic(struct hci_dev *hdev,
                        int (*activate)(struct led_classdev *led_cdev),
                        const char *name)
{
        struct hci_basic_led_trigger *htrig;

        htrig =        devm_kzalloc(&hdev->dev, sizeof(*htrig), GFP_KERNEL);
        if (!htrig)
                return NULL;

        htrig->hdev = hdev;
        htrig->led_trigger.activate = activate;
        htrig->led_trigger.name = devm_kasprintf(&hdev->dev, GFP_KERNEL,
                                                 "%s-%s", hdev->name,
                                                 name);
        if (!htrig->led_trigger.name)
                goto err_alloc;

        if (devm_led_trigger_register(&hdev->dev, &htrig->led_trigger))
                goto err_register;

        return &htrig->led_trigger;

err_register:
        devm_kfree(&hdev->dev, (void *)htrig->led_trigger.name);
err_alloc:
        devm_kfree(&hdev->dev, htrig);
        return NULL;
}

void hci_leds_init(struct hci_dev *hdev)
{
        /* initialize power_led */
        hdev->power_led = led_allocate_basic(hdev, power_activate, "power");
}

void bt_leds_init(void)
{
        led_trigger_register_simple("bluetooth-power", &bt_power_led_trigger);
}

void bt_leds_cleanup(void)
{
        led_trigger_unregister_simple(bt_power_led_trigger);
}












































    5 



































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BIO_INTEGRITY_H
#define _LINUX_BIO_INTEGRITY_H

#include <linux/bio.h>

enum bip_flags {
        BIP_BLOCK_INTEGRITY        = 1 << 0, /* block layer owns integrity data */
        BIP_MAPPED_INTEGRITY        = 1 << 1, /* ref tag has been remapped */
        BIP_DISK_NOCHECK        = 1 << 2, /* disable disk integrity checking */
        BIP_IP_CHECKSUM                = 1 << 3, /* IP checksum */
        BIP_COPY_USER                = 1 << 4, /* Kernel bounce buffer in use */
        BIP_CHECK_GUARD                = 1 << 5, /* guard check */
        BIP_CHECK_REFTAG        = 1 << 6, /* reftag check */
        BIP_CHECK_APPTAG        = 1 << 7, /* apptag check */
        BIP_P2P_DMA                = 1 << 8, /* using P2P address */
};

struct bio_integrity_payload {
        struct bvec_iter        bip_iter;

        unsigned short                bip_vcnt;        /* # of integrity bio_vecs */
        unsigned short                bip_max_vcnt;        /* integrity bio_vec slots */
        unsigned short                bip_flags;        /* control flags */
        u16                        app_tag;        /* application tag value */

        struct bio_vec                *bip_vec;
};

#define BIP_CLONE_FLAGS (BIP_MAPPED_INTEGRITY | BIP_IP_CHECKSUM | \
                         BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)

#ifdef CONFIG_BLK_DEV_INTEGRITY

#define bip_for_each_vec(bvl, bip, iter)                                \
        for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)

#define bio_for_each_integrity_vec(_bvl, _bio, _iter)                        \
        for_each_bio(_bio)                                                \
                bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)

static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
{
        if (bio->bi_opf & REQ_INTEGRITY)
                return bio->bi_integrity;

        return NULL;
}

static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
{
        struct bio_integrity_payload *bip = bio_integrity(bio);

        if (bip)
                return bip->bip_flags & flag;

        return false;
}

static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
{
        return bip->bip_iter.bi_sector;
}

static inline void bip_set_seed(struct bio_integrity_payload *bip,
                                sector_t seed)
{
        bip->bip_iter.bi_sector = seed;
}

void bio_integrity_init(struct bio *bio, struct bio_integrity_payload *bip,
                struct bio_vec *bvecs, unsigned int nr_vecs);
struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, gfp_t gfp,
                unsigned int nr);
int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len,
                unsigned int offset);
int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter);
int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta);
void bio_integrity_unmap_user(struct bio *bio);
bool bio_integrity_prep(struct bio *bio);
void bio_integrity_advance(struct bio *bio, unsigned int bytes_done);
void bio_integrity_trim(struct bio *bio);
int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask);

#else /* CONFIG_BLK_DEV_INTEGRITY */

static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
{
        return NULL;
}

static inline int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
{
        return -EINVAL;
}

static inline int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta)
{
        return -EINVAL;
}

static inline void bio_integrity_unmap_user(struct bio *bio)
{
}

static inline bool bio_integrity_prep(struct bio *bio)
{
        return true;
}

static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
                gfp_t gfp_mask)
{
        return 0;
}

static inline void bio_integrity_advance(struct bio *bio,
                unsigned int bytes_done)
{
}

static inline void bio_integrity_trim(struct bio *bio)
{
}

static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
{
        return false;
}

static inline struct bio_integrity_payload *
bio_integrity_alloc(struct bio *bio, gfp_t gfp, unsigned int nr)
{
        return ERR_PTR(-EINVAL);
}

static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
                                        unsigned int len, unsigned int offset)
{
        return 0;
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */
#endif /* _LINUX_BIO_INTEGRITY_H */










































































































































 1096 







 1102 































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/etherdevice.h>
#include "ipvlan.h"
#include <linux/if_vlan.h>
#include <linux/if_tap.h>
#include <linux/interrupt.h>
#include <linux/nsproxy.h>
#include <linux/compat.h>
#include <linux/if_tun.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/cache.h>
#include <linux/sched.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/idr.h>
#include <linux/fs.h>
#include <linux/uio.h>

#include <net/net_namespace.h>
#include <net/rtnetlink.h>
#include <net/sock.h>
#include <linux/virtio_net.h>

#define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
                      NETIF_F_TSO6)

static dev_t ipvtap_major;
static struct cdev ipvtap_cdev;

static const void *ipvtap_net_namespace(const struct device *d)
{
        const struct net_device *dev = to_net_dev(d->parent);
        return dev_net(dev);
}

static struct class ipvtap_class = {
         .name = "ipvtap",
         .ns_type = &net_ns_type_operations,
         .namespace = ipvtap_net_namespace,
};

struct ipvtap_dev {
        struct ipvl_dev vlan;
        struct tap_dev          tap;
};

static void ipvtap_count_tx_dropped(struct tap_dev *tap)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        this_cpu_inc(vlan->pcpu_stats->tx_drps);
}

static void ipvtap_count_rx_dropped(struct tap_dev *tap)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        ipvlan_count_rx(vlan, 0, 0, 0);
}

static void ipvtap_update_features(struct tap_dev *tap,
                                   netdev_features_t features)
{
        struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap);
        struct ipvl_dev *vlan = &vlantap->vlan;

        vlan->sfeatures = features;
        netdev_update_features(vlan->dev);
}

static int ipvtap_newlink(struct net_device *dev,
                          struct rtnl_newlink_params *params,
                          struct netlink_ext_ack *extack)
{
        struct ipvtap_dev *vlantap = netdev_priv(dev);
        int err;

        INIT_LIST_HEAD(&vlantap->tap.queue_list);

        /* Since macvlan supports all offloads by default, make
         * tap support all offloads also.
         */
        vlantap->tap.tap_features = TUN_OFFLOADS;
        vlantap->tap.count_tx_dropped = ipvtap_count_tx_dropped;
        vlantap->tap.update_features =        ipvtap_update_features;
        vlantap->tap.count_rx_dropped = ipvtap_count_rx_dropped;

        err = netdev_rx_handler_register(dev, tap_handle_frame, &vlantap->tap);
        if (err)
                return err;

        /* Don't put anything that may fail after macvlan_common_newlink
         * because we can't undo what it does.
         */
        err = ipvlan_link_new(dev, params, extack);
        if (err) {
                netdev_rx_handler_unregister(dev);
                return err;
        }

        vlantap->tap.dev = vlantap->vlan.dev;

        return err;
}

static void ipvtap_dellink(struct net_device *dev,
                           struct list_head *head)
{
        struct ipvtap_dev *vlan = netdev_priv(dev);

        netdev_rx_handler_unregister(dev);
        tap_del_queues(&vlan->tap);
        ipvlan_link_delete(dev, head);
}

static void ipvtap_setup(struct net_device *dev)
{
        ipvlan_link_setup(dev);
        dev->tx_queue_len = TUN_READQ_SIZE;
        dev->priv_flags &= ~IFF_NO_QUEUE;
}

static struct rtnl_link_ops ipvtap_link_ops __read_mostly = {
        .kind                = "ipvtap",
        .setup                = ipvtap_setup,
        .newlink        = ipvtap_newlink,
        .dellink        = ipvtap_dellink,
        .priv_size        = sizeof(struct ipvtap_dev),
};

static int ipvtap_device_event(struct notifier_block *unused,
                               unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct ipvtap_dev *vlantap;
        struct device *classdev;
        dev_t devt;
        int err;
        char tap_name[IFNAMSIZ];

        if (dev->rtnl_link_ops != &ipvtap_link_ops)
                return NOTIFY_DONE;

        snprintf(tap_name, IFNAMSIZ, "tap%d", dev->ifindex);
        vlantap = netdev_priv(dev);

        switch (event) {
        case NETDEV_REGISTER:
                /* Create the device node here after the network device has
                 * been registered but before register_netdevice has
                 * finished running.
                 */
                err = tap_get_minor(ipvtap_major, &vlantap->tap);
                if (err)
                        return notifier_from_errno(err);

                devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor);
                classdev = device_create(&ipvtap_class, &dev->dev, devt,
                                         dev, "%s", tap_name);
                if (IS_ERR(classdev)) {
                        tap_free_minor(ipvtap_major, &vlantap->tap);
                        return notifier_from_errno(PTR_ERR(classdev));
                }
                err = sysfs_create_link(&dev->dev.kobj, &classdev->kobj,
                                        tap_name);
                if (err)
                        return notifier_from_errno(err);
                break;
        case NETDEV_UNREGISTER:
                /* vlan->minor == 0 if NETDEV_REGISTER above failed */
                if (vlantap->tap.minor == 0)
                        break;
                sysfs_remove_link(&dev->dev.kobj, tap_name);
                devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor);
                device_destroy(&ipvtap_class, devt);
                tap_free_minor(ipvtap_major, &vlantap->tap);
                break;
        case NETDEV_CHANGE_TX_QUEUE_LEN:
                if (tap_queue_resize(&vlantap->tap))
                        return NOTIFY_BAD;
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block ipvtap_notifier_block __read_mostly = {
        .notifier_call        = ipvtap_device_event,
};

static int __init ipvtap_init(void)
{
        int err;

        err = tap_create_cdev(&ipvtap_cdev, &ipvtap_major, "ipvtap",
                              THIS_MODULE);
        if (err)
                goto out1;

        err = class_register(&ipvtap_class);
        if (err)
                goto out2;

        err = register_netdevice_notifier(&ipvtap_notifier_block);
        if (err)
                goto out3;

        err = ipvlan_link_register(&ipvtap_link_ops);
        if (err)
                goto out4;

        return 0;

out4:
        unregister_netdevice_notifier(&ipvtap_notifier_block);
out3:
        class_unregister(&ipvtap_class);
out2:
        tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
out1:
        return err;
}
module_init(ipvtap_init);

static void __exit ipvtap_exit(void)
{
        rtnl_link_unregister(&ipvtap_link_ops);
        unregister_netdevice_notifier(&ipvtap_notifier_block);
        class_unregister(&ipvtap_class);
        tap_destroy_cdev(ipvtap_major, &ipvtap_cdev);
}
module_exit(ipvtap_exit);
MODULE_ALIAS_RTNL_LINK("ipvtap");
MODULE_AUTHOR("Sainath Grandhi <sainath.grandhi@intel.com>");
MODULE_DESCRIPTION("IP-VLAN based tap driver");
MODULE_LICENSE("GPL");






































   55 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Marek Lindner
 */

#ifndef _NET_BATMAN_ADV_MESH_INTERFACE_H_
#define _NET_BATMAN_ADV_MESH_INTERFACE_H_

#include "main.h"

#include <linux/kref.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/types.h>

int batadv_skb_head_push(struct sk_buff *skb, unsigned int len);
void batadv_interface_rx(struct net_device *mesh_iface,
                         struct sk_buff *skb, int hdr_size,
                         struct batadv_orig_node *orig_node);
bool batadv_meshif_is_valid(const struct net_device *net_dev);
extern struct rtnl_link_ops batadv_link_ops;
int batadv_meshif_create_vlan(struct batadv_priv *bat_priv, unsigned short vid);
void batadv_meshif_vlan_release(struct kref *ref);
struct batadv_meshif_vlan *batadv_meshif_vlan_get(struct batadv_priv *bat_priv,
                                                  unsigned short vid);

/**
 * batadv_meshif_vlan_put() - decrease the vlan object refcounter and
 *  possibly release it
 * @vlan: the vlan object to release
 */
static inline void batadv_meshif_vlan_put(struct batadv_meshif_vlan *vlan)
{
        if (!vlan)
                return;

        kref_put(&vlan->refcount, batadv_meshif_vlan_release);
}

#endif /* _NET_BATMAN_ADV_MESH_INTERFACE_H_ */










































































































































































































































   37 












   38 
   37 
























































   59 









   57 












   36 




   30 


   29 


























































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains functions which manage clock event devices.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 */

#include <linux/clockchips.h>
#include <linux/hrtimer.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/device.h>

#include "tick-internal.h"

/* The registered clock event devices */
static LIST_HEAD(clockevent_devices);
static LIST_HEAD(clockevents_released);
/* Protection for the above */
static DEFINE_RAW_SPINLOCK(clockevents_lock);
/* Protection for unbind operations */
static DEFINE_MUTEX(clockevents_mutex);

struct ce_unbind {
        struct clock_event_device *ce;
        int res;
};

static u64 cev_delta2ns(unsigned long latch, struct clock_event_device *evt,
                        bool ismax)
{
        u64 clc = (u64) latch << evt->shift;
        u64 rnd;

        if (WARN_ON(!evt->mult))
                evt->mult = 1;
        rnd = (u64) evt->mult - 1;

        /*
         * Upper bound sanity check. If the backwards conversion is
         * not equal latch, we know that the above shift overflowed.
         */
        if ((clc >> evt->shift) != (u64)latch)
                clc = ~0ULL;

        /*
         * Scaled math oddities:
         *
         * For mult <= (1 << shift) we can safely add mult - 1 to
         * prevent integer rounding loss. So the backwards conversion
         * from nsec to device ticks will be correct.
         *
         * For mult > (1 << shift), i.e. device frequency is > 1GHz we
         * need to be careful. Adding mult - 1 will result in a value
         * which when converted back to device ticks can be larger
         * than latch by up to (mult - 1) >> shift. For the min_delta
         * calculation we still want to apply this in order to stay
         * above the minimum device ticks limit. For the upper limit
         * we would end up with a latch value larger than the upper
         * limit of the device, so we omit the add to stay below the
         * device upper boundary.
         *
         * Also omit the add if it would overflow the u64 boundary.
         */
        if ((~0ULL - clc > rnd) &&
            (!ismax || evt->mult <= (1ULL << evt->shift)))
                clc += rnd;

        do_div(clc, evt->mult);

        /* Deltas less than 1usec are pointless noise */
        return clc > 1000 ? clc : 1000;
}

/**
 * clockevent_delta2ns - Convert a latch value (device ticks) to nanoseconds
 * @latch:        value to convert
 * @evt:        pointer to clock event device descriptor
 *
 * Math helper, returns latch value converted to nanoseconds (bound checked)
 */
u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt)
{
        return cev_delta2ns(latch, evt, false);
}
EXPORT_SYMBOL_GPL(clockevent_delta2ns);

static int __clockevents_switch_state(struct clock_event_device *dev,
                                      enum clock_event_state state)
{
        if (dev->features & CLOCK_EVT_FEAT_DUMMY)
                return 0;

        /* Transition with new state-specific callbacks */
        switch (state) {
        case CLOCK_EVT_STATE_DETACHED:
                /* The clockevent device is getting replaced. Shut it down. */

        case CLOCK_EVT_STATE_SHUTDOWN:
                if (dev->set_state_shutdown)
                        return dev->set_state_shutdown(dev);
                return 0;

        case CLOCK_EVT_STATE_PERIODIC:
                /* Core internal bug */
                if (!(dev->features & CLOCK_EVT_FEAT_PERIODIC))
                        return -ENOSYS;
                if (dev->set_state_periodic)
                        return dev->set_state_periodic(dev);
                return 0;

        case CLOCK_EVT_STATE_ONESHOT:
                /* Core internal bug */
                if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                        return -ENOSYS;
                if (dev->set_state_oneshot)
                        return dev->set_state_oneshot(dev);
                return 0;

        case CLOCK_EVT_STATE_ONESHOT_STOPPED:
                /* Core internal bug */
                if (WARN_ONCE(!clockevent_state_oneshot(dev),
                              "Current state: %d\n",
                              clockevent_get_state(dev)))
                        return -EINVAL;

                if (dev->set_state_oneshot_stopped)
                        return dev->set_state_oneshot_stopped(dev);
                else
                        return -ENOSYS;

        default:
                return -ENOSYS;
        }
}

/**
 * clockevents_switch_state - set the operating state of a clock event device
 * @dev:        device to modify
 * @state:        new state
 *
 * Must be called with interrupts disabled !
 */
void clockevents_switch_state(struct clock_event_device *dev,
                              enum clock_event_state state)
{
        if (clockevent_get_state(dev) != state) {
                if (__clockevents_switch_state(dev, state))
                        return;

                clockevent_set_state(dev, state);

                /*
                 * A nsec2cyc multiplicator of 0 is invalid and we'd crash
                 * on it, so fix it up and emit a warning:
                 */
                if (clockevent_state_oneshot(dev)) {
                        if (WARN_ON(!dev->mult))
                                dev->mult = 1;
                }
        }
}

/**
 * clockevents_shutdown - shutdown the device and clear next_event
 * @dev:        device to shutdown
 */
void clockevents_shutdown(struct clock_event_device *dev)
{
        clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
        dev->next_event = KTIME_MAX;
}

/**
 * clockevents_tick_resume -        Resume the tick device before using it again
 * @dev:                        device to resume
 */
int clockevents_tick_resume(struct clock_event_device *dev)
{
        int ret = 0;

        if (dev->tick_resume)
                ret = dev->tick_resume(dev);

        return ret;
}

#ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST

/* Limit min_delta to a jiffy */
#define MIN_DELTA_LIMIT                (NSEC_PER_SEC / HZ)

/**
 * clockevents_increase_min_delta - raise minimum delta of a clock event device
 * @dev:       device to increase the minimum delta
 *
 * Returns 0 on success, -ETIME when the minimum delta reached the limit.
 */
static int clockevents_increase_min_delta(struct clock_event_device *dev)
{
        /* Nothing to do if we already reached the limit */
        if (dev->min_delta_ns >= MIN_DELTA_LIMIT) {
                printk_deferred(KERN_WARNING
                                "CE: Reprogramming failure. Giving up\n");
                dev->next_event = KTIME_MAX;
                return -ETIME;
        }

        if (dev->min_delta_ns < 5000)
                dev->min_delta_ns = 5000;
        else
                dev->min_delta_ns += dev->min_delta_ns >> 1;

        if (dev->min_delta_ns > MIN_DELTA_LIMIT)
                dev->min_delta_ns = MIN_DELTA_LIMIT;

        printk_deferred(KERN_WARNING
                        "CE: %s increased min_delta_ns to %llu nsec\n",
                        dev->name ? dev->name : "?",
                        (unsigned long long) dev->min_delta_ns);
        return 0;
}

/**
 * clockevents_program_min_delta - Set clock event device to the minimum delay.
 * @dev:        device to program
 *
 * Returns 0 on success, -ETIME when the retry loop failed.
 */
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
        unsigned long long clc;
        int64_t delta;
        int i;

        for (i = 0;;) {
                delta = dev->min_delta_ns;
                dev->next_event = ktime_add_ns(ktime_get(), delta);

                if (clockevent_state_shutdown(dev))
                        return 0;

                dev->retries++;
                clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
                if (dev->set_next_event((unsigned long) clc, dev) == 0)
                        return 0;

                if (++i > 2) {
                        /*
                         * We tried 3 times to program the device with the
                         * given min_delta_ns. Try to increase the minimum
                         * delta, if that fails as well get out of here.
                         */
                        if (clockevents_increase_min_delta(dev))
                                return -ETIME;
                        i = 0;
                }
        }
}

#else  /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */

/**
 * clockevents_program_min_delta - Set clock event device to the minimum delay.
 * @dev:        device to program
 *
 * Returns 0 on success, -ETIME when the retry loop failed.
 */
static int clockevents_program_min_delta(struct clock_event_device *dev)
{
        unsigned long long clc;
        int64_t delta = 0;
        int i;

        for (i = 0; i < 10; i++) {
                delta += dev->min_delta_ns;
                dev->next_event = ktime_add_ns(ktime_get(), delta);

                if (clockevent_state_shutdown(dev))
                        return 0;

                dev->retries++;
                clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
                if (dev->set_next_event((unsigned long) clc, dev) == 0)
                        return 0;
        }
        return -ETIME;
}

#endif /* CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST */

/**
 * clockevents_program_event - Reprogram the clock event device.
 * @dev:        device to program
 * @expires:        absolute expiry time (monotonic clock)
 * @force:        program minimum delay if expires can not be set
 *
 * Returns 0 on success, -ETIME when the event is in the past.
 */
int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
                              bool force)
{
        unsigned long long clc;
        int64_t delta;
        int rc;

        if (WARN_ON_ONCE(expires < 0))
                return -ETIME;

        dev->next_event = expires;

        if (clockevent_state_shutdown(dev))
                return 0;

        /* We must be in ONESHOT state here */
        WARN_ONCE(!clockevent_state_oneshot(dev), "Current state: %d\n",
                  clockevent_get_state(dev));

        /* Shortcut for clockevent devices that can deal with ktime. */
        if (dev->features & CLOCK_EVT_FEAT_KTIME)
                return dev->set_next_ktime(expires, dev);

        delta = ktime_to_ns(ktime_sub(expires, ktime_get()));
        if (delta <= 0)
                return force ? clockevents_program_min_delta(dev) : -ETIME;

        delta = min(delta, (int64_t) dev->max_delta_ns);
        delta = max(delta, (int64_t) dev->min_delta_ns);

        clc = ((unsigned long long) delta * dev->mult) >> dev->shift;
        rc = dev->set_next_event((unsigned long) clc, dev);

        return (rc && force) ? clockevents_program_min_delta(dev) : rc;
}

/*
 * Called after a clockevent has been added which might
 * have replaced a current regular or broadcast device. A
 * released normal device might be a suitable replacement
 * for the current broadcast device. Similarly a released
 * broadcast device might be a suitable replacement for a
 * normal device.
 */
static void clockevents_notify_released(void)
{
        struct clock_event_device *dev;

        /*
         * Keep iterating as long as tick_check_new_device()
         * replaces a device.
         */
        while (!list_empty(&clockevents_released)) {
                dev = list_entry(clockevents_released.next,
                                 struct clock_event_device, list);
                list_move(&dev->list, &clockevent_devices);
                tick_check_new_device(dev);
        }
}

/*
 * Try to install a replacement clock event device
 */
static int clockevents_replace(struct clock_event_device *ced)
{
        struct clock_event_device *dev, *newdev = NULL;

        list_for_each_entry(dev, &clockevent_devices, list) {
                if (dev == ced || !clockevent_state_detached(dev))
                        continue;

                if (!tick_check_replacement(newdev, dev))
                        continue;

                if (!try_module_get(dev->owner))
                        continue;

                if (newdev)
                        module_put(newdev->owner);
                newdev = dev;
        }
        if (newdev) {
                tick_install_replacement(newdev);
                list_del_init(&ced->list);
        }
        return newdev ? 0 : -EBUSY;
}

/*
 * Called with clockevents_mutex and clockevents_lock held
 */
static int __clockevents_try_unbind(struct clock_event_device *ced, int cpu)
{
        /* Fast track. Device is unused */
        if (clockevent_state_detached(ced)) {
                list_del_init(&ced->list);
                return 0;
        }

        return ced == per_cpu(tick_cpu_device, cpu).evtdev ? -EAGAIN : -EBUSY;
}

/*
 * SMP function call to unbind a device
 */
static void __clockevents_unbind(void *arg)
{
        struct ce_unbind *cu = arg;
        int res;

        raw_spin_lock(&clockevents_lock);
        res = __clockevents_try_unbind(cu->ce, smp_processor_id());
        if (res == -EAGAIN)
                res = clockevents_replace(cu->ce);
        cu->res = res;
        raw_spin_unlock(&clockevents_lock);
}

/*
 * Issues smp function call to unbind a per cpu device. Called with
 * clockevents_mutex held.
 */
static int clockevents_unbind(struct clock_event_device *ced, int cpu)
{
        struct ce_unbind cu = { .ce = ced, .res = -ENODEV };

        smp_call_function_single(cpu, __clockevents_unbind, &cu, 1);
        return cu.res;
}

/*
 * Unbind a clockevents device.
 */
int clockevents_unbind_device(struct clock_event_device *ced, int cpu)
{
        int ret;

        mutex_lock(&clockevents_mutex);
        ret = clockevents_unbind(ced, cpu);
        mutex_unlock(&clockevents_mutex);
        return ret;
}
EXPORT_SYMBOL_GPL(clockevents_unbind_device);

/**
 * clockevents_register_device - register a clock event device
 * @dev:        device to register
 */
void clockevents_register_device(struct clock_event_device *dev)
{
        unsigned long flags;

        /* Initialize state to DETACHED */
        clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);

        if (!dev->cpumask) {
                WARN_ON(num_possible_cpus() > 1);
                dev->cpumask = cpumask_of(smp_processor_id());
        }

        if (dev->cpumask == cpu_all_mask) {
                WARN(1, "%s cpumask == cpu_all_mask, using cpu_possible_mask instead\n",
                     dev->name);
                dev->cpumask = cpu_possible_mask;
        }

        raw_spin_lock_irqsave(&clockevents_lock, flags);

        list_add(&dev->list, &clockevent_devices);
        tick_check_new_device(dev);
        clockevents_notify_released();

        raw_spin_unlock_irqrestore(&clockevents_lock, flags);
}
EXPORT_SYMBOL_GPL(clockevents_register_device);

static void clockevents_config(struct clock_event_device *dev, u32 freq)
{
        u64 sec;

        if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                return;

        /*
         * Calculate the maximum number of seconds we can sleep. Limit
         * to 10 minutes for hardware which can program more than
         * 32bit ticks so we still get reasonable conversion values.
         */
        sec = dev->max_delta_ticks;
        do_div(sec, freq);
        if (!sec)
                sec = 1;
        else if (sec > 600 && dev->max_delta_ticks > UINT_MAX)
                sec = 600;

        clockevents_calc_mult_shift(dev, freq, sec);
        dev->min_delta_ns = cev_delta2ns(dev->min_delta_ticks, dev, false);
        dev->max_delta_ns = cev_delta2ns(dev->max_delta_ticks, dev, true);
}

/**
 * clockevents_config_and_register - Configure and register a clock event device
 * @dev:        device to register
 * @freq:        The clock frequency
 * @min_delta:        The minimum clock ticks to program in oneshot mode
 * @max_delta:        The maximum clock ticks to program in oneshot mode
 *
 * min/max_delta can be 0 for devices which do not support oneshot mode.
 */
void clockevents_config_and_register(struct clock_event_device *dev,
                                     u32 freq, unsigned long min_delta,
                                     unsigned long max_delta)
{
        dev->min_delta_ticks = min_delta;
        dev->max_delta_ticks = max_delta;
        clockevents_config(dev, freq);
        clockevents_register_device(dev);
}
EXPORT_SYMBOL_GPL(clockevents_config_and_register);

int __clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
        clockevents_config(dev, freq);

        if (clockevent_state_oneshot(dev))
                return clockevents_program_event(dev, dev->next_event, false);

        if (clockevent_state_periodic(dev))
                return __clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);

        return 0;
}

/**
 * clockevents_update_freq - Update frequency and reprogram a clock event device.
 * @dev:        device to modify
 * @freq:        new device frequency
 *
 * Reconfigure and reprogram a clock event device in oneshot
 * mode. Must be called on the cpu for which the device delivers per
 * cpu timer events. If called for the broadcast device the core takes
 * care of serialization.
 *
 * Returns 0 on success, -ETIME when the event is in the past.
 */
int clockevents_update_freq(struct clock_event_device *dev, u32 freq)
{
        unsigned long flags;
        int ret;

        local_irq_save(flags);
        ret = tick_broadcast_update_freq(dev, freq);
        if (ret == -ENODEV)
                ret = __clockevents_update_freq(dev, freq);
        local_irq_restore(flags);
        return ret;
}

/*
 * Noop handler when we shut down an event device
 */
void clockevents_handle_noop(struct clock_event_device *dev)
{
}

/**
 * clockevents_exchange_device - release and request clock devices
 * @old:        device to release (can be NULL)
 * @new:        device to request (can be NULL)
 *
 * Called from various tick functions with clockevents_lock held and
 * interrupts disabled.
 */
void clockevents_exchange_device(struct clock_event_device *old,
                                 struct clock_event_device *new)
{
        /*
         * Caller releases a clock event device. We queue it into the
         * released list and do a notify add later.
         */
        if (old) {
                module_put(old->owner);
                clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
                list_move(&old->list, &clockevents_released);
        }

        if (new) {
                BUG_ON(!clockevent_state_detached(new));
                clockevents_shutdown(new);
        }
}

/**
 * clockevents_suspend - suspend clock devices
 */
void clockevents_suspend(void)
{
        struct clock_event_device *dev;

        list_for_each_entry_reverse(dev, &clockevent_devices, list)
                if (dev->suspend && !clockevent_state_detached(dev))
                        dev->suspend(dev);
}

/**
 * clockevents_resume - resume clock devices
 */
void clockevents_resume(void)
{
        struct clock_event_device *dev;

        list_for_each_entry(dev, &clockevent_devices, list)
                if (dev->resume && !clockevent_state_detached(dev))
                        dev->resume(dev);
}

#ifdef CONFIG_HOTPLUG_CPU

/**
 * tick_offline_cpu - Shutdown all clock events related
 *                    to this CPU and take it out of the
 *                    broadcast mechanism.
 * @cpu:        The outgoing CPU
 *
 * Called by the dying CPU during teardown.
 */
void tick_offline_cpu(unsigned int cpu)
{
        struct clock_event_device *dev, *tmp;

        raw_spin_lock(&clockevents_lock);

        tick_broadcast_offline(cpu);
        tick_shutdown();

        /*
         * Unregister the clock event devices which were
         * released above.
         */
        list_for_each_entry_safe(dev, tmp, &clockevents_released, list)
                list_del(&dev->list);

        /*
         * Now check whether the CPU has left unused per cpu devices
         */
        list_for_each_entry_safe(dev, tmp, &clockevent_devices, list) {
                if (cpumask_test_cpu(cpu, dev->cpumask) &&
                    cpumask_weight(dev->cpumask) == 1 &&
                    !tick_is_broadcast_device(dev)) {
                        BUG_ON(!clockevent_state_detached(dev));
                        list_del(&dev->list);
                }
        }

        raw_spin_unlock(&clockevents_lock);
}
#endif

#ifdef CONFIG_SYSFS
static const struct bus_type clockevents_subsys = {
        .name                = "clockevents",
        .dev_name       = "clockevent",
};

static DEFINE_PER_CPU(struct device, tick_percpu_dev);
static struct tick_device *tick_get_tick_dev(struct device *dev);

static ssize_t current_device_show(struct device *dev,
                                   struct device_attribute *attr,
                                   char *buf)
{
        struct tick_device *td;
        ssize_t count = 0;

        raw_spin_lock_irq(&clockevents_lock);
        td = tick_get_tick_dev(dev);
        if (td && td->evtdev)
                count = sysfs_emit(buf, "%s\n", td->evtdev->name);
        raw_spin_unlock_irq(&clockevents_lock);
        return count;
}
static DEVICE_ATTR_RO(current_device);

/* We don't support the abomination of removable broadcast devices */
static ssize_t unbind_device_store(struct device *dev,
                                   struct device_attribute *attr,
                                   const char *buf, size_t count)
{
        char name[CS_NAME_LEN];
        ssize_t ret = sysfs_get_uname(buf, name, count);
        struct clock_event_device *ce = NULL, *iter;

        if (ret < 0)
                return ret;

        ret = -ENODEV;
        mutex_lock(&clockevents_mutex);
        raw_spin_lock_irq(&clockevents_lock);
        list_for_each_entry(iter, &clockevent_devices, list) {
                if (!strcmp(iter->name, name)) {
                        ret = __clockevents_try_unbind(iter, dev->id);
                        ce = iter;
                        break;
                }
        }
        raw_spin_unlock_irq(&clockevents_lock);
        /*
         * We hold clockevents_mutex, so ce can't go away
         */
        if (ret == -EAGAIN)
                ret = clockevents_unbind(ce, dev->id);
        mutex_unlock(&clockevents_mutex);
        return ret ? ret : count;
}
static DEVICE_ATTR_WO(unbind_device);

#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
static struct device tick_bc_dev = {
        .init_name        = "broadcast",
        .id                = 0,
        .bus                = &clockevents_subsys,
};

static struct tick_device *tick_get_tick_dev(struct device *dev)
{
        return dev == &tick_bc_dev ? tick_get_broadcast_device() :
                &per_cpu(tick_cpu_device, dev->id);
}

static __init int tick_broadcast_init_sysfs(void)
{
        int err = device_register(&tick_bc_dev);

        if (!err)
                err = device_create_file(&tick_bc_dev, &dev_attr_current_device);
        return err;
}
#else
static struct tick_device *tick_get_tick_dev(struct device *dev)
{
        return &per_cpu(tick_cpu_device, dev->id);
}
static inline int tick_broadcast_init_sysfs(void) { return 0; }
#endif

static int __init tick_init_sysfs(void)
{
        int cpu;

        for_each_possible_cpu(cpu) {
                struct device *dev = &per_cpu(tick_percpu_dev, cpu);
                int err;

                dev->id = cpu;
                dev->bus = &clockevents_subsys;
                err = device_register(dev);
                if (!err)
                        err = device_create_file(dev, &dev_attr_current_device);
                if (!err)
                        err = device_create_file(dev, &dev_attr_unbind_device);
                if (err)
                        return err;
        }
        return tick_broadcast_init_sysfs();
}

static int __init clockevents_init_sysfs(void)
{
        int err = subsys_system_register(&clockevents_subsys, NULL);

        if (!err)
                err = tick_init_sysfs();
        return err;
}
device_initcall(clockevents_init_sysfs);
#endif /* SYSFS */

































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   24 


   25 





































   25 


   25 


































































































   25 




   25 


























   25 

















   25 











   25 



   25 























   25 

   25 






























































    4 


    4 







































































    4 




    4 


















    4 




















    4 





    4 
























































































  268 




  269 























































































































































































































































































































































































































































































































































































































































































































































































   46 







   89 







































































































































  270 

















   18 
  257 
    8 

  256 







  689 




  689 





  622 


   91 
  558 











  771 


  576 

    2 








  757 


    2 

  752 






















  753 
    2 

    2 








  757 




  738 











   32 
    5 
   38 

   39 





  414 
  375 


  754 





  757 

    2 





   99 

  716 






  757 

    5 




























































































































    1 






    5 



















































  607 






  603 



  312 

  317 







  608 





  604 





  604 





  607 



























  271 



  270 




  271 




  146 
  135 

    2 
  269 

































































  613 



  617 


  617 
















   11 

  301 

















































   63 



   64 



   44 



   45 























  617 







  620 


















  576 














   28 






  763 


























   61 










  605 













  576 
   56 
   30 










   68 



   61 

  592 





   61 




























    2 
































  259 

  262 








  270 



   18 




















































  262 
   28 





  605 




  606 














  576 


   63 




  606 
  604 















  270 


  268 





  248 
   44 

  266 









  606 




















   21 


   21 












   10 












  605 


































  604 












  605 









  604 




  608 









  606 

  605 






  608 


  576 
   64 






























  602 








   20 




















  605 











  608 


  577 
   64 









  604 


























































    4 














  812 
  816 








  472 

  249 
  475 












  271 



  268 


















  270 







  271 


  270 









  246 
   45 
  266 

   10 
   10 



    2 




  270 







  269 





  271 


















  270 















   15 
   23 

   15 






















   25 






















   38 





   23 
   15 
   15 



   25 
   14 
    9 



   29 























   21 








   21 


























































































    7 







    7 



























































































































































































































   10 
   10 


































   35 
































   33 




   34 



































    1 




    1 















    9 





    9 








































   30 










   31 

   30 























   31 






























































































































































































































































































































   62 

   29 
















   12 










    2 


   78 





   91 


   92 






   89 















   45 


   46 






   22 


































    9 
   14 

    2 



































































































































































































































































































































   11 
   11 






















    9 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/base/core.c - core driver model code (device registration, etc)
 *
 * Copyright (c) 2002-3 Patrick Mochel
 * Copyright (c) 2002-3 Open Source Development Labs
 * Copyright (c) 2006 Greg Kroah-Hartman <gregkh@suse.de>
 * Copyright (c) 2006 Novell, Inc.
 */

#include <linux/acpi.h>
#include <linux/blkdev.h>
#include <linux/cleanup.h>
#include <linux/cpufreq.h>
#include <linux/device.h>
#include <linux/dma-map-ops.h> /* for dma_default_coherent */
#include <linux/err.h>
#include <linux/fwnode.h>
#include <linux/init.h>
#include <linux/kdev_t.h>
#include <linux/kstrtox.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/netdevice.h>
#include <linux/notifier.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/pm_runtime.h>
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/string_helpers.h>
#include <linux/swiotlb.h>
#include <linux/sysfs.h>

#include "base.h"
#include "physical_location.h"
#include "power/power.h"

/* Device links support. */
static LIST_HEAD(deferred_sync);
static unsigned int defer_sync_state_count = 1;
static DEFINE_MUTEX(fwnode_link_lock);
static bool fw_devlink_is_permissive(void);
static void __fw_devlink_link_to_consumers(struct device *dev);
static bool fw_devlink_drv_reg_done;
static bool fw_devlink_best_effort;
static struct workqueue_struct *device_link_wq;

/**
 * __fwnode_link_add - Create a link between two fwnode_handles.
 * @con: Consumer end of the link.
 * @sup: Supplier end of the link.
 * @flags: Link flags.
 *
 * Create a fwnode link between fwnode handles @con and @sup. The fwnode link
 * represents the detail that the firmware lists @sup fwnode as supplying a
 * resource to @con.
 *
 * The driver core will use the fwnode link to create a device link between the
 * two device objects corresponding to @con and @sup when they are created. The
 * driver core will automatically delete the fwnode link between @con and @sup
 * after doing that.
 *
 * Attempts to create duplicate links between the same pair of fwnode handles
 * are ignored and there is no reference counting.
 */
static int __fwnode_link_add(struct fwnode_handle *con,
                             struct fwnode_handle *sup, u8 flags)
{
        struct fwnode_link *link;

        list_for_each_entry(link, &sup->consumers, s_hook)
                if (link->consumer == con) {
                        link->flags |= flags;
                        return 0;
                }

        link = kzalloc(sizeof(*link), GFP_KERNEL);
        if (!link)
                return -ENOMEM;

        link->supplier = sup;
        INIT_LIST_HEAD(&link->s_hook);
        link->consumer = con;
        INIT_LIST_HEAD(&link->c_hook);
        link->flags = flags;

        list_add(&link->s_hook, &sup->consumers);
        list_add(&link->c_hook, &con->suppliers);
        pr_debug("%pfwf Linked as a fwnode consumer to %pfwf\n",
                 con, sup);

        return 0;
}

int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup,
                    u8 flags)
{
        guard(mutex)(&fwnode_link_lock);

        return __fwnode_link_add(con, sup, flags);
}

/**
 * __fwnode_link_del - Delete a link between two fwnode_handles.
 * @link: the fwnode_link to be deleted
 *
 * The fwnode_link_lock needs to be held when this function is called.
 */
static void __fwnode_link_del(struct fwnode_link *link)
{
        pr_debug("%pfwf Dropping the fwnode link to %pfwf\n",
                 link->consumer, link->supplier);
        list_del(&link->s_hook);
        list_del(&link->c_hook);
        kfree(link);
}

/**
 * __fwnode_link_cycle - Mark a fwnode link as being part of a cycle.
 * @link: the fwnode_link to be marked
 *
 * The fwnode_link_lock needs to be held when this function is called.
 */
static void __fwnode_link_cycle(struct fwnode_link *link)
{
        pr_debug("%pfwf: cycle: depends on %pfwf\n",
                 link->consumer, link->supplier);
        link->flags |= FWLINK_FLAG_CYCLE;
}

/**
 * fwnode_links_purge_suppliers - Delete all supplier links of fwnode_handle.
 * @fwnode: fwnode whose supplier links need to be deleted
 *
 * Deletes all supplier links connecting directly to @fwnode.
 */
static void fwnode_links_purge_suppliers(struct fwnode_handle *fwnode)
{
        struct fwnode_link *link, *tmp;

        guard(mutex)(&fwnode_link_lock);

        list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook)
                __fwnode_link_del(link);
}

/**
 * fwnode_links_purge_consumers - Delete all consumer links of fwnode_handle.
 * @fwnode: fwnode whose consumer links need to be deleted
 *
 * Deletes all consumer links connecting directly to @fwnode.
 */
static void fwnode_links_purge_consumers(struct fwnode_handle *fwnode)
{
        struct fwnode_link *link, *tmp;

        guard(mutex)(&fwnode_link_lock);

        list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook)
                __fwnode_link_del(link);
}

/**
 * fwnode_links_purge - Delete all links connected to a fwnode_handle.
 * @fwnode: fwnode whose links needs to be deleted
 *
 * Deletes all links connecting directly to a fwnode.
 */
void fwnode_links_purge(struct fwnode_handle *fwnode)
{
        fwnode_links_purge_suppliers(fwnode);
        fwnode_links_purge_consumers(fwnode);
}

void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode)
{
        struct fwnode_handle *child;

        /* Don't purge consumer links of an added child */
        if (fwnode->dev)
                return;

        fwnode->flags |= FWNODE_FLAG_NOT_DEVICE;
        fwnode_links_purge_consumers(fwnode);

        fwnode_for_each_available_child_node(fwnode, child)
                fw_devlink_purge_absent_suppliers(child);
}
EXPORT_SYMBOL_GPL(fw_devlink_purge_absent_suppliers);

/**
 * __fwnode_links_move_consumers - Move consumer from @from to @to fwnode_handle
 * @from: move consumers away from this fwnode
 * @to: move consumers to this fwnode
 *
 * Move all consumer links from @from fwnode to @to fwnode.
 */
static void __fwnode_links_move_consumers(struct fwnode_handle *from,
                                          struct fwnode_handle *to)
{
        struct fwnode_link *link, *tmp;

        list_for_each_entry_safe(link, tmp, &from->consumers, s_hook) {
                __fwnode_link_add(link->consumer, to, link->flags);
                __fwnode_link_del(link);
        }
}

/**
 * __fw_devlink_pickup_dangling_consumers - Pick up dangling consumers
 * @fwnode: fwnode from which to pick up dangling consumers
 * @new_sup: fwnode of new supplier
 *
 * If the @fwnode has a corresponding struct device and the device supports
 * probing (that is, added to a bus), then we want to let fw_devlink create
 * MANAGED device links to this device, so leave @fwnode and its descendant's
 * fwnode links alone.
 *
 * Otherwise, move its consumers to the new supplier @new_sup.
 */
static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode,
                                                   struct fwnode_handle *new_sup)
{
        struct fwnode_handle *child;

        if (fwnode->dev && fwnode->dev->bus)
                return;

        fwnode->flags |= FWNODE_FLAG_NOT_DEVICE;
        __fwnode_links_move_consumers(fwnode, new_sup);

        fwnode_for_each_available_child_node(fwnode, child)
                __fw_devlink_pickup_dangling_consumers(child, new_sup);
}

static DEFINE_MUTEX(device_links_lock);
DEFINE_STATIC_SRCU(device_links_srcu);

static inline void device_links_write_lock(void)
{
        mutex_lock(&device_links_lock);
}

static inline void device_links_write_unlock(void)
{
        mutex_unlock(&device_links_lock);
}

int device_links_read_lock(void) __acquires(&device_links_srcu)
{
        return srcu_read_lock(&device_links_srcu);
}

void device_links_read_unlock(int idx) __releases(&device_links_srcu)
{
        srcu_read_unlock(&device_links_srcu, idx);
}

int device_links_read_lock_held(void)
{
        return srcu_read_lock_held(&device_links_srcu);
}

static void device_link_synchronize_removal(void)
{
        synchronize_srcu(&device_links_srcu);
}

static void device_link_remove_from_lists(struct device_link *link)
{
        list_del_rcu(&link->s_node);
        list_del_rcu(&link->c_node);
}

static bool device_is_ancestor(struct device *dev, struct device *target)
{
        while (target->parent) {
                target = target->parent;
                if (dev == target)
                        return true;
        }
        return false;
}

#define DL_MARKER_FLAGS                (DL_FLAG_INFERRED | \
                                 DL_FLAG_CYCLE | \
                                 DL_FLAG_MANAGED)
bool device_link_flag_is_sync_state_only(u32 flags)
{
        return (flags & ~DL_MARKER_FLAGS) == DL_FLAG_SYNC_STATE_ONLY;
}

/**
 * device_is_dependent - Check if one device depends on another one
 * @dev: Device to check dependencies for.
 * @target: Device to check against.
 *
 * Check if @target depends on @dev or any device dependent on it (its child or
 * its consumer etc).  Return 1 if that is the case or 0 otherwise.
 */
static int device_is_dependent(struct device *dev, void *target)
{
        struct device_link *link;
        int ret;

        /*
         * The "ancestors" check is needed to catch the case when the target
         * device has not been completely initialized yet and it is still
         * missing from the list of children of its parent device.
         */
        if (dev == target || device_is_ancestor(dev, target))
                return 1;

        ret = device_for_each_child(dev, target, device_is_dependent);
        if (ret)
                return ret;

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (device_link_flag_is_sync_state_only(link->flags))
                        continue;

                if (link->consumer == target)
                        return 1;

                ret = device_is_dependent(link->consumer, target);
                if (ret)
                        break;
        }
        return ret;
}

static void device_link_init_status(struct device_link *link,
                                    struct device *consumer,
                                    struct device *supplier)
{
        switch (supplier->links.status) {
        case DL_DEV_PROBING:
                switch (consumer->links.status) {
                case DL_DEV_PROBING:
                        /*
                         * A consumer driver can create a link to a supplier
                         * that has not completed its probing yet as long as it
                         * knows that the supplier is already functional (for
                         * example, it has just acquired some resources from the
                         * supplier).
                         */
                        link->status = DL_STATE_CONSUMER_PROBE;
                        break;
                default:
                        link->status = DL_STATE_DORMANT;
                        break;
                }
                break;
        case DL_DEV_DRIVER_BOUND:
                switch (consumer->links.status) {
                case DL_DEV_PROBING:
                        link->status = DL_STATE_CONSUMER_PROBE;
                        break;
                case DL_DEV_DRIVER_BOUND:
                        link->status = DL_STATE_ACTIVE;
                        break;
                default:
                        link->status = DL_STATE_AVAILABLE;
                        break;
                }
                break;
        case DL_DEV_UNBINDING:
                link->status = DL_STATE_SUPPLIER_UNBIND;
                break;
        default:
                link->status = DL_STATE_DORMANT;
                break;
        }
}

static int device_reorder_to_tail(struct device *dev, void *not_used)
{
        struct device_link *link;

        /*
         * Devices that have not been registered yet will be put to the ends
         * of the lists during the registration, so skip them here.
         */
        if (device_is_registered(dev))
                devices_kset_move_last(dev);

        if (device_pm_initialized(dev))
                device_pm_move_last(dev);

        device_for_each_child(dev, NULL, device_reorder_to_tail);
        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (device_link_flag_is_sync_state_only(link->flags))
                        continue;
                device_reorder_to_tail(link->consumer, NULL);
        }

        return 0;
}

/**
 * device_pm_move_to_tail - Move set of devices to the end of device lists
 * @dev: Device to move
 *
 * This is a device_reorder_to_tail() wrapper taking the requisite locks.
 *
 * It moves the @dev along with all of its children and all of its consumers
 * to the ends of the device_kset and dpm_list, recursively.
 */
void device_pm_move_to_tail(struct device *dev)
{
        int idx;

        idx = device_links_read_lock();
        device_pm_lock();
        device_reorder_to_tail(dev, NULL);
        device_pm_unlock();
        device_links_read_unlock(idx);
}

#define to_devlink(dev)        container_of((dev), struct device_link, link_dev)

static ssize_t status_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        const char *output;

        switch (to_devlink(dev)->status) {
        case DL_STATE_NONE:
                output = "not tracked";
                break;
        case DL_STATE_DORMANT:
                output = "dormant";
                break;
        case DL_STATE_AVAILABLE:
                output = "available";
                break;
        case DL_STATE_CONSUMER_PROBE:
                output = "consumer probing";
                break;
        case DL_STATE_ACTIVE:
                output = "active";
                break;
        case DL_STATE_SUPPLIER_UNBIND:
                output = "supplier unbinding";
                break;
        default:
                output = "unknown";
                break;
        }

        return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(status);

static ssize_t auto_remove_on_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        struct device_link *link = to_devlink(dev);
        const char *output;

        if (device_link_test(link, DL_FLAG_AUTOREMOVE_SUPPLIER))
                output = "supplier unbind";
        else if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER))
                output = "consumer unbind";
        else
                output = "never";

        return sysfs_emit(buf, "%s\n", output);
}
static DEVICE_ATTR_RO(auto_remove_on);

static ssize_t runtime_pm_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct device_link *link = to_devlink(dev);

        return sysfs_emit(buf, "%d\n", device_link_test(link, DL_FLAG_PM_RUNTIME));
}
static DEVICE_ATTR_RO(runtime_pm);

static ssize_t sync_state_only_show(struct device *dev,
                                    struct device_attribute *attr, char *buf)
{
        struct device_link *link = to_devlink(dev);

        return sysfs_emit(buf, "%d\n", device_link_test(link, DL_FLAG_SYNC_STATE_ONLY));
}
static DEVICE_ATTR_RO(sync_state_only);

static struct attribute *devlink_attrs[] = {
        &dev_attr_status.attr,
        &dev_attr_auto_remove_on.attr,
        &dev_attr_runtime_pm.attr,
        &dev_attr_sync_state_only.attr,
        NULL,
};
ATTRIBUTE_GROUPS(devlink);

static void device_link_release_fn(struct work_struct *work)
{
        struct device_link *link = container_of(work, struct device_link, rm_work);

        /* Ensure that all references to the link object have been dropped. */
        device_link_synchronize_removal();

        pm_runtime_release_supplier(link);
        /*
         * If supplier_preactivated is set, the link has been dropped between
         * the pm_runtime_get_suppliers() and pm_runtime_put_suppliers() calls
         * in __driver_probe_device().  In that case, drop the supplier's
         * PM-runtime usage counter to remove the reference taken by
         * pm_runtime_get_suppliers().
         */
        if (link->supplier_preactivated)
                pm_runtime_put_noidle(link->supplier);

        pm_request_idle(link->supplier);

        put_device(link->consumer);
        put_device(link->supplier);
        kfree(link);
}

static void devlink_dev_release(struct device *dev)
{
        struct device_link *link = to_devlink(dev);

        INIT_WORK(&link->rm_work, device_link_release_fn);
        /*
         * It may take a while to complete this work because of the SRCU
         * synchronization in device_link_release_fn() and if the consumer or
         * supplier devices get deleted when it runs, so put it into the
         * dedicated workqueue.
         */
        queue_work(device_link_wq, &link->rm_work);
}

/**
 * device_link_wait_removal - Wait for ongoing devlink removal jobs to terminate
 */
void device_link_wait_removal(void)
{
        /*
         * devlink removal jobs are queued in the dedicated work queue.
         * To be sure that all removal jobs are terminated, ensure that any
         * scheduled work has run to completion.
         */
        flush_workqueue(device_link_wq);
}
EXPORT_SYMBOL_GPL(device_link_wait_removal);

static const struct class devlink_class = {
        .name = "devlink",
        .dev_groups = devlink_groups,
        .dev_release = devlink_dev_release,
};

static int devlink_add_symlinks(struct device *dev)
{
        char *buf_con __free(kfree) = NULL, *buf_sup __free(kfree) = NULL;
        int ret;
        struct device_link *link = to_devlink(dev);
        struct device *sup = link->supplier;
        struct device *con = link->consumer;

        ret = sysfs_create_link(&link->link_dev.kobj, &sup->kobj, "supplier");
        if (ret)
                goto out;

        ret = sysfs_create_link(&link->link_dev.kobj, &con->kobj, "consumer");
        if (ret)
                goto err_con;

        buf_con = kasprintf(GFP_KERNEL, "consumer:%s:%s", dev_bus_name(con), dev_name(con));
        if (!buf_con) {
                ret = -ENOMEM;
                goto err_con_dev;
        }

        ret = sysfs_create_link(&sup->kobj, &link->link_dev.kobj, buf_con);
        if (ret)
                goto err_con_dev;

        buf_sup = kasprintf(GFP_KERNEL, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup));
        if (!buf_sup) {
                ret = -ENOMEM;
                goto err_sup_dev;
        }

        ret = sysfs_create_link(&con->kobj, &link->link_dev.kobj, buf_sup);
        if (ret)
                goto err_sup_dev;

        goto out;

err_sup_dev:
        sysfs_remove_link(&sup->kobj, buf_con);
err_con_dev:
        sysfs_remove_link(&link->link_dev.kobj, "consumer");
err_con:
        sysfs_remove_link(&link->link_dev.kobj, "supplier");
out:
        return ret;
}

static void devlink_remove_symlinks(struct device *dev)
{
        char *buf_con __free(kfree) = NULL, *buf_sup __free(kfree) = NULL;
        struct device_link *link = to_devlink(dev);
        struct device *sup = link->supplier;
        struct device *con = link->consumer;

        sysfs_remove_link(&link->link_dev.kobj, "consumer");
        sysfs_remove_link(&link->link_dev.kobj, "supplier");

        if (device_is_registered(con)) {
                buf_sup = kasprintf(GFP_KERNEL, "supplier:%s:%s", dev_bus_name(sup), dev_name(sup));
                if (!buf_sup)
                        goto out;
                sysfs_remove_link(&con->kobj, buf_sup);
        }

        buf_con = kasprintf(GFP_KERNEL, "consumer:%s:%s", dev_bus_name(con), dev_name(con));
        if (!buf_con)
                goto out;
        sysfs_remove_link(&sup->kobj, buf_con);

        return;

out:
        WARN(1, "Unable to properly free device link symlinks!\n");
}

static struct class_interface devlink_class_intf = {
        .class = &devlink_class,
        .add_dev = devlink_add_symlinks,
        .remove_dev = devlink_remove_symlinks,
};

static int __init devlink_class_init(void)
{
        int ret;

        ret = class_register(&devlink_class);
        if (ret)
                return ret;

        ret = class_interface_register(&devlink_class_intf);
        if (ret)
                class_unregister(&devlink_class);

        return ret;
}
postcore_initcall(devlink_class_init);

#define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \
                               DL_FLAG_AUTOREMOVE_SUPPLIER | \
                               DL_FLAG_AUTOPROBE_CONSUMER  | \
                               DL_FLAG_SYNC_STATE_ONLY | \
                               DL_FLAG_INFERRED | \
                               DL_FLAG_CYCLE)

#define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \
                            DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)

/**
 * device_link_add - Create a link between two devices.
 * @consumer: Consumer end of the link.
 * @supplier: Supplier end of the link.
 * @flags: Link flags.
 *
 * Return: On success, a device_link struct will be returned.
 *         On error or invalid flag settings, NULL will be returned.
 *
 * The caller is responsible for the proper synchronization of the link creation
 * with runtime PM.  First, setting the DL_FLAG_PM_RUNTIME flag will cause the
 * runtime PM framework to take the link into account.  Second, if the
 * DL_FLAG_RPM_ACTIVE flag is set in addition to it, the supplier devices will
 * be forced into the active meta state and reference-counted upon the creation
 * of the link.  If DL_FLAG_PM_RUNTIME is not set, DL_FLAG_RPM_ACTIVE will be
 * ignored.
 *
 * If DL_FLAG_STATELESS is set in @flags, the caller of this function is
 * expected to release the link returned by it directly with the help of either
 * device_link_del() or device_link_remove().
 *
 * If that flag is not set, however, the caller of this function is handing the
 * management of the link over to the driver core entirely and its return value
 * can only be used to check whether or not the link is present.  In that case,
 * the DL_FLAG_AUTOREMOVE_CONSUMER and DL_FLAG_AUTOREMOVE_SUPPLIER device link
 * flags can be used to indicate to the driver core when the link can be safely
 * deleted.  Namely, setting one of them in @flags indicates to the driver core
 * that the link is not going to be used (by the given caller of this function)
 * after unbinding the consumer or supplier driver, respectively, from its
 * device, so the link can be deleted at that point.  If none of them is set,
 * the link will be maintained until one of the devices pointed to by it (either
 * the consumer or the supplier) is unregistered.
 *
 * Also, if DL_FLAG_STATELESS, DL_FLAG_AUTOREMOVE_CONSUMER and
 * DL_FLAG_AUTOREMOVE_SUPPLIER are not set in @flags (that is, a persistent
 * managed device link is being added), the DL_FLAG_AUTOPROBE_CONSUMER flag can
 * be used to request the driver core to automatically probe for a consumer
 * driver after successfully binding a driver to the supplier device.
 *
 * The combination of DL_FLAG_STATELESS and one of DL_FLAG_AUTOREMOVE_CONSUMER,
 * DL_FLAG_AUTOREMOVE_SUPPLIER, or DL_FLAG_AUTOPROBE_CONSUMER set in @flags at
 * the same time is invalid and will cause NULL to be returned upfront.
 * However, if a device link between the given @consumer and @supplier pair
 * exists already when this function is called for them, the existing link will
 * be returned regardless of its current type and status (the link's flags may
 * be modified then).  The caller of this function is then expected to treat
 * the link as though it has just been created, so (in particular) if
 * DL_FLAG_STATELESS was passed in @flags, the link needs to be released
 * explicitly when not needed any more (as stated above).
 *
 * A side effect of the link creation is re-ordering of dpm_list and the
 * devices_kset list by moving the consumer device and all devices depending
 * on it to the ends of these lists (that does not happen to devices that have
 * not been registered when this function is called).
 *
 * The supplier device is required to be registered when this function is called
 * and NULL will be returned if that is not the case.  The consumer device need
 * not be registered, however.
 */
struct device_link *device_link_add(struct device *consumer,
                                    struct device *supplier, u32 flags)
{
        struct device_link *link;

        if (!consumer || !supplier || consumer == supplier ||
            flags & ~DL_ADD_VALID_FLAGS ||
            (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) ||
            (flags & DL_FLAG_AUTOPROBE_CONSUMER &&
             flags & (DL_FLAG_AUTOREMOVE_CONSUMER |
                      DL_FLAG_AUTOREMOVE_SUPPLIER)))
                return NULL;

        if (flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) {
                if (pm_runtime_get_sync(supplier) < 0) {
                        pm_runtime_put_noidle(supplier);
                        return NULL;
                }
        }

        if (!(flags & DL_FLAG_STATELESS))
                flags |= DL_FLAG_MANAGED;

        if (flags & DL_FLAG_SYNC_STATE_ONLY &&
            !device_link_flag_is_sync_state_only(flags))
                return NULL;

        device_links_write_lock();
        device_pm_lock();

        /*
         * If the supplier has not been fully registered yet or there is a
         * reverse (non-SYNC_STATE_ONLY) dependency between the consumer and
         * the supplier already in the graph, return NULL. If the link is a
         * SYNC_STATE_ONLY link, we don't check for reverse dependencies
         * because it only affects sync_state() callbacks.
         */
        if (!device_pm_initialized(supplier)
            || (!(flags & DL_FLAG_SYNC_STATE_ONLY) &&
                  device_is_dependent(consumer, supplier))) {
                link = NULL;
                goto out;
        }

        /*
         * SYNC_STATE_ONLY links are useless once a consumer device has probed.
         * So, only create it if the consumer hasn't probed yet.
         */
        if (flags & DL_FLAG_SYNC_STATE_ONLY &&
            consumer->links.status != DL_DEV_NO_DRIVER &&
            consumer->links.status != DL_DEV_PROBING) {
                link = NULL;
                goto out;
        }

        /*
         * DL_FLAG_AUTOREMOVE_SUPPLIER indicates that the link will be needed
         * longer than for DL_FLAG_AUTOREMOVE_CONSUMER and setting them both
         * together doesn't make sense, so prefer DL_FLAG_AUTOREMOVE_SUPPLIER.
         */
        if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER)
                flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER;

        list_for_each_entry(link, &supplier->links.consumers, s_node) {
                if (link->consumer != consumer)
                        continue;

                if (device_link_test(link, DL_FLAG_INFERRED) &&
                    !(flags & DL_FLAG_INFERRED))
                        link->flags &= ~DL_FLAG_INFERRED;

                if (flags & DL_FLAG_PM_RUNTIME) {
                        if (!device_link_test(link, DL_FLAG_PM_RUNTIME)) {
                                pm_runtime_new_link(consumer);
                                link->flags |= DL_FLAG_PM_RUNTIME;
                        }
                        if (flags & DL_FLAG_RPM_ACTIVE)
                                refcount_inc(&link->rpm_active);
                }

                if (flags & DL_FLAG_STATELESS) {
                        kref_get(&link->kref);
                        if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY) &&
                            !device_link_test(link, DL_FLAG_STATELESS)) {
                                link->flags |= DL_FLAG_STATELESS;
                                goto reorder;
                        } else {
                                link->flags |= DL_FLAG_STATELESS;
                                goto out;
                        }
                }

                /*
                 * If the life time of the link following from the new flags is
                 * longer than indicated by the flags of the existing link,
                 * update the existing link to stay around longer.
                 */
                if (flags & DL_FLAG_AUTOREMOVE_SUPPLIER) {
                        if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)) {
                                link->flags &= ~DL_FLAG_AUTOREMOVE_CONSUMER;
                                link->flags |= DL_FLAG_AUTOREMOVE_SUPPLIER;
                        }
                } else if (!(flags & DL_FLAG_AUTOREMOVE_CONSUMER)) {
                        link->flags &= ~(DL_FLAG_AUTOREMOVE_CONSUMER |
                                         DL_FLAG_AUTOREMOVE_SUPPLIER);
                }
                if (!device_link_test(link, DL_FLAG_MANAGED)) {
                        kref_get(&link->kref);
                        link->flags |= DL_FLAG_MANAGED;
                        device_link_init_status(link, consumer, supplier);
                }
                if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY) &&
                    !(flags & DL_FLAG_SYNC_STATE_ONLY)) {
                        link->flags &= ~DL_FLAG_SYNC_STATE_ONLY;
                        goto reorder;
                }

                goto out;
        }

        link = kzalloc(sizeof(*link), GFP_KERNEL);
        if (!link)
                goto out;

        refcount_set(&link->rpm_active, 1);

        get_device(supplier);
        link->supplier = supplier;
        INIT_LIST_HEAD(&link->s_node);
        get_device(consumer);
        link->consumer = consumer;
        INIT_LIST_HEAD(&link->c_node);
        link->flags = flags;
        kref_init(&link->kref);

        link->link_dev.class = &devlink_class;
        device_set_pm_not_required(&link->link_dev);
        dev_set_name(&link->link_dev, "%s:%s--%s:%s",
                     dev_bus_name(supplier), dev_name(supplier),
                     dev_bus_name(consumer), dev_name(consumer));
        if (device_register(&link->link_dev)) {
                put_device(&link->link_dev);
                link = NULL;
                goto out;
        }

        if (flags & DL_FLAG_PM_RUNTIME) {
                if (flags & DL_FLAG_RPM_ACTIVE)
                        refcount_inc(&link->rpm_active);

                pm_runtime_new_link(consumer);
        }

        /* Determine the initial link state. */
        if (flags & DL_FLAG_STATELESS)
                link->status = DL_STATE_NONE;
        else
                device_link_init_status(link, consumer, supplier);

        /*
         * Some callers expect the link creation during consumer driver probe to
         * resume the supplier even without DL_FLAG_RPM_ACTIVE.
         */
        if (link->status == DL_STATE_CONSUMER_PROBE &&
            flags & DL_FLAG_PM_RUNTIME)
                pm_runtime_resume(supplier);

        list_add_tail_rcu(&link->s_node, &supplier->links.consumers);
        list_add_tail_rcu(&link->c_node, &consumer->links.suppliers);

        if (flags & DL_FLAG_SYNC_STATE_ONLY) {
                dev_dbg(consumer,
                        "Linked as a sync state only consumer to %s\n",
                        dev_name(supplier));
                goto out;
        }

reorder:
        /*
         * Move the consumer and all of the devices depending on it to the end
         * of dpm_list and the devices_kset list.
         *
         * It is necessary to hold dpm_list locked throughout all that or else
         * we may end up suspending with a wrong ordering of it.
         */
        device_reorder_to_tail(consumer, NULL);

        dev_dbg(consumer, "Linked as a consumer to %s\n", dev_name(supplier));

out:
        device_pm_unlock();
        device_links_write_unlock();

        if ((flags & DL_FLAG_PM_RUNTIME && flags & DL_FLAG_RPM_ACTIVE) && !link)
                pm_runtime_put(supplier);

        return link;
}
EXPORT_SYMBOL_GPL(device_link_add);

static void __device_link_del(struct kref *kref)
{
        struct device_link *link = container_of(kref, struct device_link, kref);

        dev_dbg(link->consumer, "Dropping the link to %s\n",
                dev_name(link->supplier));

        pm_runtime_drop_link(link);

        device_link_remove_from_lists(link);
        device_unregister(&link->link_dev);
}

static void device_link_put_kref(struct device_link *link)
{
        if (device_link_test(link, DL_FLAG_STATELESS))
                kref_put(&link->kref, __device_link_del);
        else if (!device_is_registered(link->consumer))
                __device_link_del(&link->kref);
        else
                WARN(1, "Unable to drop a managed device link reference\n");
}

/**
 * device_link_del - Delete a stateless link between two devices.
 * @link: Device link to delete.
 *
 * The caller must ensure proper synchronization of this function with runtime
 * PM.  If the link was added multiple times, it needs to be deleted as often.
 * Care is required for hotplugged devices:  Their links are purged on removal
 * and calling device_link_del() is then no longer allowed.
 */
void device_link_del(struct device_link *link)
{
        device_links_write_lock();
        device_link_put_kref(link);
        device_links_write_unlock();
}
EXPORT_SYMBOL_GPL(device_link_del);

/**
 * device_link_remove - Delete a stateless link between two devices.
 * @consumer: Consumer end of the link.
 * @supplier: Supplier end of the link.
 *
 * The caller must ensure proper synchronization of this function with runtime
 * PM.
 */
void device_link_remove(void *consumer, struct device *supplier)
{
        struct device_link *link;

        if (WARN_ON(consumer == supplier))
                return;

        device_links_write_lock();

        list_for_each_entry(link, &supplier->links.consumers, s_node) {
                if (link->consumer == consumer) {
                        device_link_put_kref(link);
                        break;
                }
        }

        device_links_write_unlock();
}
EXPORT_SYMBOL_GPL(device_link_remove);

static void device_links_missing_supplier(struct device *dev)
{
        struct device_link *link;

        list_for_each_entry(link, &dev->links.suppliers, c_node) {
                if (link->status != DL_STATE_CONSUMER_PROBE)
                        continue;

                if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) {
                        WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
                } else {
                        WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY));
                        WRITE_ONCE(link->status, DL_STATE_DORMANT);
                }
        }
}

static bool dev_is_best_effort(struct device *dev)
{
        return (fw_devlink_best_effort && dev->can_match) ||
                (dev->fwnode && (dev->fwnode->flags & FWNODE_FLAG_BEST_EFFORT));
}

static struct fwnode_handle *fwnode_links_check_suppliers(
                                                struct fwnode_handle *fwnode)
{
        struct fwnode_link *link;

        if (!fwnode || fw_devlink_is_permissive())
                return NULL;

        list_for_each_entry(link, &fwnode->suppliers, c_hook)
                if (!(link->flags &
                      (FWLINK_FLAG_CYCLE | FWLINK_FLAG_IGNORE)))
                        return link->supplier;

        return NULL;
}

/**
 * device_links_check_suppliers - Check presence of supplier drivers.
 * @dev: Consumer device.
 *
 * Check links from this device to any suppliers.  Walk the list of the device's
 * links to suppliers and see if all of them are available.  If not, simply
 * return -EPROBE_DEFER.
 *
 * We need to guarantee that the supplier will not go away after the check has
 * been positive here.  It only can go away in __device_release_driver() and
 * that function  checks the device's links to consumers.  This means we need to
 * mark the link as "consumer probe in progress" to make the supplier removal
 * wait for us to complete (or bad things may happen).
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
int device_links_check_suppliers(struct device *dev)
{
        struct device_link *link;
        int ret = 0, fwnode_ret = 0;
        struct fwnode_handle *sup_fw;

        /*
         * Device waiting for supplier to become available is not allowed to
         * probe.
         */
        scoped_guard(mutex, &fwnode_link_lock) {
                sup_fw = fwnode_links_check_suppliers(dev->fwnode);
                if (sup_fw) {
                        if (dev_is_best_effort(dev))
                                fwnode_ret = -EAGAIN;
                        else
                                return dev_err_probe(dev, -EPROBE_DEFER,
                                                     "wait for supplier %pfwf\n", sup_fw);
                }
        }

        device_links_write_lock();

        list_for_each_entry(link, &dev->links.suppliers, c_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                if (link->status != DL_STATE_AVAILABLE &&
                    !device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)) {

                        if (dev_is_best_effort(dev) &&
                            device_link_test(link, DL_FLAG_INFERRED) &&
                            !link->supplier->can_match) {
                                ret = -EAGAIN;
                                continue;
                        }

                        device_links_missing_supplier(dev);
                        ret = dev_err_probe(dev, -EPROBE_DEFER,
                                            "supplier %s not ready\n", dev_name(link->supplier));
                        break;
                }
                WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
        }
        dev->links.status = DL_DEV_PROBING;

        device_links_write_unlock();

        return ret ? ret : fwnode_ret;
}

/**
 * __device_links_queue_sync_state - Queue a device for sync_state() callback
 * @dev: Device to call sync_state() on
 * @list: List head to queue the @dev on
 *
 * Queues a device for a sync_state() callback when the device links write lock
 * isn't held. This allows the sync_state() execution flow to use device links
 * APIs.  The caller must ensure this function is called with
 * device_links_write_lock() held.
 *
 * This function does a get_device() to make sure the device is not freed while
 * on this list.
 *
 * So the caller must also ensure that device_links_flush_sync_list() is called
 * as soon as the caller releases device_links_write_lock().  This is necessary
 * to make sure the sync_state() is called in a timely fashion and the
 * put_device() is called on this device.
 */
static void __device_links_queue_sync_state(struct device *dev,
                                            struct list_head *list)
{
        struct device_link *link;

        if (!dev_has_sync_state(dev))
                return;
        if (dev->state_synced)
                return;

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;
                if (link->status != DL_STATE_ACTIVE)
                        return;
        }

        /*
         * Set the flag here to avoid adding the same device to a list more
         * than once. This can happen if new consumers get added to the device
         * and probed before the list is flushed.
         */
        dev->state_synced = true;

        if (WARN_ON(!list_empty(&dev->links.defer_sync)))
                return;

        get_device(dev);
        list_add_tail(&dev->links.defer_sync, list);
}

/**
 * device_links_flush_sync_list - Call sync_state() on a list of devices
 * @list: List of devices to call sync_state() on
 * @dont_lock_dev: Device for which lock is already held by the caller
 *
 * Calls sync_state() on all the devices that have been queued for it. This
 * function is used in conjunction with __device_links_queue_sync_state(). The
 * @dont_lock_dev parameter is useful when this function is called from a
 * context where a device lock is already held.
 */
static void device_links_flush_sync_list(struct list_head *list,
                                         struct device *dont_lock_dev)
{
        struct device *dev, *tmp;

        list_for_each_entry_safe(dev, tmp, list, links.defer_sync) {
                list_del_init(&dev->links.defer_sync);

                if (dev != dont_lock_dev)
                        device_lock(dev);

                dev_sync_state(dev);

                if (dev != dont_lock_dev)
                        device_unlock(dev);

                put_device(dev);
        }
}

void device_links_supplier_sync_state_pause(void)
{
        device_links_write_lock();
        defer_sync_state_count++;
        device_links_write_unlock();
}

void device_links_supplier_sync_state_resume(void)
{
        struct device *dev, *tmp;
        LIST_HEAD(sync_list);

        device_links_write_lock();
        if (!defer_sync_state_count) {
                WARN(true, "Unmatched sync_state pause/resume!");
                goto out;
        }
        defer_sync_state_count--;
        if (defer_sync_state_count)
                goto out;

        list_for_each_entry_safe(dev, tmp, &deferred_sync, links.defer_sync) {
                /*
                 * Delete from deferred_sync list before queuing it to
                 * sync_list because defer_sync is used for both lists.
                 */
                list_del_init(&dev->links.defer_sync);
                __device_links_queue_sync_state(dev, &sync_list);
        }
out:
        device_links_write_unlock();

        device_links_flush_sync_list(&sync_list, NULL);
}

static int sync_state_resume_initcall(void)
{
        device_links_supplier_sync_state_resume();
        return 0;
}
late_initcall(sync_state_resume_initcall);

static void __device_links_supplier_defer_sync(struct device *sup)
{
        if (list_empty(&sup->links.defer_sync) && dev_has_sync_state(sup))
                list_add_tail(&sup->links.defer_sync, &deferred_sync);
}

static void device_link_drop_managed(struct device_link *link)
{
        link->flags &= ~DL_FLAG_MANAGED;
        WRITE_ONCE(link->status, DL_STATE_NONE);
        kref_put(&link->kref, __device_link_del);
}

static ssize_t waiting_for_supplier_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        bool val;

        device_lock(dev);
        scoped_guard(mutex, &fwnode_link_lock)
                val = !!fwnode_links_check_suppliers(dev->fwnode);
        device_unlock(dev);
        return sysfs_emit(buf, "%u\n", val);
}
static DEVICE_ATTR_RO(waiting_for_supplier);

/**
 * device_links_force_bind - Prepares device to be force bound
 * @dev: Consumer device.
 *
 * device_bind_driver() force binds a device to a driver without calling any
 * driver probe functions. So the consumer really isn't going to wait for any
 * supplier before it's bound to the driver. We still want the device link
 * states to be sensible when this happens.
 *
 * In preparation for device_bind_driver(), this function goes through each
 * supplier device links and checks if the supplier is bound. If it is, then
 * the device link status is set to CONSUMER_PROBE. Otherwise, the device link
 * is dropped. Links without the DL_FLAG_MANAGED flag set are ignored.
 */
void device_links_force_bind(struct device *dev)
{
        struct device_link *link, *ln;

        device_links_write_lock();

        list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                if (link->status != DL_STATE_AVAILABLE) {
                        device_link_drop_managed(link);
                        continue;
                }
                WRITE_ONCE(link->status, DL_STATE_CONSUMER_PROBE);
        }
        dev->links.status = DL_DEV_PROBING;

        device_links_write_unlock();
}

/**
 * device_links_driver_bound - Update device links after probing its driver.
 * @dev: Device to update the links for.
 *
 * The probe has been successful, so update links from this device to any
 * consumers by changing their status to "available".
 *
 * Also change the status of @dev's links to suppliers to "active".
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
void device_links_driver_bound(struct device *dev)
{
        struct device_link *link, *ln;
        LIST_HEAD(sync_list);

        /*
         * If a device binds successfully, it's expected to have created all
         * the device links it needs to or make new device links as it needs
         * them. So, fw_devlink no longer needs to create device links to any
         * of the device's suppliers.
         *
         * Also, if a child firmware node of this bound device is not added as a
         * device by now, assume it is never going to be added. Make this bound
         * device the fallback supplier to the dangling consumers of the child
         * firmware node because this bound device is probably implementing the
         * child firmware node functionality and we don't want the dangling
         * consumers to defer probe indefinitely waiting for a device for the
         * child firmware node.
         */
        if (dev->fwnode && dev->fwnode->dev == dev) {
                struct fwnode_handle *child;

                fwnode_links_purge_suppliers(dev->fwnode);

                guard(mutex)(&fwnode_link_lock);

                fwnode_for_each_available_child_node(dev->fwnode, child)
                        __fw_devlink_pickup_dangling_consumers(child,
                                                               dev->fwnode);
                __fw_devlink_link_to_consumers(dev);
        }
        device_remove_file(dev, &dev_attr_waiting_for_supplier);

        device_links_write_lock();

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                /*
                 * Links created during consumer probe may be in the "consumer
                 * probe" state to start with if the supplier is still probing
                 * when they are created and they may become "active" if the
                 * consumer probe returns first.  Skip them here.
                 */
                if (link->status == DL_STATE_CONSUMER_PROBE ||
                    link->status == DL_STATE_ACTIVE)
                        continue;

                WARN_ON(link->status != DL_STATE_DORMANT);
                WRITE_ONCE(link->status, DL_STATE_AVAILABLE);

                if (device_link_test(link, DL_FLAG_AUTOPROBE_CONSUMER))
                        driver_deferred_probe_add(link->consumer);
        }

        if (defer_sync_state_count)
                __device_links_supplier_defer_sync(dev);
        else
                __device_links_queue_sync_state(dev, &sync_list);

        list_for_each_entry_safe(link, ln, &dev->links.suppliers, c_node) {
                struct device *supplier;

                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                supplier = link->supplier;
                if (device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)) {
                        /*
                         * When DL_FLAG_SYNC_STATE_ONLY is set, it means no
                         * other DL_MANAGED_LINK_FLAGS have been set. So, it's
                         * save to drop the managed link completely.
                         */
                        device_link_drop_managed(link);
                } else if (dev_is_best_effort(dev) &&
                           device_link_test(link, DL_FLAG_INFERRED) &&
                           link->status != DL_STATE_CONSUMER_PROBE &&
                           !link->supplier->can_match) {
                        /*
                         * When dev_is_best_effort() is true, we ignore device
                         * links to suppliers that don't have a driver.  If the
                         * consumer device still managed to probe, there's no
                         * point in maintaining a device link in a weird state
                         * (consumer probed before supplier). So delete it.
                         */
                        device_link_drop_managed(link);
                } else {
                        WARN_ON(link->status != DL_STATE_CONSUMER_PROBE);
                        WRITE_ONCE(link->status, DL_STATE_ACTIVE);
                }

                /*
                 * This needs to be done even for the deleted
                 * DL_FLAG_SYNC_STATE_ONLY device link in case it was the last
                 * device link that was preventing the supplier from getting a
                 * sync_state() call.
                 */
                if (defer_sync_state_count)
                        __device_links_supplier_defer_sync(supplier);
                else
                        __device_links_queue_sync_state(supplier, &sync_list);
        }

        dev->links.status = DL_DEV_DRIVER_BOUND;

        device_links_write_unlock();

        device_links_flush_sync_list(&sync_list, dev);
}

/**
 * __device_links_no_driver - Update links of a device without a driver.
 * @dev: Device without a drvier.
 *
 * Delete all non-persistent links from this device to any suppliers.
 *
 * Persistent links stay around, but their status is changed to "available",
 * unless they already are in the "supplier unbind in progress" state in which
 * case they need not be updated.
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
static void __device_links_no_driver(struct device *dev)
{
        struct device_link *link, *ln;

        list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                if (device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER)) {
                        device_link_drop_managed(link);
                        continue;
                }

                if (link->status != DL_STATE_CONSUMER_PROBE &&
                    link->status != DL_STATE_ACTIVE)
                        continue;

                if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) {
                        WRITE_ONCE(link->status, DL_STATE_AVAILABLE);
                } else {
                        WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY));
                        WRITE_ONCE(link->status, DL_STATE_DORMANT);
                }
        }

        dev->links.status = DL_DEV_NO_DRIVER;
}

/**
 * device_links_no_driver - Update links after failing driver probe.
 * @dev: Device whose driver has just failed to probe.
 *
 * Clean up leftover links to consumers for @dev and invoke
 * %__device_links_no_driver() to update links to suppliers for it as
 * appropriate.
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
void device_links_no_driver(struct device *dev)
{
        struct device_link *link;

        device_links_write_lock();

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                /*
                 * The probe has failed, so if the status of the link is
                 * "consumer probe" or "active", it must have been added by
                 * a probing consumer while this device was still probing.
                 * Change its state to "dormant", as it represents a valid
                 * relationship, but it is not functionally meaningful.
                 */
                if (link->status == DL_STATE_CONSUMER_PROBE ||
                    link->status == DL_STATE_ACTIVE)
                        WRITE_ONCE(link->status, DL_STATE_DORMANT);
        }

        __device_links_no_driver(dev);

        device_links_write_unlock();
}

/**
 * device_links_driver_cleanup - Update links after driver removal.
 * @dev: Device whose driver has just gone away.
 *
 * Update links to consumers for @dev by changing their status to "dormant" and
 * invoke %__device_links_no_driver() to update links to suppliers for it as
 * appropriate.
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
void device_links_driver_cleanup(struct device *dev)
{
        struct device_link *link, *ln;

        device_links_write_lock();

        list_for_each_entry_safe(link, ln, &dev->links.consumers, s_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                WARN_ON(device_link_test(link, DL_FLAG_AUTOREMOVE_CONSUMER));
                WARN_ON(link->status != DL_STATE_SUPPLIER_UNBIND);

                /*
                 * autoremove the links between this @dev and its consumer
                 * devices that are not active, i.e. where the link state
                 * has moved to DL_STATE_SUPPLIER_UNBIND.
                 */
                if (link->status == DL_STATE_SUPPLIER_UNBIND &&
                    device_link_test(link, DL_FLAG_AUTOREMOVE_SUPPLIER))
                        device_link_drop_managed(link);

                WRITE_ONCE(link->status, DL_STATE_DORMANT);
        }

        list_del_init(&dev->links.defer_sync);
        __device_links_no_driver(dev);

        device_links_write_unlock();
}

/**
 * device_links_busy - Check if there are any busy links to consumers.
 * @dev: Device to check.
 *
 * Check each consumer of the device and return 'true' if its link's status
 * is one of "consumer probe" or "active" (meaning that the given consumer is
 * probing right now or its driver is present).  Otherwise, change the link
 * state to "supplier unbind" to prevent the consumer from being probed
 * successfully going forward.
 *
 * Return 'false' if there are no probing or active consumers.
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
bool device_links_busy(struct device *dev)
{
        struct device_link *link;
        bool ret = false;

        device_links_write_lock();

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                if (!device_link_test(link, DL_FLAG_MANAGED))
                        continue;

                if (link->status == DL_STATE_CONSUMER_PROBE
                    || link->status == DL_STATE_ACTIVE) {
                        ret = true;
                        break;
                }
                WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
        }

        dev->links.status = DL_DEV_UNBINDING;

        device_links_write_unlock();
        return ret;
}

/**
 * device_links_unbind_consumers - Force unbind consumers of the given device.
 * @dev: Device to unbind the consumers of.
 *
 * Walk the list of links to consumers for @dev and if any of them is in the
 * "consumer probe" state, wait for all device probes in progress to complete
 * and start over.
 *
 * If that's not the case, change the status of the link to "supplier unbind"
 * and check if the link was in the "active" state.  If so, force the consumer
 * driver to unbind and start over (the consumer will not re-probe as we have
 * changed the state of the link already).
 *
 * Links without the DL_FLAG_MANAGED flag set are ignored.
 */
void device_links_unbind_consumers(struct device *dev)
{
        struct device_link *link;

 start:
        device_links_write_lock();

        list_for_each_entry(link, &dev->links.consumers, s_node) {
                enum device_link_state status;

                if (!device_link_test(link, DL_FLAG_MANAGED) ||
                    device_link_test(link, DL_FLAG_SYNC_STATE_ONLY))
                        continue;

                status = link->status;
                if (status == DL_STATE_CONSUMER_PROBE) {
                        device_links_write_unlock();

                        wait_for_device_probe();
                        goto start;
                }
                WRITE_ONCE(link->status, DL_STATE_SUPPLIER_UNBIND);
                if (status == DL_STATE_ACTIVE) {
                        struct device *consumer = link->consumer;

                        get_device(consumer);

                        device_links_write_unlock();

                        device_release_driver_internal(consumer, NULL,
                                                       consumer->parent);
                        put_device(consumer);
                        goto start;
                }
        }

        device_links_write_unlock();
}

/**
 * device_links_purge - Delete existing links to other devices.
 * @dev: Target device.
 */
static void device_links_purge(struct device *dev)
{
        struct device_link *link, *ln;

        if (dev->class == &devlink_class)
                return;

        /*
         * Delete all of the remaining links from this device to any other
         * devices (either consumers or suppliers).
         */
        device_links_write_lock();

        list_for_each_entry_safe_reverse(link, ln, &dev->links.suppliers, c_node) {
                WARN_ON(link->status == DL_STATE_ACTIVE);
                __device_link_del(&link->kref);
        }

        list_for_each_entry_safe_reverse(link, ln, &dev->links.consumers, s_node) {
                WARN_ON(link->status != DL_STATE_DORMANT &&
                        link->status != DL_STATE_NONE);
                __device_link_del(&link->kref);
        }

        device_links_write_unlock();
}

#define FW_DEVLINK_FLAGS_PERMISSIVE        (DL_FLAG_INFERRED | \
                                         DL_FLAG_SYNC_STATE_ONLY)
#define FW_DEVLINK_FLAGS_ON                (DL_FLAG_INFERRED | \
                                         DL_FLAG_AUTOPROBE_CONSUMER)
#define FW_DEVLINK_FLAGS_RPM                (FW_DEVLINK_FLAGS_ON | \
                                         DL_FLAG_PM_RUNTIME)

static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_RPM;
static int __init fw_devlink_setup(char *arg)
{
        if (!arg)
                return -EINVAL;

        if (strcmp(arg, "off") == 0) {
                fw_devlink_flags = 0;
        } else if (strcmp(arg, "permissive") == 0) {
                fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE;
        } else if (strcmp(arg, "on") == 0) {
                fw_devlink_flags = FW_DEVLINK_FLAGS_ON;
        } else if (strcmp(arg, "rpm") == 0) {
                fw_devlink_flags = FW_DEVLINK_FLAGS_RPM;
        }
        return 0;
}
early_param("fw_devlink", fw_devlink_setup);

static bool fw_devlink_strict;
static int __init fw_devlink_strict_setup(char *arg)
{
        return kstrtobool(arg, &fw_devlink_strict);
}
early_param("fw_devlink.strict", fw_devlink_strict_setup);

#define FW_DEVLINK_SYNC_STATE_STRICT        0
#define FW_DEVLINK_SYNC_STATE_TIMEOUT        1

#ifndef CONFIG_FW_DEVLINK_SYNC_STATE_TIMEOUT
static int fw_devlink_sync_state;
#else
static int fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
#endif

static int __init fw_devlink_sync_state_setup(char *arg)
{
        if (!arg)
                return -EINVAL;

        if (strcmp(arg, "strict") == 0) {
                fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_STRICT;
                return 0;
        } else if (strcmp(arg, "timeout") == 0) {
                fw_devlink_sync_state = FW_DEVLINK_SYNC_STATE_TIMEOUT;
                return 0;
        }
        return -EINVAL;
}
early_param("fw_devlink.sync_state", fw_devlink_sync_state_setup);

static inline u32 fw_devlink_get_flags(u8 fwlink_flags)
{
        if (fwlink_flags & FWLINK_FLAG_CYCLE)
                return FW_DEVLINK_FLAGS_PERMISSIVE | DL_FLAG_CYCLE;

        return fw_devlink_flags;
}

static bool fw_devlink_is_permissive(void)
{
        return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE;
}

bool fw_devlink_is_strict(void)
{
        return fw_devlink_strict && !fw_devlink_is_permissive();
}

static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode)
{
        if (fwnode->flags & FWNODE_FLAG_LINKS_ADDED)
                return;

        fwnode_call_int_op(fwnode, add_links);
        fwnode->flags |= FWNODE_FLAG_LINKS_ADDED;
}

static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode)
{
        struct fwnode_handle *child = NULL;

        fw_devlink_parse_fwnode(fwnode);

        while ((child = fwnode_get_next_available_child_node(fwnode, child)))
                fw_devlink_parse_fwtree(child);
}

static void fw_devlink_relax_link(struct device_link *link)
{
        if (!device_link_test(link, DL_FLAG_INFERRED))
                return;

        if (device_link_flag_is_sync_state_only(link->flags))
                return;

        pm_runtime_drop_link(link);
        link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE;
        dev_dbg(link->consumer, "Relaxing link with %s\n",
                dev_name(link->supplier));
}

static int fw_devlink_no_driver(struct device *dev, void *data)
{
        struct device_link *link = to_devlink(dev);

        if (!link->supplier->can_match)
                fw_devlink_relax_link(link);

        return 0;
}

void fw_devlink_drivers_done(void)
{
        fw_devlink_drv_reg_done = true;
        device_links_write_lock();
        class_for_each_device(&devlink_class, NULL, NULL,
                              fw_devlink_no_driver);
        device_links_write_unlock();
}

static int fw_devlink_dev_sync_state(struct device *dev, void *data)
{
        struct device_link *link = to_devlink(dev);
        struct device *sup = link->supplier;

        if (!device_link_test(link, DL_FLAG_MANAGED) ||
            link->status == DL_STATE_ACTIVE || sup->state_synced ||
            !dev_has_sync_state(sup))
                return 0;

        if (fw_devlink_sync_state == FW_DEVLINK_SYNC_STATE_STRICT) {
                dev_info(sup, "sync_state() pending due to %s\n",
                         dev_name(link->consumer));
                return 0;
        }

        if (!list_empty(&sup->links.defer_sync))
                return 0;

        dev_warn(sup, "Timed out. Forcing sync_state()\n");
        sup->state_synced = true;
        get_device(sup);
        list_add_tail(&sup->links.defer_sync, data);

        return 0;
}

void fw_devlink_probing_done(void)
{
        LIST_HEAD(sync_list);

        device_links_write_lock();
        class_for_each_device(&devlink_class, NULL, &sync_list,
                              fw_devlink_dev_sync_state);
        device_links_write_unlock();
        device_links_flush_sync_list(&sync_list, NULL);
}

/**
 * wait_for_init_devices_probe - Try to probe any device needed for init
 *
 * Some devices might need to be probed and bound successfully before the kernel
 * boot sequence can finish and move on to init/userspace. For example, a
 * network interface might need to be bound to be able to mount a NFS rootfs.
 *
 * With fw_devlink=on by default, some of these devices might be blocked from
 * probing because they are waiting on a optional supplier that doesn't have a
 * driver. While fw_devlink will eventually identify such devices and unblock
 * the probing automatically, it might be too late by the time it unblocks the
 * probing of devices. For example, the IP4 autoconfig might timeout before
 * fw_devlink unblocks probing of the network interface.
 *
 * This function is available to temporarily try and probe all devices that have
 * a driver even if some of their suppliers haven't been added or don't have
 * drivers.
 *
 * The drivers can then decide which of the suppliers are optional vs mandatory
 * and probe the device if possible. By the time this function returns, all such
 * "best effort" probes are guaranteed to be completed. If a device successfully
 * probes in this mode, we delete all fw_devlink discovered dependencies of that
 * device where the supplier hasn't yet probed successfully because they have to
 * be optional dependencies.
 *
 * Any devices that didn't successfully probe go back to being treated as if
 * this function was never called.
 *
 * This also means that some devices that aren't needed for init and could have
 * waited for their optional supplier to probe (when the supplier's module is
 * loaded later on) would end up probing prematurely with limited functionality.
 * So call this function only when boot would fail without it.
 */
void __init wait_for_init_devices_probe(void)
{
        if (!fw_devlink_flags || fw_devlink_is_permissive())
                return;

        /*
         * Wait for all ongoing probes to finish so that the "best effort" is
         * only applied to devices that can't probe otherwise.
         */
        wait_for_device_probe();

        pr_info("Trying to probe devices needed for running init ...\n");
        fw_devlink_best_effort = true;
        driver_deferred_probe_trigger();

        /*
         * Wait for all "best effort" probes to finish before going back to
         * normal enforcement.
         */
        wait_for_device_probe();
        fw_devlink_best_effort = false;
}

static void fw_devlink_unblock_consumers(struct device *dev)
{
        struct device_link *link;

        if (!fw_devlink_flags || fw_devlink_is_permissive())
                return;

        device_links_write_lock();
        list_for_each_entry(link, &dev->links.consumers, s_node)
                fw_devlink_relax_link(link);
        device_links_write_unlock();
}

static bool fwnode_init_without_drv(struct fwnode_handle *fwnode)
{
        struct device *dev;
        bool ret;

        if (!(fwnode->flags & FWNODE_FLAG_INITIALIZED))
                return false;

        dev = get_dev_from_fwnode(fwnode);
        ret = !dev || dev->links.status == DL_DEV_NO_DRIVER;
        put_device(dev);

        return ret;
}

static bool fwnode_ancestor_init_without_drv(struct fwnode_handle *fwnode)
{
        struct fwnode_handle *parent;

        fwnode_for_each_parent_node(fwnode, parent) {
                if (fwnode_init_without_drv(parent)) {
                        fwnode_handle_put(parent);
                        return true;
                }
        }

        return false;
}

/**
 * fwnode_is_ancestor_of - Test if @ancestor is ancestor of @child
 * @ancestor: Firmware which is tested for being an ancestor
 * @child: Firmware which is tested for being the child
 *
 * A node is considered an ancestor of itself too.
 *
 * Return: true if @ancestor is an ancestor of @child. Otherwise, returns false.
 */
static bool fwnode_is_ancestor_of(const struct fwnode_handle *ancestor,
                                  const struct fwnode_handle *child)
{
        struct fwnode_handle *parent;

        if (IS_ERR_OR_NULL(ancestor))
                return false;

        if (child == ancestor)
                return true;

        fwnode_for_each_parent_node(child, parent) {
                if (parent == ancestor) {
                        fwnode_handle_put(parent);
                        return true;
                }
        }
        return false;
}

/**
 * fwnode_get_next_parent_dev - Find device of closest ancestor fwnode
 * @fwnode: firmware node
 *
 * Given a firmware node (@fwnode), this function finds its closest ancestor
 * firmware node that has a corresponding struct device and returns that struct
 * device.
 *
 * The caller is responsible for calling put_device() on the returned device
 * pointer.
 *
 * Return: a pointer to the device of the @fwnode's closest ancestor.
 */
static struct device *fwnode_get_next_parent_dev(const struct fwnode_handle *fwnode)
{
        struct fwnode_handle *parent;
        struct device *dev;

        fwnode_for_each_parent_node(fwnode, parent) {
                dev = get_dev_from_fwnode(parent);
                if (dev) {
                        fwnode_handle_put(parent);
                        return dev;
                }
        }
        return NULL;
}

/**
 * __fw_devlink_relax_cycles - Relax and mark dependency cycles.
 * @con_handle: Potential consumer device fwnode.
 * @sup_handle: Potential supplier's fwnode.
 *
 * Needs to be called with fwnode_lock and device link lock held.
 *
 * Check if @sup_handle or any of its ancestors or suppliers direct/indirectly
 * depend on @con. This function can detect multiple cyles between @sup_handle
 * and @con. When such dependency cycles are found, convert all device links
 * created solely by fw_devlink into SYNC_STATE_ONLY device links. Also, mark
 * all fwnode links in the cycle with FWLINK_FLAG_CYCLE so that when they are
 * converted into a device link in the future, they are created as
 * SYNC_STATE_ONLY device links. This is the equivalent of doing
 * fw_devlink=permissive just between the devices in the cycle. We need to do
 * this because, at this point, fw_devlink can't tell which of these
 * dependencies is not a real dependency.
 *
 * Return true if one or more cycles were found. Otherwise, return false.
 */
static bool __fw_devlink_relax_cycles(struct fwnode_handle *con_handle,
                                 struct fwnode_handle *sup_handle)
{
        struct device *sup_dev = NULL, *par_dev = NULL, *con_dev = NULL;
        struct fwnode_link *link;
        struct device_link *dev_link;
        bool ret = false;

        if (!sup_handle)
                return false;

        /*
         * We aren't trying to find all cycles. Just a cycle between con and
         * sup_handle.
         */
        if (sup_handle->flags & FWNODE_FLAG_VISITED)
                return false;

        sup_handle->flags |= FWNODE_FLAG_VISITED;

        /* Termination condition. */
        if (sup_handle == con_handle) {
                pr_debug("----- cycle: start -----\n");
                ret = true;
                goto out;
        }

        sup_dev = get_dev_from_fwnode(sup_handle);
        con_dev = get_dev_from_fwnode(con_handle);
        /*
         * If sup_dev is bound to a driver and @con hasn't started binding to a
         * driver, sup_dev can't be a consumer of @con. So, no need to check
         * further.
         */
        if (sup_dev && sup_dev->links.status ==  DL_DEV_DRIVER_BOUND &&
            con_dev && con_dev->links.status == DL_DEV_NO_DRIVER) {
                ret = false;
                goto out;
        }

        list_for_each_entry(link, &sup_handle->suppliers, c_hook) {
                if (link->flags & FWLINK_FLAG_IGNORE)
                        continue;

                if (__fw_devlink_relax_cycles(con_handle, link->supplier)) {
                        __fwnode_link_cycle(link);
                        ret = true;
                }
        }

        /*
         * Give priority to device parent over fwnode parent to account for any
         * quirks in how fwnodes are converted to devices.
         */
        if (sup_dev)
                par_dev = get_device(sup_dev->parent);
        else
                par_dev = fwnode_get_next_parent_dev(sup_handle);

        if (par_dev && __fw_devlink_relax_cycles(con_handle, par_dev->fwnode)) {
                pr_debug("%pfwf: cycle: child of %pfwf\n", sup_handle,
                         par_dev->fwnode);
                ret = true;
        }

        if (!sup_dev)
                goto out;

        list_for_each_entry(dev_link, &sup_dev->links.suppliers, c_node) {
                /*
                 * Ignore a SYNC_STATE_ONLY flag only if it wasn't marked as
                 * such due to a cycle.
                 */
                if (device_link_flag_is_sync_state_only(dev_link->flags) &&
                    !device_link_test(dev_link, DL_FLAG_CYCLE))
                        continue;

                if (__fw_devlink_relax_cycles(con_handle,
                                              dev_link->supplier->fwnode)) {
                        pr_debug("%pfwf: cycle: depends on %pfwf\n", sup_handle,
                                 dev_link->supplier->fwnode);
                        fw_devlink_relax_link(dev_link);
                        dev_link->flags |= DL_FLAG_CYCLE;
                        ret = true;
                }
        }

out:
        sup_handle->flags &= ~FWNODE_FLAG_VISITED;
        put_device(sup_dev);
        put_device(con_dev);
        put_device(par_dev);
        return ret;
}

/**
 * fw_devlink_create_devlink - Create a device link from a consumer to fwnode
 * @con: consumer device for the device link
 * @sup_handle: fwnode handle of supplier
 * @link: fwnode link that's being converted to a device link
 *
 * This function will try to create a device link between the consumer device
 * @con and the supplier device represented by @sup_handle.
 *
 * The supplier has to be provided as a fwnode because incorrect cycles in
 * fwnode links can sometimes cause the supplier device to never be created.
 * This function detects such cases and returns an error if it cannot create a
 * device link from the consumer to a missing supplier.
 *
 * Returns,
 * 0 on successfully creating a device link
 * -EINVAL if the device link cannot be created as expected
 * -EAGAIN if the device link cannot be created right now, but it may be
 *  possible to do that in the future
 */
static int fw_devlink_create_devlink(struct device *con,
                                     struct fwnode_handle *sup_handle,
                                     struct fwnode_link *link)
{
        struct device *sup_dev;
        int ret = 0;
        u32 flags;

        if (link->flags & FWLINK_FLAG_IGNORE)
                return 0;

        /*
         * In some cases, a device P might also be a supplier to its child node
         * C. However, this would defer the probe of C until the probe of P
         * completes successfully. This is perfectly fine in the device driver
         * model. device_add() doesn't guarantee probe completion of the device
         * by the time it returns.
         *
         * However, there are a few drivers that assume C will finish probing
         * as soon as it's added and before P finishes probing. So, we provide
         * a flag to let fw_devlink know not to delay the probe of C until the
         * probe of P completes successfully.
         *
         * When such a flag is set, we can't create device links where P is the
         * supplier of C as that would delay the probe of C.
         */
        if (sup_handle->flags & FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD &&
            fwnode_is_ancestor_of(sup_handle, con->fwnode))
                return -EINVAL;

        /*
         * Don't try to optimize by not calling the cycle detection logic under
         * certain conditions. There's always some corner case that won't get
         * detected.
         */
        device_links_write_lock();
        if (__fw_devlink_relax_cycles(link->consumer, sup_handle)) {
                __fwnode_link_cycle(link);
                pr_debug("----- cycle: end -----\n");
                pr_info("%pfwf: Fixed dependency cycle(s) with %pfwf\n",
                        link->consumer, sup_handle);
        }
        device_links_write_unlock();

        if (con->fwnode == link->consumer)
                flags = fw_devlink_get_flags(link->flags);
        else
                flags = FW_DEVLINK_FLAGS_PERMISSIVE;

        if (sup_handle->flags & FWNODE_FLAG_NOT_DEVICE)
                sup_dev = fwnode_get_next_parent_dev(sup_handle);
        else
                sup_dev = get_dev_from_fwnode(sup_handle);

        if (sup_dev) {
                /*
                 * If it's one of those drivers that don't actually bind to
                 * their device using driver core, then don't wait on this
                 * supplier device indefinitely.
                 */
                if (sup_dev->links.status == DL_DEV_NO_DRIVER &&
                    sup_handle->flags & FWNODE_FLAG_INITIALIZED) {
                        dev_dbg(con,
                                "Not linking %pfwf - dev might never probe\n",
                                sup_handle);
                        ret = -EINVAL;
                        goto out;
                }

                if (con != sup_dev && !device_link_add(con, sup_dev, flags)) {
                        dev_err(con, "Failed to create device link (0x%x) with supplier %s for %pfwf\n",
                                flags, dev_name(sup_dev), link->consumer);
                        ret = -EINVAL;
                }

                goto out;
        }

        /*
         * Supplier or supplier's ancestor already initialized without a struct
         * device or being probed by a driver.
         */
        if (fwnode_init_without_drv(sup_handle) ||
            fwnode_ancestor_init_without_drv(sup_handle)) {
                dev_dbg(con, "Not linking %pfwf - might never become dev\n",
                        sup_handle);
                return -EINVAL;
        }

        ret = -EAGAIN;
out:
        put_device(sup_dev);
        return ret;
}

/**
 * __fw_devlink_link_to_consumers - Create device links to consumers of a device
 * @dev: Device that needs to be linked to its consumers
 *
 * This function looks at all the consumer fwnodes of @dev and creates device
 * links between the consumer device and @dev (supplier).
 *
 * If the consumer device has not been added yet, then this function creates a
 * SYNC_STATE_ONLY link between @dev (supplier) and the closest ancestor device
 * of the consumer fwnode. This is necessary to make sure @dev doesn't get a
 * sync_state() callback before the real consumer device gets to be added and
 * then probed.
 *
 * Once device links are created from the real consumer to @dev (supplier), the
 * fwnode links are deleted.
 */
static void __fw_devlink_link_to_consumers(struct device *dev)
{
        struct fwnode_handle *fwnode = dev->fwnode;
        struct fwnode_link *link, *tmp;

        list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) {
                struct device *con_dev;
                bool own_link = true;
                int ret;

                con_dev = get_dev_from_fwnode(link->consumer);
                /*
                 * If consumer device is not available yet, make a "proxy"
                 * SYNC_STATE_ONLY link from the consumer's parent device to
                 * the supplier device. This is necessary to make sure the
                 * supplier doesn't get a sync_state() callback before the real
                 * consumer can create a device link to the supplier.
                 *
                 * This proxy link step is needed to handle the case where the
                 * consumer's parent device is added before the supplier.
                 */
                if (!con_dev) {
                        con_dev = fwnode_get_next_parent_dev(link->consumer);
                        /*
                         * However, if the consumer's parent device is also the
                         * parent of the supplier, don't create a
                         * consumer-supplier link from the parent to its child
                         * device. Such a dependency is impossible.
                         */
                        if (con_dev &&
                            fwnode_is_ancestor_of(con_dev->fwnode, fwnode)) {
                                put_device(con_dev);
                                con_dev = NULL;
                        } else {
                                own_link = false;
                        }
                }

                if (!con_dev)
                        continue;

                ret = fw_devlink_create_devlink(con_dev, fwnode, link);
                put_device(con_dev);
                if (!own_link || ret == -EAGAIN)
                        continue;

                __fwnode_link_del(link);
        }
}

/**
 * __fw_devlink_link_to_suppliers - Create device links to suppliers of a device
 * @dev: The consumer device that needs to be linked to its suppliers
 * @fwnode: Root of the fwnode tree that is used to create device links
 *
 * This function looks at all the supplier fwnodes of fwnode tree rooted at
 * @fwnode and creates device links between @dev (consumer) and all the
 * supplier devices of the entire fwnode tree at @fwnode.
 *
 * The function creates normal (non-SYNC_STATE_ONLY) device links between @dev
 * and the real suppliers of @dev. Once these device links are created, the
 * fwnode links are deleted.
 *
 * In addition, it also looks at all the suppliers of the entire fwnode tree
 * because some of the child devices of @dev that have not been added yet
 * (because @dev hasn't probed) might already have their suppliers added to
 * driver core. So, this function creates SYNC_STATE_ONLY device links between
 * @dev (consumer) and these suppliers to make sure they don't execute their
 * sync_state() callbacks before these child devices have a chance to create
 * their device links. The fwnode links that correspond to the child devices
 * aren't delete because they are needed later to create the device links
 * between the real consumer and supplier devices.
 */
static void __fw_devlink_link_to_suppliers(struct device *dev,
                                           struct fwnode_handle *fwnode)
{
        bool own_link = (dev->fwnode == fwnode);
        struct fwnode_link *link, *tmp;
        struct fwnode_handle *child = NULL;

        list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) {
                int ret;
                struct fwnode_handle *sup = link->supplier;

                ret = fw_devlink_create_devlink(dev, sup, link);
                if (!own_link || ret == -EAGAIN)
                        continue;

                __fwnode_link_del(link);
        }

        /*
         * Make "proxy" SYNC_STATE_ONLY device links to represent the needs of
         * all the descendants. This proxy link step is needed to handle the
         * case where the supplier is added before the consumer's parent device
         * (@dev).
         */
        while ((child = fwnode_get_next_available_child_node(fwnode, child)))
                __fw_devlink_link_to_suppliers(dev, child);
}

static void fw_devlink_link_device(struct device *dev)
{
        struct fwnode_handle *fwnode = dev->fwnode;

        if (!fw_devlink_flags)
                return;

        fw_devlink_parse_fwtree(fwnode);

        guard(mutex)(&fwnode_link_lock);

        __fw_devlink_link_to_consumers(dev);
        __fw_devlink_link_to_suppliers(dev, fwnode);
}

/* Device links support end. */

static struct kobject *dev_kobj;

/* /sys/dev/char */
static struct kobject *sysfs_dev_char_kobj;

/* /sys/dev/block */
static struct kobject *sysfs_dev_block_kobj;

static DEFINE_MUTEX(device_hotplug_lock);

void lock_device_hotplug(void)
{
        mutex_lock(&device_hotplug_lock);
}

void unlock_device_hotplug(void)
{
        mutex_unlock(&device_hotplug_lock);
}

int lock_device_hotplug_sysfs(void)
{
        if (mutex_trylock(&device_hotplug_lock))
                return 0;

        /* Avoid busy looping (5 ms of sleep should do). */
        msleep(5);
        return restart_syscall();
}

#ifdef CONFIG_BLOCK
static inline int device_is_not_partition(struct device *dev)
{
        return !(dev->type == &part_type);
}
#else
static inline int device_is_not_partition(struct device *dev)
{
        return 1;
}
#endif

static void device_platform_notify(struct device *dev)
{
        acpi_device_notify(dev);

        software_node_notify(dev);
}

static void device_platform_notify_remove(struct device *dev)
{
        software_node_notify_remove(dev);

        acpi_device_notify_remove(dev);
}

/**
 * dev_driver_string - Return a device's driver name, if at all possible
 * @dev: struct device to get the name of
 *
 * Will return the device's driver's name if it is bound to a device.  If
 * the device is not bound to a driver, it will return the name of the bus
 * it is attached to.  If it is not attached to a bus either, an empty
 * string will be returned.
 */
const char *dev_driver_string(const struct device *dev)
{
        struct device_driver *drv;

        /* dev->driver can change to NULL underneath us because of unbinding,
         * so be careful about accessing it.  dev->bus and dev->class should
         * never change once they are set, so they don't need special care.
         */
        drv = READ_ONCE(dev->driver);
        return drv ? drv->name : dev_bus_name(dev);
}
EXPORT_SYMBOL(dev_driver_string);

#define to_dev_attr(_attr) container_of(_attr, struct device_attribute, attr)

static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
                             char *buf)
{
        struct device_attribute *dev_attr = to_dev_attr(attr);
        struct device *dev = kobj_to_dev(kobj);
        ssize_t ret = -EIO;

        if (dev_attr->show)
                ret = dev_attr->show(dev, dev_attr, buf);
        if (ret >= (ssize_t)PAGE_SIZE) {
                printk("dev_attr_show: %pS returned bad count\n",
                                dev_attr->show);
        }
        return ret;
}

static ssize_t dev_attr_store(struct kobject *kobj, struct attribute *attr,
                              const char *buf, size_t count)
{
        struct device_attribute *dev_attr = to_dev_attr(attr);
        struct device *dev = kobj_to_dev(kobj);
        ssize_t ret = -EIO;

        if (dev_attr->store)
                ret = dev_attr->store(dev, dev_attr, buf, count);
        return ret;
}

static const struct sysfs_ops dev_sysfs_ops = {
        .show        = dev_attr_show,
        .store        = dev_attr_store,
};

#define to_ext_attr(x) container_of(x, struct dev_ext_attribute, attr)

ssize_t device_store_ulong(struct device *dev,
                           struct device_attribute *attr,
                           const char *buf, size_t size)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);
        int ret;
        unsigned long new;

        ret = kstrtoul(buf, 0, &new);
        if (ret)
                return ret;
        *(unsigned long *)(ea->var) = new;
        /* Always return full write size even if we didn't consume all */
        return size;
}
EXPORT_SYMBOL_GPL(device_store_ulong);

ssize_t device_show_ulong(struct device *dev,
                          struct device_attribute *attr,
                          char *buf)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);
        return sysfs_emit(buf, "%lx\n", *(unsigned long *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_ulong);

ssize_t device_store_int(struct device *dev,
                         struct device_attribute *attr,
                         const char *buf, size_t size)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);
        int ret;
        long new;

        ret = kstrtol(buf, 0, &new);
        if (ret)
                return ret;

        if (new > INT_MAX || new < INT_MIN)
                return -EINVAL;
        *(int *)(ea->var) = new;
        /* Always return full write size even if we didn't consume all */
        return size;
}
EXPORT_SYMBOL_GPL(device_store_int);

ssize_t device_show_int(struct device *dev,
                        struct device_attribute *attr,
                        char *buf)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);

        return sysfs_emit(buf, "%d\n", *(int *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_int);

ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
                          const char *buf, size_t size)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);

        if (kstrtobool(buf, ea->var) < 0)
                return -EINVAL;

        return size;
}
EXPORT_SYMBOL_GPL(device_store_bool);

ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);

        return sysfs_emit(buf, "%d\n", *(bool *)(ea->var));
}
EXPORT_SYMBOL_GPL(device_show_bool);

ssize_t device_show_string(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct dev_ext_attribute *ea = to_ext_attr(attr);

        return sysfs_emit(buf, "%s\n", (char *)ea->var);
}
EXPORT_SYMBOL_GPL(device_show_string);

/**
 * device_release - free device structure.
 * @kobj: device's kobject.
 *
 * This is called once the reference count for the object
 * reaches 0. We forward the call to the device's release
 * method, which should handle actually freeing the structure.
 */
static void device_release(struct kobject *kobj)
{
        struct device *dev = kobj_to_dev(kobj);
        struct device_private *p = dev->p;

        /*
         * Some platform devices are driven without driver attached
         * and managed resources may have been acquired.  Make sure
         * all resources are released.
         *
         * Drivers still can add resources into device after device
         * is deleted but alive, so release devres here to avoid
         * possible memory leak.
         */
        devres_release_all(dev);

        kfree(dev->dma_range_map);

        if (dev->release)
                dev->release(dev);
        else if (dev->type && dev->type->release)
                dev->type->release(dev);
        else if (dev->class && dev->class->dev_release)
                dev->class->dev_release(dev);
        else
                WARN(1, KERN_ERR "Device '%s' does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n",
                        dev_name(dev));
        kfree(p);
}

static const void *device_namespace(const struct kobject *kobj)
{
        const struct device *dev = kobj_to_dev(kobj);
        const void *ns = NULL;

        if (dev->class && dev->class->namespace)
                ns = dev->class->namespace(dev);

        return ns;
}

static void device_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
        const struct device *dev = kobj_to_dev(kobj);

        if (dev->class && dev->class->get_ownership)
                dev->class->get_ownership(dev, uid, gid);
}

static const struct kobj_type device_ktype = {
        .release        = device_release,
        .sysfs_ops        = &dev_sysfs_ops,
        .namespace        = device_namespace,
        .get_ownership        = device_get_ownership,
};


static int dev_uevent_filter(const struct kobject *kobj)
{
        const struct kobj_type *ktype = get_ktype(kobj);

        if (ktype == &device_ktype) {
                const struct device *dev = kobj_to_dev(kobj);
                if (dev->bus)
                        return 1;
                if (dev->class)
                        return 1;
        }
        return 0;
}

static const char *dev_uevent_name(const struct kobject *kobj)
{
        const struct device *dev = kobj_to_dev(kobj);

        if (dev->bus)
                return dev->bus->name;
        if (dev->class)
                return dev->class->name;
        return NULL;
}

/*
 * Try filling "DRIVER=<name>" uevent variable for a device. Because this
 * function may race with binding and unbinding the device from a driver,
 * we need to be careful. Binding is generally safe, at worst we miss the
 * fact that the device is already bound to a driver (but the driver
 * information that is delivered through uevents is best-effort, it may
 * become obsolete as soon as it is generated anyways). Unbinding is more
 * risky as driver pointer is transitioning to NULL, so READ_ONCE() should
 * be used to make sure we are dealing with the same pointer, and to
 * ensure that driver structure is not going to disappear from under us
 * we take bus' drivers klist lock. The assumption that only registered
 * driver can be bound to a device, and to unregister a driver bus code
 * will take the same lock.
 */
static void dev_driver_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
        struct subsys_private *sp = bus_to_subsys(dev->bus);

        if (sp) {
                scoped_guard(spinlock, &sp->klist_drivers.k_lock) {
                        struct device_driver *drv = READ_ONCE(dev->driver);
                        if (drv)
                                add_uevent_var(env, "DRIVER=%s", drv->name);
                }

                subsys_put(sp);
        }
}

static int dev_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)
{
        const struct device *dev = kobj_to_dev(kobj);
        int retval = 0;

        /* add device node properties if present */
        if (MAJOR(dev->devt)) {
                const char *tmp;
                const char *name;
                umode_t mode = 0;
                kuid_t uid = GLOBAL_ROOT_UID;
                kgid_t gid = GLOBAL_ROOT_GID;

                add_uevent_var(env, "MAJOR=%u", MAJOR(dev->devt));
                add_uevent_var(env, "MINOR=%u", MINOR(dev->devt));
                name = device_get_devnode(dev, &mode, &uid, &gid, &tmp);
                if (name) {
                        add_uevent_var(env, "DEVNAME=%s", name);
                        if (mode)
                                add_uevent_var(env, "DEVMODE=%#o", mode & 0777);
                        if (!uid_eq(uid, GLOBAL_ROOT_UID))
                                add_uevent_var(env, "DEVUID=%u", from_kuid(&init_user_ns, uid));
                        if (!gid_eq(gid, GLOBAL_ROOT_GID))
                                add_uevent_var(env, "DEVGID=%u", from_kgid(&init_user_ns, gid));
                        kfree(tmp);
                }
        }

        if (dev->type && dev->type->name)
                add_uevent_var(env, "DEVTYPE=%s", dev->type->name);

        /* Add "DRIVER=%s" variable if the device is bound to a driver */
        dev_driver_uevent(dev, env);

        /* Add common DT information about the device */
        of_device_uevent(dev, env);

        /* have the bus specific function add its stuff */
        if (dev->bus && dev->bus->uevent) {
                retval = dev->bus->uevent(dev, env);
                if (retval)
                        pr_debug("device: '%s': %s: bus uevent() returned %d\n",
                                 dev_name(dev), __func__, retval);
        }

        /* have the class specific function add its stuff */
        if (dev->class && dev->class->dev_uevent) {
                retval = dev->class->dev_uevent(dev, env);
                if (retval)
                        pr_debug("device: '%s': %s: class uevent() "
                                 "returned %d\n", dev_name(dev),
                                 __func__, retval);
        }

        /* have the device type specific function add its stuff */
        if (dev->type && dev->type->uevent) {
                retval = dev->type->uevent(dev, env);
                if (retval)
                        pr_debug("device: '%s': %s: dev_type uevent() "
                                 "returned %d\n", dev_name(dev),
                                 __func__, retval);
        }

        return retval;
}

static const struct kset_uevent_ops device_uevent_ops = {
        .filter =        dev_uevent_filter,
        .name =                dev_uevent_name,
        .uevent =        dev_uevent,
};

static ssize_t uevent_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        struct kobject *top_kobj;
        struct kset *kset;
        struct kobj_uevent_env *env = NULL;
        int i;
        int len = 0;
        int retval;

        /* search the kset, the device belongs to */
        top_kobj = &dev->kobj;
        while (!top_kobj->kset && top_kobj->parent)
                top_kobj = top_kobj->parent;
        if (!top_kobj->kset)
                goto out;

        kset = top_kobj->kset;
        if (!kset->uevent_ops || !kset->uevent_ops->uevent)
                goto out;

        /* respect filter */
        if (kset->uevent_ops && kset->uevent_ops->filter)
                if (!kset->uevent_ops->filter(&dev->kobj))
                        goto out;

        env = kzalloc(sizeof(struct kobj_uevent_env), GFP_KERNEL);
        if (!env)
                return -ENOMEM;

        /* let the kset specific function add its keys */
        retval = kset->uevent_ops->uevent(&dev->kobj, env);
        if (retval)
                goto out;

        /* copy keys to file */
        for (i = 0; i < env->envp_idx; i++)
                len += sysfs_emit_at(buf, len, "%s\n", env->envp[i]);
out:
        kfree(env);
        return len;
}

static ssize_t uevent_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t count)
{
        int rc;

        rc = kobject_synth_uevent(&dev->kobj, buf, count);

        if (rc) {
                dev_err(dev, "uevent: failed to send synthetic uevent: %d\n", rc);
                return rc;
        }

        return count;
}
static DEVICE_ATTR_RW(uevent);

static ssize_t online_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        bool val;

        device_lock(dev);
        val = !dev->offline;
        device_unlock(dev);
        return sysfs_emit(buf, "%u\n", val);
}

static ssize_t online_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t count)
{
        bool val;
        int ret;

        ret = kstrtobool(buf, &val);
        if (ret < 0)
                return ret;

        ret = lock_device_hotplug_sysfs();
        if (ret)
                return ret;

        ret = val ? device_online(dev) : device_offline(dev);
        unlock_device_hotplug();
        return ret < 0 ? ret : count;
}
static DEVICE_ATTR_RW(online);

static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
                              char *buf)
{
        const char *loc;

        switch (dev->removable) {
        case DEVICE_REMOVABLE:
                loc = "removable";
                break;
        case DEVICE_FIXED:
                loc = "fixed";
                break;
        default:
                loc = "unknown";
        }
        return sysfs_emit(buf, "%s\n", loc);
}
static DEVICE_ATTR_RO(removable);

int device_add_groups(struct device *dev, const struct attribute_group **groups)
{
        return sysfs_create_groups(&dev->kobj, groups);
}
EXPORT_SYMBOL_GPL(device_add_groups);

void device_remove_groups(struct device *dev,
                          const struct attribute_group **groups)
{
        sysfs_remove_groups(&dev->kobj, groups);
}
EXPORT_SYMBOL_GPL(device_remove_groups);

union device_attr_group_devres {
        const struct attribute_group *group;
        const struct attribute_group **groups;
};

static void devm_attr_group_remove(struct device *dev, void *res)
{
        union device_attr_group_devres *devres = res;
        const struct attribute_group *group = devres->group;

        dev_dbg(dev, "%s: removing group %p\n", __func__, group);
        sysfs_remove_group(&dev->kobj, group);
}

/**
 * devm_device_add_group - given a device, create a managed attribute group
 * @dev:        The device to create the group for
 * @grp:        The attribute group to create
 *
 * This function creates a group for the first time.  It will explicitly
 * warn and error if any of the attribute files being created already exist.
 *
 * Returns 0 on success or error code on failure.
 */
int devm_device_add_group(struct device *dev, const struct attribute_group *grp)
{
        union device_attr_group_devres *devres;
        int error;

        devres = devres_alloc(devm_attr_group_remove,
                              sizeof(*devres), GFP_KERNEL);
        if (!devres)
                return -ENOMEM;

        error = sysfs_create_group(&dev->kobj, grp);
        if (error) {
                devres_free(devres);
                return error;
        }

        devres->group = grp;
        devres_add(dev, devres);
        return 0;
}
EXPORT_SYMBOL_GPL(devm_device_add_group);

static int device_add_attrs(struct device *dev)
{
        const struct class *class = dev->class;
        const struct device_type *type = dev->type;
        int error;

        if (class) {
                error = device_add_groups(dev, class->dev_groups);
                if (error)
                        return error;
        }

        if (type) {
                error = device_add_groups(dev, type->groups);
                if (error)
                        goto err_remove_class_groups;
        }

        error = device_add_groups(dev, dev->groups);
        if (error)
                goto err_remove_type_groups;

        if (device_supports_offline(dev) && !dev->offline_disabled) {
                error = device_create_file(dev, &dev_attr_online);
                if (error)
                        goto err_remove_dev_groups;
        }

        if (fw_devlink_flags && !fw_devlink_is_permissive() && dev->fwnode) {
                error = device_create_file(dev, &dev_attr_waiting_for_supplier);
                if (error)
                        goto err_remove_dev_online;
        }

        if (dev_removable_is_valid(dev)) {
                error = device_create_file(dev, &dev_attr_removable);
                if (error)
                        goto err_remove_dev_waiting_for_supplier;
        }

        if (dev_add_physical_location(dev)) {
                error = device_add_group(dev,
                        &dev_attr_physical_location_group);
                if (error)
                        goto err_remove_dev_removable;
        }

        return 0;

 err_remove_dev_removable:
        device_remove_file(dev, &dev_attr_removable);
 err_remove_dev_waiting_for_supplier:
        device_remove_file(dev, &dev_attr_waiting_for_supplier);
 err_remove_dev_online:
        device_remove_file(dev, &dev_attr_online);
 err_remove_dev_groups:
        device_remove_groups(dev, dev->groups);
 err_remove_type_groups:
        if (type)
                device_remove_groups(dev, type->groups);
 err_remove_class_groups:
        if (class)
                device_remove_groups(dev, class->dev_groups);

        return error;
}

static void device_remove_attrs(struct device *dev)
{
        const struct class *class = dev->class;
        const struct device_type *type = dev->type;

        if (dev->physical_location) {
                device_remove_group(dev, &dev_attr_physical_location_group);
                kfree(dev->physical_location);
        }

        device_remove_file(dev, &dev_attr_removable);
        device_remove_file(dev, &dev_attr_waiting_for_supplier);
        device_remove_file(dev, &dev_attr_online);
        device_remove_groups(dev, dev->groups);

        if (type)
                device_remove_groups(dev, type->groups);

        if (class)
                device_remove_groups(dev, class->dev_groups);
}

static ssize_t dev_show(struct device *dev, struct device_attribute *attr,
                        char *buf)
{
        return print_dev_t(buf, dev->devt);
}
static DEVICE_ATTR_RO(dev);

/* /sys/devices/ */
struct kset *devices_kset;

/**
 * devices_kset_move_before - Move device in the devices_kset's list.
 * @deva: Device to move.
 * @devb: Device @deva should come before.
 */
static void devices_kset_move_before(struct device *deva, struct device *devb)
{
        if (!devices_kset)
                return;
        pr_debug("devices_kset: Moving %s before %s\n",
                 dev_name(deva), dev_name(devb));
        spin_lock(&devices_kset->list_lock);
        list_move_tail(&deva->kobj.entry, &devb->kobj.entry);
        spin_unlock(&devices_kset->list_lock);
}

/**
 * devices_kset_move_after - Move device in the devices_kset's list.
 * @deva: Device to move
 * @devb: Device @deva should come after.
 */
static void devices_kset_move_after(struct device *deva, struct device *devb)
{
        if (!devices_kset)
                return;
        pr_debug("devices_kset: Moving %s after %s\n",
                 dev_name(deva), dev_name(devb));
        spin_lock(&devices_kset->list_lock);
        list_move(&deva->kobj.entry, &devb->kobj.entry);
        spin_unlock(&devices_kset->list_lock);
}

/**
 * devices_kset_move_last - move the device to the end of devices_kset's list.
 * @dev: device to move
 */
void devices_kset_move_last(struct device *dev)
{
        if (!devices_kset)
                return;
        pr_debug("devices_kset: Moving %s to end of list\n", dev_name(dev));
        spin_lock(&devices_kset->list_lock);
        list_move_tail(&dev->kobj.entry, &devices_kset->list);
        spin_unlock(&devices_kset->list_lock);
}

/**
 * device_create_file - create sysfs attribute file for device.
 * @dev: device.
 * @attr: device attribute descriptor.
 */
int device_create_file(struct device *dev,
                       const struct device_attribute *attr)
{
        int error = 0;

        if (dev) {
                WARN(((attr->attr.mode & S_IWUGO) && !attr->store),
                        "Attribute %s: write permission without 'store'\n",
                        attr->attr.name);
                WARN(((attr->attr.mode & S_IRUGO) && !attr->show),
                        "Attribute %s: read permission without 'show'\n",
                        attr->attr.name);
                error = sysfs_create_file(&dev->kobj, &attr->attr);
        }

        return error;
}
EXPORT_SYMBOL_GPL(device_create_file);

/**
 * device_remove_file - remove sysfs attribute file.
 * @dev: device.
 * @attr: device attribute descriptor.
 */
void device_remove_file(struct device *dev,
                        const struct device_attribute *attr)
{
        if (dev)
                sysfs_remove_file(&dev->kobj, &attr->attr);
}
EXPORT_SYMBOL_GPL(device_remove_file);

/**
 * device_remove_file_self - remove sysfs attribute file from its own method.
 * @dev: device.
 * @attr: device attribute descriptor.
 *
 * See kernfs_remove_self() for details.
 */
bool device_remove_file_self(struct device *dev,
                             const struct device_attribute *attr)
{
        if (dev)
                return sysfs_remove_file_self(&dev->kobj, &attr->attr);
        else
                return false;
}
EXPORT_SYMBOL_GPL(device_remove_file_self);

/**
 * device_create_bin_file - create sysfs binary attribute file for device.
 * @dev: device.
 * @attr: device binary attribute descriptor.
 */
int device_create_bin_file(struct device *dev,
                           const struct bin_attribute *attr)
{
        int error = -EINVAL;
        if (dev)
                error = sysfs_create_bin_file(&dev->kobj, attr);
        return error;
}
EXPORT_SYMBOL_GPL(device_create_bin_file);

/**
 * device_remove_bin_file - remove sysfs binary attribute file
 * @dev: device.
 * @attr: device binary attribute descriptor.
 */
void device_remove_bin_file(struct device *dev,
                            const struct bin_attribute *attr)
{
        if (dev)
                sysfs_remove_bin_file(&dev->kobj, attr);
}
EXPORT_SYMBOL_GPL(device_remove_bin_file);

static void klist_children_get(struct klist_node *n)
{
        struct device_private *p = to_device_private_parent(n);
        struct device *dev = p->device;

        get_device(dev);
}

static void klist_children_put(struct klist_node *n)
{
        struct device_private *p = to_device_private_parent(n);
        struct device *dev = p->device;

        put_device(dev);
}

/**
 * device_initialize - init device structure.
 * @dev: device.
 *
 * This prepares the device for use by other layers by initializing
 * its fields.
 * It is the first half of device_register(), if called by
 * that function, though it can also be called separately, so one
 * may use @dev's fields. In particular, get_device()/put_device()
 * may be used for reference counting of @dev after calling this
 * function.
 *
 * All fields in @dev must be initialized by the caller to 0, except
 * for those explicitly set to some other value.  The simplest
 * approach is to use kzalloc() to allocate the structure containing
 * @dev.
 *
 * NOTE: Use put_device() to give up your reference instead of freeing
 * @dev directly once you have called this function.
 */
void device_initialize(struct device *dev)
{
        dev->kobj.kset = devices_kset;
        kobject_init(&dev->kobj, &device_ktype);
        INIT_LIST_HEAD(&dev->dma_pools);
        mutex_init(&dev->mutex);
        lockdep_set_novalidate_class(&dev->mutex);
        spin_lock_init(&dev->devres_lock);
        INIT_LIST_HEAD(&dev->devres_head);
        device_pm_init(dev);
        set_dev_node(dev, NUMA_NO_NODE);
        INIT_LIST_HEAD(&dev->links.consumers);
        INIT_LIST_HEAD(&dev->links.suppliers);
        INIT_LIST_HEAD(&dev->links.defer_sync);
        dev->links.status = DL_DEV_NO_DRIVER;
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
        dev->dma_coherent = dma_default_coherent;
#endif
        swiotlb_dev_init(dev);
}
EXPORT_SYMBOL_GPL(device_initialize);

struct kobject *virtual_device_parent(void)
{
        static struct kobject *virtual_dir = NULL;

        if (!virtual_dir)
                virtual_dir = kobject_create_and_add("virtual",
                                                     &devices_kset->kobj);

        return virtual_dir;
}

struct class_dir {
        struct kobject kobj;
        const struct class *class;
};

#define to_class_dir(obj) container_of(obj, struct class_dir, kobj)

static void class_dir_release(struct kobject *kobj)
{
        struct class_dir *dir = to_class_dir(kobj);
        kfree(dir);
}

static const
struct kobj_ns_type_operations *class_dir_child_ns_type(const struct kobject *kobj)
{
        const struct class_dir *dir = to_class_dir(kobj);
        return dir->class->ns_type;
}

static const struct kobj_type class_dir_ktype = {
        .release        = class_dir_release,
        .sysfs_ops        = &kobj_sysfs_ops,
        .child_ns_type        = class_dir_child_ns_type
};

static struct kobject *class_dir_create_and_add(struct subsys_private *sp,
                                                struct kobject *parent_kobj)
{
        struct class_dir *dir;
        int retval;

        dir = kzalloc(sizeof(*dir), GFP_KERNEL);
        if (!dir)
                return ERR_PTR(-ENOMEM);

        dir->class = sp->class;
        kobject_init(&dir->kobj, &class_dir_ktype);

        dir->kobj.kset = &sp->glue_dirs;

        retval = kobject_add(&dir->kobj, parent_kobj, "%s", sp->class->name);
        if (retval < 0) {
                kobject_put(&dir->kobj);
                return ERR_PTR(retval);
        }
        return &dir->kobj;
}

static DEFINE_MUTEX(gdp_mutex);

static struct kobject *get_device_parent(struct device *dev,
                                         struct device *parent)
{
        struct subsys_private *sp = class_to_subsys(dev->class);
        struct kobject *kobj = NULL;

        if (sp) {
                struct kobject *parent_kobj;
                struct kobject *k;

                /*
                 * If we have no parent, we live in "virtual".
                 * Class-devices with a non class-device as parent, live
                 * in a "glue" directory to prevent namespace collisions.
                 */
                if (parent == NULL)
                        parent_kobj = virtual_device_parent();
                else if (parent->class && !dev->class->ns_type) {
                        subsys_put(sp);
                        return &parent->kobj;
                } else {
                        parent_kobj = &parent->kobj;
                }

                mutex_lock(&gdp_mutex);

                /* find our class-directory at the parent and reference it */
                spin_lock(&sp->glue_dirs.list_lock);
                list_for_each_entry(k, &sp->glue_dirs.list, entry)
                        if (k->parent == parent_kobj) {
                                kobj = kobject_get(k);
                                break;
                        }
                spin_unlock(&sp->glue_dirs.list_lock);
                if (kobj) {
                        mutex_unlock(&gdp_mutex);
                        subsys_put(sp);
                        return kobj;
                }

                /* or create a new class-directory at the parent device */
                k = class_dir_create_and_add(sp, parent_kobj);
                /* do not emit an uevent for this simple "glue" directory */
                mutex_unlock(&gdp_mutex);
                subsys_put(sp);
                return k;
        }

        /* subsystems can specify a default root directory for their devices */
        if (!parent && dev->bus) {
                struct device *dev_root = bus_get_dev_root(dev->bus);

                if (dev_root) {
                        kobj = &dev_root->kobj;
                        put_device(dev_root);
                        return kobj;
                }
        }

        if (parent)
                return &parent->kobj;
        return NULL;
}

static inline bool live_in_glue_dir(struct kobject *kobj,
                                    struct device *dev)
{
        struct subsys_private *sp;
        bool retval;

        if (!kobj || !dev->class)
                return false;

        sp = class_to_subsys(dev->class);
        if (!sp)
                return false;

        if (kobj->kset == &sp->glue_dirs)
                retval = true;
        else
                retval = false;

        subsys_put(sp);
        return retval;
}

static inline struct kobject *get_glue_dir(struct device *dev)
{
        return dev->kobj.parent;
}

/**
 * kobject_has_children - Returns whether a kobject has children.
 * @kobj: the object to test
 *
 * This will return whether a kobject has other kobjects as children.
 *
 * It does NOT account for the presence of attribute files, only sub
 * directories. It also assumes there is no concurrent addition or
 * removal of such children, and thus relies on external locking.
 */
static inline bool kobject_has_children(struct kobject *kobj)
{
        WARN_ON_ONCE(kref_read(&kobj->kref) == 0);

        return kobj->sd && kobj->sd->dir.subdirs;
}

/*
 * make sure cleaning up dir as the last step, we need to make
 * sure .release handler of kobject is run with holding the
 * global lock
 */
static void cleanup_glue_dir(struct device *dev, struct kobject *glue_dir)
{
        unsigned int ref;

        /* see if we live in a "glue" directory */
        if (!live_in_glue_dir(glue_dir, dev))
                return;

        mutex_lock(&gdp_mutex);
        /**
         * There is a race condition between removing glue directory
         * and adding a new device under the glue directory.
         *
         * CPU1:                                         CPU2:
         *
         * device_add()
         *   get_device_parent()
         *     class_dir_create_and_add()
         *       kobject_add_internal()
         *         create_dir()    // create glue_dir
         *
         *                                               device_add()
         *                                                 get_device_parent()
         *                                                   kobject_get() // get glue_dir
         *
         * device_del()
         *   cleanup_glue_dir()
         *     kobject_del(glue_dir)
         *
         *                                               kobject_add()
         *                                                 kobject_add_internal()
         *                                                   create_dir() // in glue_dir
         *                                                     sysfs_create_dir_ns()
         *                                                       kernfs_create_dir_ns(sd)
         *
         *       sysfs_remove_dir() // glue_dir->sd=NULL
         *       sysfs_put()        // free glue_dir->sd
         *
         *                                                         // sd is freed
         *                                                         kernfs_new_node(sd)
         *                                                           kernfs_get(glue_dir)
         *                                                           kernfs_add_one()
         *                                                           kernfs_put()
         *
         * Before CPU1 remove last child device under glue dir, if CPU2 add
         * a new device under glue dir, the glue_dir kobject reference count
         * will be increase to 2 in kobject_get(k). And CPU2 has been called
         * kernfs_create_dir_ns(). Meanwhile, CPU1 call sysfs_remove_dir()
         * and sysfs_put(). This result in glue_dir->sd is freed.
         *
         * Then the CPU2 will see a stale "empty" but still potentially used
         * glue dir around in kernfs_new_node().
         *
         * In order to avoid this happening, we also should make sure that
         * kernfs_node for glue_dir is released in CPU1 only when refcount
         * for glue_dir kobj is 1.
         */
        ref = kref_read(&glue_dir->kref);
        if (!kobject_has_children(glue_dir) && !--ref)
                kobject_del(glue_dir);
        kobject_put(glue_dir);
        mutex_unlock(&gdp_mutex);
}

static int device_add_class_symlinks(struct device *dev)
{
        struct device_node *of_node = dev_of_node(dev);
        struct subsys_private *sp;
        int error;

        if (of_node) {
                error = sysfs_create_link(&dev->kobj, of_node_kobj(of_node), "of_node");
                if (error)
                        dev_warn(dev, "Error %d creating of_node link\n",error);
                /* An error here doesn't warrant bringing down the device */
        }

        sp = class_to_subsys(dev->class);
        if (!sp)
                return 0;

        error = sysfs_create_link(&dev->kobj, &sp->subsys.kobj, "subsystem");
        if (error)
                goto out_devnode;

        if (dev->parent && device_is_not_partition(dev)) {
                error = sysfs_create_link(&dev->kobj, &dev->parent->kobj,
                                          "device");
                if (error)
                        goto out_subsys;
        }

        /* link in the class directory pointing to the device */
        error = sysfs_create_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
        if (error)
                goto out_device;
        goto exit;

out_device:
        sysfs_remove_link(&dev->kobj, "device");
out_subsys:
        sysfs_remove_link(&dev->kobj, "subsystem");
out_devnode:
        sysfs_remove_link(&dev->kobj, "of_node");
exit:
        subsys_put(sp);
        return error;
}

static void device_remove_class_symlinks(struct device *dev)
{
        struct subsys_private *sp = class_to_subsys(dev->class);

        if (dev_of_node(dev))
                sysfs_remove_link(&dev->kobj, "of_node");

        if (!sp)
                return;

        if (dev->parent && device_is_not_partition(dev))
                sysfs_remove_link(&dev->kobj, "device");
        sysfs_remove_link(&dev->kobj, "subsystem");
        sysfs_delete_link(&sp->subsys.kobj, &dev->kobj, dev_name(dev));
        subsys_put(sp);
}

/**
 * dev_set_name - set a device name
 * @dev: device
 * @fmt: format string for the device's name
 */
int dev_set_name(struct device *dev, const char *fmt, ...)
{
        va_list vargs;
        int err;

        va_start(vargs, fmt);
        err = kobject_set_name_vargs(&dev->kobj, fmt, vargs);
        va_end(vargs);
        return err;
}
EXPORT_SYMBOL_GPL(dev_set_name);

/* select a /sys/dev/ directory for the device */
static struct kobject *device_to_dev_kobj(struct device *dev)
{
        if (is_blockdev(dev))
                return sysfs_dev_block_kobj;
        else
                return sysfs_dev_char_kobj;
}

static int device_create_sys_dev_entry(struct device *dev)
{
        struct kobject *kobj = device_to_dev_kobj(dev);
        int error = 0;
        char devt_str[15];

        if (kobj) {
                format_dev_t(devt_str, dev->devt);
                error = sysfs_create_link(kobj, &dev->kobj, devt_str);
        }

        return error;
}

static void device_remove_sys_dev_entry(struct device *dev)
{
        struct kobject *kobj = device_to_dev_kobj(dev);
        char devt_str[15];

        if (kobj) {
                format_dev_t(devt_str, dev->devt);
                sysfs_remove_link(kobj, devt_str);
        }
}

static int device_private_init(struct device *dev)
{
        dev->p = kzalloc(sizeof(*dev->p), GFP_KERNEL);
        if (!dev->p)
                return -ENOMEM;
        dev->p->device = dev;
        klist_init(&dev->p->klist_children, klist_children_get,
                   klist_children_put);
        INIT_LIST_HEAD(&dev->p->deferred_probe);
        return 0;
}

/**
 * device_add - add device to device hierarchy.
 * @dev: device.
 *
 * This is part 2 of device_register(), though may be called
 * separately _iff_ device_initialize() has been called separately.
 *
 * This adds @dev to the kobject hierarchy via kobject_add(), adds it
 * to the global and sibling lists for the device, then
 * adds it to the other relevant subsystems of the driver model.
 *
 * Do not call this routine or device_register() more than once for
 * any device structure.  The driver model core is not designed to work
 * with devices that get unregistered and then spring back to life.
 * (Among other things, it's very hard to guarantee that all references
 * to the previous incarnation of @dev have been dropped.)  Allocate
 * and register a fresh new struct device instead.
 *
 * NOTE: _Never_ directly free @dev after calling this function, even
 * if it returned an error! Always use put_device() to give up your
 * reference instead.
 *
 * Rule of thumb is: if device_add() succeeds, you should call
 * device_del() when you want to get rid of it. If device_add() has
 * *not* succeeded, use *only* put_device() to drop the reference
 * count.
 */
int device_add(struct device *dev)
{
        struct subsys_private *sp;
        struct device *parent;
        struct kobject *kobj;
        struct class_interface *class_intf;
        int error = -EINVAL;
        struct kobject *glue_dir = NULL;

        dev = get_device(dev);
        if (!dev)
                goto done;

        if (!dev->p) {
                error = device_private_init(dev);
                if (error)
                        goto done;
        }

        /*
         * for statically allocated devices, which should all be converted
         * some day, we need to initialize the name. We prevent reading back
         * the name, and force the use of dev_name()
         */
        if (dev->init_name) {
                error = dev_set_name(dev, "%s", dev->init_name);
                dev->init_name = NULL;
        }

        if (dev_name(dev))
                error = 0;
        /* subsystems can specify simple device enumeration */
        else if (dev->bus && dev->bus->dev_name)
                error = dev_set_name(dev, "%s%u", dev->bus->dev_name, dev->id);
        else
                error = -EINVAL;
        if (error)
                goto name_error;

        pr_debug("device: '%s': %s\n", dev_name(dev), __func__);

        parent = get_device(dev->parent);
        kobj = get_device_parent(dev, parent);
        if (IS_ERR(kobj)) {
                error = PTR_ERR(kobj);
                goto parent_error;
        }
        if (kobj)
                dev->kobj.parent = kobj;

        /* use parent numa_node */
        if (parent && (dev_to_node(dev) == NUMA_NO_NODE))
                set_dev_node(dev, dev_to_node(parent));

        /* first, register with generic layer. */
        /* we require the name to be set before, and pass NULL */
        error = kobject_add(&dev->kobj, dev->kobj.parent, NULL);
        if (error) {
                glue_dir = kobj;
                goto Error;
        }

        /* notify platform of device entry */
        device_platform_notify(dev);

        error = device_create_file(dev, &dev_attr_uevent);
        if (error)
                goto attrError;

        error = device_add_class_symlinks(dev);
        if (error)
                goto SymlinkError;
        error = device_add_attrs(dev);
        if (error)
                goto AttrsError;
        error = bus_add_device(dev);
        if (error)
                goto BusError;
        error = dpm_sysfs_add(dev);
        if (error)
                goto DPMError;
        device_pm_add(dev);

        if (MAJOR(dev->devt)) {
                error = device_create_file(dev, &dev_attr_dev);
                if (error)
                        goto DevAttrError;

                error = device_create_sys_dev_entry(dev);
                if (error)
                        goto SysEntryError;

                devtmpfs_create_node(dev);
        }

        /* Notify clients of device addition.  This call must come
         * after dpm_sysfs_add() and before kobject_uevent().
         */
        bus_notify(dev, BUS_NOTIFY_ADD_DEVICE);
        kobject_uevent(&dev->kobj, KOBJ_ADD);

        /*
         * Check if any of the other devices (consumers) have been waiting for
         * this device (supplier) to be added so that they can create a device
         * link to it.
         *
         * This needs to happen after device_pm_add() because device_link_add()
         * requires the supplier be registered before it's called.
         *
         * But this also needs to happen before bus_probe_device() to make sure
         * waiting consumers can link to it before the driver is bound to the
         * device and the driver sync_state callback is called for this device.
         */
        if (dev->fwnode && !dev->fwnode->dev) {
                dev->fwnode->dev = dev;
                fw_devlink_link_device(dev);
        }

        bus_probe_device(dev);

        /*
         * If all driver registration is done and a newly added device doesn't
         * match with any driver, don't block its consumers from probing in
         * case the consumer device is able to operate without this supplier.
         */
        if (dev->fwnode && fw_devlink_drv_reg_done && !dev->can_match)
                fw_devlink_unblock_consumers(dev);

        if (parent)
                klist_add_tail(&dev->p->knode_parent,
                               &parent->p->klist_children);

        sp = class_to_subsys(dev->class);
        if (sp) {
                mutex_lock(&sp->mutex);
                /* tie the class to the device */
                klist_add_tail(&dev->p->knode_class, &sp->klist_devices);

                /* notify any interfaces that the device is here */
                list_for_each_entry(class_intf, &sp->interfaces, node)
                        if (class_intf->add_dev)
                                class_intf->add_dev(dev);
                mutex_unlock(&sp->mutex);
                subsys_put(sp);
        }
done:
        put_device(dev);
        return error;
 SysEntryError:
        if (MAJOR(dev->devt))
                device_remove_file(dev, &dev_attr_dev);
 DevAttrError:
        device_pm_remove(dev);
        dpm_sysfs_remove(dev);
 DPMError:
        device_set_driver(dev, NULL);
        bus_remove_device(dev);
 BusError:
        device_remove_attrs(dev);
 AttrsError:
        device_remove_class_symlinks(dev);
 SymlinkError:
        device_remove_file(dev, &dev_attr_uevent);
 attrError:
        device_platform_notify_remove(dev);
        kobject_uevent(&dev->kobj, KOBJ_REMOVE);
        glue_dir = get_glue_dir(dev);
        kobject_del(&dev->kobj);
 Error:
        cleanup_glue_dir(dev, glue_dir);
parent_error:
        put_device(parent);
name_error:
        kfree(dev->p);
        dev->p = NULL;
        goto done;
}
EXPORT_SYMBOL_GPL(device_add);

/**
 * device_register - register a device with the system.
 * @dev: pointer to the device structure
 *
 * This happens in two clean steps - initialize the device
 * and add it to the system. The two steps can be called
 * separately, but this is the easiest and most common.
 * I.e. you should only call the two helpers separately if
 * have a clearly defined need to use and refcount the device
 * before it is added to the hierarchy.
 *
 * For more information, see the kerneldoc for device_initialize()
 * and device_add().
 *
 * NOTE: _Never_ directly free @dev after calling this function, even
 * if it returned an error! Always use put_device() to give up the
 * reference initialized in this function instead.
 */
int device_register(struct device *dev)
{
        device_initialize(dev);
        return device_add(dev);
}
EXPORT_SYMBOL_GPL(device_register);

/**
 * get_device - increment reference count for device.
 * @dev: device.
 *
 * This simply forwards the call to kobject_get(), though
 * we do take care to provide for the case that we get a NULL
 * pointer passed in.
 */
struct device *get_device(struct device *dev)
{
        return dev ? kobj_to_dev(kobject_get(&dev->kobj)) : NULL;
}
EXPORT_SYMBOL_GPL(get_device);

/**
 * put_device - decrement reference count.
 * @dev: device in question.
 */
void put_device(struct device *dev)
{
        /* might_sleep(); */
        if (dev)
                kobject_put(&dev->kobj);
}
EXPORT_SYMBOL_GPL(put_device);

bool kill_device(struct device *dev)
{
        /*
         * Require the device lock and set the "dead" flag to guarantee that
         * the update behavior is consistent with the other bitfields near
         * it and that we cannot have an asynchronous probe routine trying
         * to run while we are tearing out the bus/class/sysfs from
         * underneath the device.
         */
        device_lock_assert(dev);

        if (dev->p->dead)
                return false;
        dev->p->dead = true;
        return true;
}
EXPORT_SYMBOL_GPL(kill_device);

/**
 * device_del - delete device from system.
 * @dev: device.
 *
 * This is the first part of the device unregistration
 * sequence. This removes the device from the lists we control
 * from here, has it removed from the other driver model
 * subsystems it was added to in device_add(), and removes it
 * from the kobject hierarchy.
 *
 * NOTE: this should be called manually _iff_ device_add() was
 * also called manually.
 */
void device_del(struct device *dev)
{
        struct subsys_private *sp;
        struct device *parent = dev->parent;
        struct kobject *glue_dir = NULL;
        struct class_interface *class_intf;
        unsigned int noio_flag;

        device_lock(dev);
        kill_device(dev);
        device_unlock(dev);

        if (dev->fwnode && dev->fwnode->dev == dev)
                dev->fwnode->dev = NULL;

        /* Notify clients of device removal.  This call must come
         * before dpm_sysfs_remove().
         */
        noio_flag = memalloc_noio_save();
        bus_notify(dev, BUS_NOTIFY_DEL_DEVICE);

        dpm_sysfs_remove(dev);
        if (parent)
                klist_del(&dev->p->knode_parent);
        if (MAJOR(dev->devt)) {
                devtmpfs_delete_node(dev);
                device_remove_sys_dev_entry(dev);
                device_remove_file(dev, &dev_attr_dev);
        }

        sp = class_to_subsys(dev->class);
        if (sp) {
                device_remove_class_symlinks(dev);

                mutex_lock(&sp->mutex);
                /* notify any interfaces that the device is now gone */
                list_for_each_entry(class_intf, &sp->interfaces, node)
                        if (class_intf->remove_dev)
                                class_intf->remove_dev(dev);
                /* remove the device from the class list */
                klist_del(&dev->p->knode_class);
                mutex_unlock(&sp->mutex);
                subsys_put(sp);
        }
        device_remove_file(dev, &dev_attr_uevent);
        device_remove_attrs(dev);
        bus_remove_device(dev);
        device_pm_remove(dev);
        driver_deferred_probe_del(dev);
        device_platform_notify_remove(dev);
        device_links_purge(dev);

        /*
         * If a device does not have a driver attached, we need to clean
         * up any managed resources. We do this in device_release(), but
         * it's never called (and we leak the device) if a managed
         * resource holds a reference to the device. So release all
         * managed resources here, like we do in driver_detach(). We
         * still need to do so again in device_release() in case someone
         * adds a new resource after this point, though.
         */
        devres_release_all(dev);

        bus_notify(dev, BUS_NOTIFY_REMOVED_DEVICE);
        kobject_uevent(&dev->kobj, KOBJ_REMOVE);
        glue_dir = get_glue_dir(dev);
        kobject_del(&dev->kobj);
        cleanup_glue_dir(dev, glue_dir);
        memalloc_noio_restore(noio_flag);
        put_device(parent);
}
EXPORT_SYMBOL_GPL(device_del);

/**
 * device_unregister - unregister device from system.
 * @dev: device going away.
 *
 * We do this in two parts, like we do device_register(). First,
 * we remove it from all the subsystems with device_del(), then
 * we decrement the reference count via put_device(). If that
 * is the final reference count, the device will be cleaned up
 * via device_release() above. Otherwise, the structure will
 * stick around until the final reference to the device is dropped.
 */
void device_unregister(struct device *dev)
{
        pr_debug("device: '%s': %s\n", dev_name(dev), __func__);
        device_del(dev);
        put_device(dev);
}
EXPORT_SYMBOL_GPL(device_unregister);

static struct device *prev_device(struct klist_iter *i)
{
        struct klist_node *n = klist_prev(i);
        struct device *dev = NULL;
        struct device_private *p;

        if (n) {
                p = to_device_private_parent(n);
                dev = p->device;
        }
        return dev;
}

static struct device *next_device(struct klist_iter *i)
{
        struct klist_node *n = klist_next(i);
        struct device *dev = NULL;
        struct device_private *p;

        if (n) {
                p = to_device_private_parent(n);
                dev = p->device;
        }
        return dev;
}

/**
 * device_get_devnode - path of device node file
 * @dev: device
 * @mode: returned file access mode
 * @uid: returned file owner
 * @gid: returned file group
 * @tmp: possibly allocated string
 *
 * Return the relative path of a possible device node.
 * Non-default names may need to allocate a memory to compose
 * a name. This memory is returned in tmp and needs to be
 * freed by the caller.
 */
const char *device_get_devnode(const struct device *dev,
                               umode_t *mode, kuid_t *uid, kgid_t *gid,
                               const char **tmp)
{
        char *s;

        *tmp = NULL;

        /* the device type may provide a specific name */
        if (dev->type && dev->type->devnode)
                *tmp = dev->type->devnode(dev, mode, uid, gid);
        if (*tmp)
                return *tmp;

        /* the class may provide a specific name */
        if (dev->class && dev->class->devnode)
                *tmp = dev->class->devnode(dev, mode);
        if (*tmp)
                return *tmp;

        /* return name without allocation, tmp == NULL */
        if (strchr(dev_name(dev), '!') == NULL)
                return dev_name(dev);

        /* replace '!' in the name with '/' */
        s = kstrdup_and_replace(dev_name(dev), '!', '/', GFP_KERNEL);
        if (!s)
                return NULL;
        return *tmp = s;
}

/**
 * device_for_each_child - device child iterator.
 * @parent: parent struct device.
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over @parent's child devices, and call @fn for each,
 * passing it @data.
 *
 * We check the return of @fn each time. If it returns anything
 * other than 0, we break out and return that value.
 */
int device_for_each_child(struct device *parent, void *data,
                          device_iter_t fn)
{
        struct klist_iter i;
        struct device *child;
        int error = 0;

        if (!parent || !parent->p)
                return 0;

        klist_iter_init(&parent->p->klist_children, &i);
        while (!error && (child = next_device(&i)))
                error = fn(child, data);
        klist_iter_exit(&i);
        return error;
}
EXPORT_SYMBOL_GPL(device_for_each_child);

/**
 * device_for_each_child_reverse - device child iterator in reversed order.
 * @parent: parent struct device.
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over @parent's child devices, and call @fn for each,
 * passing it @data.
 *
 * We check the return of @fn each time. If it returns anything
 * other than 0, we break out and return that value.
 */
int device_for_each_child_reverse(struct device *parent, void *data,
                                  device_iter_t fn)
{
        struct klist_iter i;
        struct device *child;
        int error = 0;

        if (!parent || !parent->p)
                return 0;

        klist_iter_init(&parent->p->klist_children, &i);
        while ((child = prev_device(&i)) && !error)
                error = fn(child, data);
        klist_iter_exit(&i);
        return error;
}
EXPORT_SYMBOL_GPL(device_for_each_child_reverse);

/**
 * device_for_each_child_reverse_from - device child iterator in reversed order.
 * @parent: parent struct device.
 * @from: optional starting point in child list
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over @parent's child devices, starting at @from, and call @fn
 * for each, passing it @data. This helper is identical to
 * device_for_each_child_reverse() when @from is NULL.
 *
 * @fn is checked each iteration. If it returns anything other than 0,
 * iteration stop and that value is returned to the caller of
 * device_for_each_child_reverse_from();
 */
int device_for_each_child_reverse_from(struct device *parent,
                                       struct device *from, void *data,
                                       device_iter_t fn)
{
        struct klist_iter i;
        struct device *child;
        int error = 0;

        if (!parent || !parent->p)
                return 0;

        klist_iter_init_node(&parent->p->klist_children, &i,
                             (from ? &from->p->knode_parent : NULL));
        while ((child = prev_device(&i)) && !error)
                error = fn(child, data);
        klist_iter_exit(&i);
        return error;
}
EXPORT_SYMBOL_GPL(device_for_each_child_reverse_from);

/**
 * device_find_child - device iterator for locating a particular device.
 * @parent: parent struct device
 * @data: Data to pass to match function
 * @match: Callback function to check device
 *
 * This is similar to the device_for_each_child() function above, but it
 * returns a reference to a device that is 'found' for later use, as
 * determined by the @match callback.
 *
 * The callback should return 0 if the device doesn't match and non-zero
 * if it does.  If the callback returns non-zero and a reference to the
 * current device can be obtained, this function will return to the caller
 * and not iterate over any more devices.
 *
 * NOTE: you will need to drop the reference with put_device() after use.
 */
struct device *device_find_child(struct device *parent, const void *data,
                                 device_match_t match)
{
        struct klist_iter i;
        struct device *child;

        if (!parent || !parent->p)
                return NULL;

        klist_iter_init(&parent->p->klist_children, &i);
        while ((child = next_device(&i))) {
                if (match(child, data)) {
                        get_device(child);
                        break;
                }
        }
        klist_iter_exit(&i);
        return child;
}
EXPORT_SYMBOL_GPL(device_find_child);

int __init devices_init(void)
{
        devices_kset = kset_create_and_add("devices", &device_uevent_ops, NULL);
        if (!devices_kset)
                return -ENOMEM;
        dev_kobj = kobject_create_and_add("dev", NULL);
        if (!dev_kobj)
                goto dev_kobj_err;
        sysfs_dev_block_kobj = kobject_create_and_add("block", dev_kobj);
        if (!sysfs_dev_block_kobj)
                goto block_kobj_err;
        sysfs_dev_char_kobj = kobject_create_and_add("char", dev_kobj);
        if (!sysfs_dev_char_kobj)
                goto char_kobj_err;
        device_link_wq = alloc_workqueue("device_link_wq", 0, 0);
        if (!device_link_wq)
                goto wq_err;

        return 0;

 wq_err:
        kobject_put(sysfs_dev_char_kobj);
 char_kobj_err:
        kobject_put(sysfs_dev_block_kobj);
 block_kobj_err:
        kobject_put(dev_kobj);
 dev_kobj_err:
        kset_unregister(devices_kset);
        return -ENOMEM;
}

static int device_check_offline(struct device *dev, void *not_used)
{
        int ret;

        ret = device_for_each_child(dev, NULL, device_check_offline);
        if (ret)
                return ret;

        return device_supports_offline(dev) && !dev->offline ? -EBUSY : 0;
}

/**
 * device_offline - Prepare the device for hot-removal.
 * @dev: Device to be put offline.
 *
 * Execute the device bus type's .offline() callback, if present, to prepare
 * the device for a subsequent hot-removal.  If that succeeds, the device must
 * not be used until either it is removed or its bus type's .online() callback
 * is executed.
 *
 * Call under device_hotplug_lock.
 */
int device_offline(struct device *dev)
{
        int ret;

        if (dev->offline_disabled)
                return -EPERM;

        ret = device_for_each_child(dev, NULL, device_check_offline);
        if (ret)
                return ret;

        device_lock(dev);
        if (device_supports_offline(dev)) {
                if (dev->offline) {
                        ret = 1;
                } else {
                        ret = dev->bus->offline(dev);
                        if (!ret) {
                                kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
                                dev->offline = true;
                        }
                }
        }
        device_unlock(dev);

        return ret;
}

/**
 * device_online - Put the device back online after successful device_offline().
 * @dev: Device to be put back online.
 *
 * If device_offline() has been successfully executed for @dev, but the device
 * has not been removed subsequently, execute its bus type's .online() callback
 * to indicate that the device can be used again.
 *
 * Call under device_hotplug_lock.
 */
int device_online(struct device *dev)
{
        int ret = 0;

        device_lock(dev);
        if (device_supports_offline(dev)) {
                if (dev->offline) {
                        ret = dev->bus->online(dev);
                        if (!ret) {
                                kobject_uevent(&dev->kobj, KOBJ_ONLINE);
                                dev->offline = false;
                        }
                } else {
                        ret = 1;
                }
        }
        device_unlock(dev);

        return ret;
}

struct root_device {
        struct device dev;
        struct module *owner;
};

static inline struct root_device *to_root_device(struct device *d)
{
        return container_of(d, struct root_device, dev);
}

static void root_device_release(struct device *dev)
{
        kfree(to_root_device(dev));
}

/**
 * __root_device_register - allocate and register a root device
 * @name: root device name
 * @owner: owner module of the root device, usually THIS_MODULE
 *
 * This function allocates a root device and registers it
 * using device_register(). In order to free the returned
 * device, use root_device_unregister().
 *
 * Root devices are dummy devices which allow other devices
 * to be grouped under /sys/devices. Use this function to
 * allocate a root device and then use it as the parent of
 * any device which should appear under /sys/devices/{name}
 *
 * The /sys/devices/{name} directory will also contain a
 * 'module' symlink which points to the @owner directory
 * in sysfs.
 *
 * Returns &struct device pointer on success, or ERR_PTR() on error.
 *
 * Note: You probably want to use root_device_register().
 */
struct device *__root_device_register(const char *name, struct module *owner)
{
        struct root_device *root;
        int err = -ENOMEM;

        root = kzalloc(sizeof(struct root_device), GFP_KERNEL);
        if (!root)
                return ERR_PTR(err);

        err = dev_set_name(&root->dev, "%s", name);
        if (err) {
                kfree(root);
                return ERR_PTR(err);
        }

        root->dev.release = root_device_release;

        err = device_register(&root->dev);
        if (err) {
                put_device(&root->dev);
                return ERR_PTR(err);
        }

#ifdef CONFIG_MODULES        /* gotta find a "cleaner" way to do this */
        if (owner) {
                struct module_kobject *mk = &owner->mkobj;

                err = sysfs_create_link(&root->dev.kobj, &mk->kobj, "module");
                if (err) {
                        device_unregister(&root->dev);
                        return ERR_PTR(err);
                }
                root->owner = owner;
        }
#endif

        return &root->dev;
}
EXPORT_SYMBOL_GPL(__root_device_register);

/**
 * root_device_unregister - unregister and free a root device
 * @dev: device going away
 *
 * This function unregisters and cleans up a device that was created by
 * root_device_register().
 */
void root_device_unregister(struct device *dev)
{
        struct root_device *root = to_root_device(dev);

        if (root->owner)
                sysfs_remove_link(&root->dev.kobj, "module");

        device_unregister(dev);
}
EXPORT_SYMBOL_GPL(root_device_unregister);


static void device_create_release(struct device *dev)
{
        pr_debug("device: '%s': %s\n", dev_name(dev), __func__);
        kfree(dev);
}

static __printf(6, 0) struct device *
device_create_groups_vargs(const struct class *class, struct device *parent,
                           dev_t devt, void *drvdata,
                           const struct attribute_group **groups,
                           const char *fmt, va_list args)
{
        struct device *dev = NULL;
        int retval = -ENODEV;

        if (IS_ERR_OR_NULL(class))
                goto error;

        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
        if (!dev) {
                retval = -ENOMEM;
                goto error;
        }

        device_initialize(dev);
        dev->devt = devt;
        dev->class = class;
        dev->parent = parent;
        dev->groups = groups;
        dev->release = device_create_release;
        dev_set_drvdata(dev, drvdata);

        retval = kobject_set_name_vargs(&dev->kobj, fmt, args);
        if (retval)
                goto error;

        retval = device_add(dev);
        if (retval)
                goto error;

        return dev;

error:
        put_device(dev);
        return ERR_PTR(retval);
}

/**
 * device_create - creates a device and registers it with sysfs
 * @class: pointer to the struct class that this device should be registered to
 * @parent: pointer to the parent struct device of this new device, if any
 * @devt: the dev_t for the char device to be added
 * @drvdata: the data to be added to the device for callbacks
 * @fmt: string for the device's name
 *
 * This function can be used by char device classes.  A struct device
 * will be created in sysfs, registered to the specified class.
 *
 * A "dev" file will be created, showing the dev_t for the device, if
 * the dev_t is not 0,0.
 * If a pointer to a parent struct device is passed in, the newly created
 * struct device will be a child of that device in sysfs.
 * The pointer to the struct device will be returned from the call.
 * Any further sysfs files that might be required can be created using this
 * pointer.
 *
 * Returns &struct device pointer on success, or ERR_PTR() on error.
 */
struct device *device_create(const struct class *class, struct device *parent,
                             dev_t devt, void *drvdata, const char *fmt, ...)
{
        va_list vargs;
        struct device *dev;

        va_start(vargs, fmt);
        dev = device_create_groups_vargs(class, parent, devt, drvdata, NULL,
                                          fmt, vargs);
        va_end(vargs);
        return dev;
}
EXPORT_SYMBOL_GPL(device_create);

/**
 * device_create_with_groups - creates a device and registers it with sysfs
 * @class: pointer to the struct class that this device should be registered to
 * @parent: pointer to the parent struct device of this new device, if any
 * @devt: the dev_t for the char device to be added
 * @drvdata: the data to be added to the device for callbacks
 * @groups: NULL-terminated list of attribute groups to be created
 * @fmt: string for the device's name
 *
 * This function can be used by char device classes.  A struct device
 * will be created in sysfs, registered to the specified class.
 * Additional attributes specified in the groups parameter will also
 * be created automatically.
 *
 * A "dev" file will be created, showing the dev_t for the device, if
 * the dev_t is not 0,0.
 * If a pointer to a parent struct device is passed in, the newly created
 * struct device will be a child of that device in sysfs.
 * The pointer to the struct device will be returned from the call.
 * Any further sysfs files that might be required can be created using this
 * pointer.
 *
 * Returns &struct device pointer on success, or ERR_PTR() on error.
 */
struct device *device_create_with_groups(const struct class *class,
                                         struct device *parent, dev_t devt,
                                         void *drvdata,
                                         const struct attribute_group **groups,
                                         const char *fmt, ...)
{
        va_list vargs;
        struct device *dev;

        va_start(vargs, fmt);
        dev = device_create_groups_vargs(class, parent, devt, drvdata, groups,
                                         fmt, vargs);
        va_end(vargs);
        return dev;
}
EXPORT_SYMBOL_GPL(device_create_with_groups);

/**
 * device_destroy - removes a device that was created with device_create()
 * @class: pointer to the struct class that this device was registered with
 * @devt: the dev_t of the device that was previously registered
 *
 * This call unregisters and cleans up a device that was created with a
 * call to device_create().
 */
void device_destroy(const struct class *class, dev_t devt)
{
        struct device *dev;

        dev = class_find_device_by_devt(class, devt);
        if (dev) {
                put_device(dev);
                device_unregister(dev);
        }
}
EXPORT_SYMBOL_GPL(device_destroy);

/**
 * device_rename - renames a device
 * @dev: the pointer to the struct device to be renamed
 * @new_name: the new name of the device
 *
 * It is the responsibility of the caller to provide mutual
 * exclusion between two different calls of device_rename
 * on the same device to ensure that new_name is valid and
 * won't conflict with other devices.
 *
 * Note: given that some subsystems (networking and infiniband) use this
 * function, with no immediate plans for this to change, we cannot assume or
 * require that this function not be called at all.
 *
 * However, if you're writing new code, do not call this function. The following
 * text from Kay Sievers offers some insight:
 *
 * Renaming devices is racy at many levels, symlinks and other stuff are not
 * replaced atomically, and you get a "move" uevent, but it's not easy to
 * connect the event to the old and new device. Device nodes are not renamed at
 * all, there isn't even support for that in the kernel now.
 *
 * In the meantime, during renaming, your target name might be taken by another
 * driver, creating conflicts. Or the old name is taken directly after you
 * renamed it -- then you get events for the same DEVPATH, before you even see
 * the "move" event. It's just a mess, and nothing new should ever rely on
 * kernel device renaming. Besides that, it's not even implemented now for
 * other things than (driver-core wise very simple) network devices.
 *
 * Make up a "real" name in the driver before you register anything, or add
 * some other attributes for userspace to find the device, or use udev to add
 * symlinks -- but never rename kernel devices later, it's a complete mess. We
 * don't even want to get into that and try to implement the missing pieces in
 * the core. We really have other pieces to fix in the driver core mess. :)
 */
int device_rename(struct device *dev, const char *new_name)
{
        struct subsys_private *sp = NULL;
        struct kobject *kobj = &dev->kobj;
        char *old_device_name = NULL;
        int error;
        bool is_link_renamed = false;

        dev = get_device(dev);
        if (!dev)
                return -EINVAL;

        dev_dbg(dev, "renaming to %s\n", new_name);

        old_device_name = kstrdup(dev_name(dev), GFP_KERNEL);
        if (!old_device_name) {
                error = -ENOMEM;
                goto out;
        }

        if (dev->class) {
                sp = class_to_subsys(dev->class);

                if (!sp) {
                        error = -EINVAL;
                        goto out;
                }

                error = sysfs_rename_link_ns(&sp->subsys.kobj, kobj, old_device_name,
                                             new_name, kobject_namespace(kobj));
                if (error)
                        goto out;

                is_link_renamed = true;
        }

        error = kobject_rename(kobj, new_name);
out:
        if (error && is_link_renamed)
                sysfs_rename_link_ns(&sp->subsys.kobj, kobj, new_name,
                                     old_device_name, kobject_namespace(kobj));
        subsys_put(sp);

        put_device(dev);

        kfree(old_device_name);

        return error;
}
EXPORT_SYMBOL_GPL(device_rename);

static int device_move_class_links(struct device *dev,
                                   struct device *old_parent,
                                   struct device *new_parent)
{
        int error = 0;

        if (old_parent)
                sysfs_remove_link(&dev->kobj, "device");
        if (new_parent)
                error = sysfs_create_link(&dev->kobj, &new_parent->kobj,
                                          "device");
        return error;
}

/**
 * device_move - moves a device to a new parent
 * @dev: the pointer to the struct device to be moved
 * @new_parent: the new parent of the device (can be NULL)
 * @dpm_order: how to reorder the dpm_list
 */
int device_move(struct device *dev, struct device *new_parent,
                enum dpm_order dpm_order)
{
        int error;
        struct device *old_parent;
        struct kobject *new_parent_kobj;

        dev = get_device(dev);
        if (!dev)
                return -EINVAL;

        device_pm_lock();
        new_parent = get_device(new_parent);
        new_parent_kobj = get_device_parent(dev, new_parent);
        if (IS_ERR(new_parent_kobj)) {
                error = PTR_ERR(new_parent_kobj);
                put_device(new_parent);
                goto out;
        }

        pr_debug("device: '%s': %s: moving to '%s'\n", dev_name(dev),
                 __func__, new_parent ? dev_name(new_parent) : "<NULL>");
        error = kobject_move(&dev->kobj, new_parent_kobj);
        if (error) {
                cleanup_glue_dir(dev, new_parent_kobj);
                put_device(new_parent);
                goto out;
        }
        old_parent = dev->parent;
        dev->parent = new_parent;
        if (old_parent)
                klist_remove(&dev->p->knode_parent);
        if (new_parent) {
                klist_add_tail(&dev->p->knode_parent,
                               &new_parent->p->klist_children);
                set_dev_node(dev, dev_to_node(new_parent));
        }

        if (dev->class) {
                error = device_move_class_links(dev, old_parent, new_parent);
                if (error) {
                        /* We ignore errors on cleanup since we're hosed anyway... */
                        device_move_class_links(dev, new_parent, old_parent);
                        if (!kobject_move(&dev->kobj, &old_parent->kobj)) {
                                if (new_parent)
                                        klist_remove(&dev->p->knode_parent);
                                dev->parent = old_parent;
                                if (old_parent) {
                                        klist_add_tail(&dev->p->knode_parent,
                                                       &old_parent->p->klist_children);
                                        set_dev_node(dev, dev_to_node(old_parent));
                                }
                        }
                        cleanup_glue_dir(dev, new_parent_kobj);
                        put_device(new_parent);
                        goto out;
                }
        }
        switch (dpm_order) {
        case DPM_ORDER_NONE:
                break;
        case DPM_ORDER_DEV_AFTER_PARENT:
                device_pm_move_after(dev, new_parent);
                devices_kset_move_after(dev, new_parent);
                break;
        case DPM_ORDER_PARENT_BEFORE_DEV:
                device_pm_move_before(new_parent, dev);
                devices_kset_move_before(new_parent, dev);
                break;
        case DPM_ORDER_DEV_LAST:
                device_pm_move_last(dev);
                devices_kset_move_last(dev);
                break;
        }

        put_device(old_parent);
out:
        device_pm_unlock();
        put_device(dev);
        return error;
}
EXPORT_SYMBOL_GPL(device_move);

static int device_attrs_change_owner(struct device *dev, kuid_t kuid,
                                     kgid_t kgid)
{
        struct kobject *kobj = &dev->kobj;
        const struct class *class = dev->class;
        const struct device_type *type = dev->type;
        int error;

        if (class) {
                /*
                 * Change the device groups of the device class for @dev to
                 * @kuid/@kgid.
                 */
                error = sysfs_groups_change_owner(kobj, class->dev_groups, kuid,
                                                  kgid);
                if (error)
                        return error;
        }

        if (type) {
                /*
                 * Change the device groups of the device type for @dev to
                 * @kuid/@kgid.
                 */
                error = sysfs_groups_change_owner(kobj, type->groups, kuid,
                                                  kgid);
                if (error)
                        return error;
        }

        /* Change the device groups of @dev to @kuid/@kgid. */
        error = sysfs_groups_change_owner(kobj, dev->groups, kuid, kgid);
        if (error)
                return error;

        if (device_supports_offline(dev) && !dev->offline_disabled) {
                /* Change online device attributes of @dev to @kuid/@kgid. */
                error = sysfs_file_change_owner(kobj, dev_attr_online.attr.name,
                                                kuid, kgid);
                if (error)
                        return error;
        }

        return 0;
}

/**
 * device_change_owner - change the owner of an existing device.
 * @dev: device.
 * @kuid: new owner's kuid
 * @kgid: new owner's kgid
 *
 * This changes the owner of @dev and its corresponding sysfs entries to
 * @kuid/@kgid. This function closely mirrors how @dev was added via driver
 * core.
 *
 * Returns 0 on success or error code on failure.
 */
int device_change_owner(struct device *dev, kuid_t kuid, kgid_t kgid)
{
        int error;
        struct kobject *kobj = &dev->kobj;
        struct subsys_private *sp;

        dev = get_device(dev);
        if (!dev)
                return -EINVAL;

        /*
         * Change the kobject and the default attributes and groups of the
         * ktype associated with it to @kuid/@kgid.
         */
        error = sysfs_change_owner(kobj, kuid, kgid);
        if (error)
                goto out;

        /*
         * Change the uevent file for @dev to the new owner. The uevent file
         * was created in a separate step when @dev got added and we mirror
         * that step here.
         */
        error = sysfs_file_change_owner(kobj, dev_attr_uevent.attr.name, kuid,
                                        kgid);
        if (error)
                goto out;

        /*
         * Change the device groups, the device groups associated with the
         * device class, and the groups associated with the device type of @dev
         * to @kuid/@kgid.
         */
        error = device_attrs_change_owner(dev, kuid, kgid);
        if (error)
                goto out;

        error = dpm_sysfs_change_owner(dev, kuid, kgid);
        if (error)
                goto out;

        /*
         * Change the owner of the symlink located in the class directory of
         * the device class associated with @dev which points to the actual
         * directory entry for @dev to @kuid/@kgid. This ensures that the
         * symlink shows the same permissions as its target.
         */
        sp = class_to_subsys(dev->class);
        if (!sp) {
                error = -EINVAL;
                goto out;
        }
        error = sysfs_link_change_owner(&sp->subsys.kobj, &dev->kobj, dev_name(dev), kuid, kgid);
        subsys_put(sp);

out:
        put_device(dev);
        return error;
}
EXPORT_SYMBOL_GPL(device_change_owner);

/**
 * device_shutdown - call ->shutdown() on each device to shutdown.
 */
void device_shutdown(void)
{
        struct device *dev, *parent;

        wait_for_device_probe();
        device_block_probing();

        cpufreq_suspend();

        spin_lock(&devices_kset->list_lock);
        /*
         * Walk the devices list backward, shutting down each in turn.
         * Beware that device unplug events may also start pulling
         * devices offline, even as the system is shutting down.
         */
        while (!list_empty(&devices_kset->list)) {
                dev = list_entry(devices_kset->list.prev, struct device,
                                kobj.entry);

                /*
                 * hold reference count of device's parent to
                 * prevent it from being freed because parent's
                 * lock is to be held
                 */
                parent = get_device(dev->parent);
                get_device(dev);
                /*
                 * Make sure the device is off the kset list, in the
                 * event that dev->*->shutdown() doesn't remove it.
                 */
                list_del_init(&dev->kobj.entry);
                spin_unlock(&devices_kset->list_lock);

                /* hold lock to avoid race with probe/release */
                if (parent)
                        device_lock(parent);
                device_lock(dev);

                /* Don't allow any more runtime suspends */
                pm_runtime_get_noresume(dev);
                pm_runtime_barrier(dev);

                if (dev->class && dev->class->shutdown_pre) {
                        if (initcall_debug)
                                dev_info(dev, "shutdown_pre\n");
                        dev->class->shutdown_pre(dev);
                }
                if (dev->bus && dev->bus->shutdown) {
                        if (initcall_debug)
                                dev_info(dev, "shutdown\n");
                        dev->bus->shutdown(dev);
                } else if (dev->driver && dev->driver->shutdown) {
                        if (initcall_debug)
                                dev_info(dev, "shutdown\n");
                        dev->driver->shutdown(dev);
                }

                device_unlock(dev);
                if (parent)
                        device_unlock(parent);

                put_device(dev);
                put_device(parent);

                spin_lock(&devices_kset->list_lock);
        }
        spin_unlock(&devices_kset->list_lock);
}

/*
 * Device logging functions
 */

#ifdef CONFIG_PRINTK
static void
set_dev_info(const struct device *dev, struct dev_printk_info *dev_info)
{
        const char *subsys;

        memset(dev_info, 0, sizeof(*dev_info));

        if (dev->class)
                subsys = dev->class->name;
        else if (dev->bus)
                subsys = dev->bus->name;
        else
                return;

        strscpy(dev_info->subsystem, subsys);

        /*
         * Add device identifier DEVICE=:
         *   b12:8         block dev_t
         *   c127:3        char dev_t
         *   n8            netdev ifindex
         *   +sound:card0  subsystem:devname
         */
        if (MAJOR(dev->devt)) {
                char c;

                if (strcmp(subsys, "block") == 0)
                        c = 'b';
                else
                        c = 'c';

                snprintf(dev_info->device, sizeof(dev_info->device),
                         "%c%u:%u", c, MAJOR(dev->devt), MINOR(dev->devt));
        } else if (strcmp(subsys, "net") == 0) {
                struct net_device *net = to_net_dev(dev);

                snprintf(dev_info->device, sizeof(dev_info->device),
                         "n%u", net->ifindex);
        } else {
                snprintf(dev_info->device, sizeof(dev_info->device),
                         "+%s:%s", subsys, dev_name(dev));
        }
}

int dev_vprintk_emit(int level, const struct device *dev,
                     const char *fmt, va_list args)
{
        struct dev_printk_info dev_info;

        set_dev_info(dev, &dev_info);

        return vprintk_emit(0, level, &dev_info, fmt, args);
}
EXPORT_SYMBOL(dev_vprintk_emit);

int dev_printk_emit(int level, const struct device *dev, const char *fmt, ...)
{
        va_list args;
        int r;

        va_start(args, fmt);

        r = dev_vprintk_emit(level, dev, fmt, args);

        va_end(args);

        return r;
}
EXPORT_SYMBOL(dev_printk_emit);

static void __dev_printk(const char *level, const struct device *dev,
                        struct va_format *vaf)
{
        if (dev)
                dev_printk_emit(level[1] - '0', dev, "%s %s: %pV",
                                dev_driver_string(dev), dev_name(dev), vaf);
        else
                printk("%s(NULL device *): %pV", level, vaf);
}

void _dev_printk(const char *level, const struct device *dev,
                 const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);

        vaf.fmt = fmt;
        vaf.va = &args;

        __dev_printk(level, dev, &vaf);

        va_end(args);
}
EXPORT_SYMBOL(_dev_printk);

#define define_dev_printk_level(func, kern_level)                \
void func(const struct device *dev, const char *fmt, ...)        \
{                                                                \
        struct va_format vaf;                                        \
        va_list args;                                                \
                                                                \
        va_start(args, fmt);                                        \
                                                                \
        vaf.fmt = fmt;                                                \
        vaf.va = &args;                                                \
                                                                \
        __dev_printk(kern_level, dev, &vaf);                        \
                                                                \
        va_end(args);                                                \
}                                                                \
EXPORT_SYMBOL(func);

define_dev_printk_level(_dev_emerg, KERN_EMERG);
define_dev_printk_level(_dev_alert, KERN_ALERT);
define_dev_printk_level(_dev_crit, KERN_CRIT);
define_dev_printk_level(_dev_err, KERN_ERR);
define_dev_printk_level(_dev_warn, KERN_WARNING);
define_dev_printk_level(_dev_notice, KERN_NOTICE);
define_dev_printk_level(_dev_info, KERN_INFO);

#endif

static void __dev_probe_failed(const struct device *dev, int err, bool fatal,
                               const char *fmt, va_list vargsp)
{
        struct va_format vaf;
        va_list vargs;

        /*
         * On x86_64 and possibly on other architectures, va_list is actually a
         * size-1 array containing a structure.  As a result, function parameter
         * vargsp decays from T[1] to T*, and &vargsp has type T** rather than
         * T(*)[1], which is expected by its assignment to vaf.va below.
         *
         * One standard way to solve this mess is by creating a copy in a local
         * variable of type va_list and then using a pointer to that local copy
         * instead, which is the approach employed here.
         */
        va_copy(vargs, vargsp);

        vaf.fmt = fmt;
        vaf.va = &vargs;

        switch (err) {
        case -EPROBE_DEFER:
                device_set_deferred_probe_reason(dev, &vaf);
                dev_dbg(dev, "error %pe: %pV", ERR_PTR(err), &vaf);
                break;

        case -ENOMEM:
                /* Don't print anything on -ENOMEM, there's already enough output */
                break;

        default:
                /* Log fatal final failures as errors, otherwise produce warnings */
                if (fatal)
                        dev_err(dev, "error %pe: %pV", ERR_PTR(err), &vaf);
                else
                        dev_warn(dev, "error %pe: %pV", ERR_PTR(err), &vaf);
                break;
        }

        va_end(vargs);
}

/**
 * dev_err_probe - probe error check and log helper
 * @dev: the pointer to the struct device
 * @err: error value to test
 * @fmt: printf-style format string
 * @...: arguments as specified in the format string
 *
 * This helper implements common pattern present in probe functions for error
 * checking: print debug or error message depending if the error value is
 * -EPROBE_DEFER and propagate error upwards.
 * In case of -EPROBE_DEFER it sets also defer probe reason, which can be
 * checked later by reading devices_deferred debugfs attribute.
 * It replaces the following code sequence::
 *
 *         if (err != -EPROBE_DEFER)
 *                 dev_err(dev, ...);
 *         else
 *                 dev_dbg(dev, ...);
 *         return err;
 *
 * with::
 *
 *         return dev_err_probe(dev, err, ...);
 *
 * Using this helper in your probe function is totally fine even if @err
 * is known to never be -EPROBE_DEFER.
 * The benefit compared to a normal dev_err() is the standardized format
 * of the error code, which is emitted symbolically (i.e. you get "EAGAIN"
 * instead of "-35"), and having the error code returned allows more
 * compact error paths.
 *
 * Returns @err.
 */
int dev_err_probe(const struct device *dev, int err, const char *fmt, ...)
{
        va_list vargs;

        va_start(vargs, fmt);

        /* Use dev_err() for logging when err doesn't equal -EPROBE_DEFER */
        __dev_probe_failed(dev, err, true, fmt, vargs);

        va_end(vargs);

        return err;
}
EXPORT_SYMBOL_GPL(dev_err_probe);

/**
 * dev_warn_probe - probe error check and log helper
 * @dev: the pointer to the struct device
 * @err: error value to test
 * @fmt: printf-style format string
 * @...: arguments as specified in the format string
 *
 * This helper implements common pattern present in probe functions for error
 * checking: print debug or warning message depending if the error value is
 * -EPROBE_DEFER and propagate error upwards.
 * In case of -EPROBE_DEFER it sets also defer probe reason, which can be
 * checked later by reading devices_deferred debugfs attribute.
 * It replaces the following code sequence::
 *
 *         if (err != -EPROBE_DEFER)
 *                 dev_warn(dev, ...);
 *         else
 *                 dev_dbg(dev, ...);
 *         return err;
 *
 * with::
 *
 *         return dev_warn_probe(dev, err, ...);
 *
 * Using this helper in your probe function is totally fine even if @err
 * is known to never be -EPROBE_DEFER.
 * The benefit compared to a normal dev_warn() is the standardized format
 * of the error code, which is emitted symbolically (i.e. you get "EAGAIN"
 * instead of "-35"), and having the error code returned allows more
 * compact error paths.
 *
 * Returns @err.
 */
int dev_warn_probe(const struct device *dev, int err, const char *fmt, ...)
{
        va_list vargs;

        va_start(vargs, fmt);

        /* Use dev_warn() for logging when err doesn't equal -EPROBE_DEFER */
        __dev_probe_failed(dev, err, false, fmt, vargs);

        va_end(vargs);

        return err;
}
EXPORT_SYMBOL_GPL(dev_warn_probe);

static inline bool fwnode_is_primary(struct fwnode_handle *fwnode)
{
        return fwnode && !IS_ERR(fwnode->secondary);
}

/**
 * set_primary_fwnode - Change the primary firmware node of a given device.
 * @dev: Device to handle.
 * @fwnode: New primary firmware node of the device.
 *
 * Set the device's firmware node pointer to @fwnode, but if a secondary
 * firmware node of the device is present, preserve it.
 *
 * Valid fwnode cases are:
 *  - primary --> secondary --> -ENODEV
 *  - primary --> NULL
 *  - secondary --> -ENODEV
 *  - NULL
 */
void set_primary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
{
        struct device *parent = dev->parent;
        struct fwnode_handle *fn = dev->fwnode;

        if (fwnode) {
                if (fwnode_is_primary(fn))
                        fn = fn->secondary;

                if (fn) {
                        WARN_ON(fwnode->secondary);
                        fwnode->secondary = fn;
                }
                dev->fwnode = fwnode;
        } else {
                if (fwnode_is_primary(fn)) {
                        dev->fwnode = fn->secondary;

                        /* Skip nullifying fn->secondary if the primary is shared */
                        if (parent && fn == parent->fwnode)
                                return;

                        /* Set fn->secondary = NULL, so fn remains the primary fwnode */
                        fn->secondary = NULL;
                } else {
                        dev->fwnode = NULL;
                }
        }
}
EXPORT_SYMBOL_GPL(set_primary_fwnode);

/**
 * set_secondary_fwnode - Change the secondary firmware node of a given device.
 * @dev: Device to handle.
 * @fwnode: New secondary firmware node of the device.
 *
 * If a primary firmware node of the device is present, set its secondary
 * pointer to @fwnode.  Otherwise, set the device's firmware node pointer to
 * @fwnode.
 */
void set_secondary_fwnode(struct device *dev, struct fwnode_handle *fwnode)
{
        if (fwnode)
                fwnode->secondary = ERR_PTR(-ENODEV);

        if (fwnode_is_primary(dev->fwnode))
                dev->fwnode->secondary = fwnode;
        else
                dev->fwnode = fwnode;
}
EXPORT_SYMBOL_GPL(set_secondary_fwnode);

/**
 * device_remove_of_node - Remove an of_node from a device
 * @dev: device whose device tree node is being removed
 */
void device_remove_of_node(struct device *dev)
{
        dev = get_device(dev);
        if (!dev)
                return;

        if (!dev->of_node)
                goto end;

        if (dev->fwnode == of_fwnode_handle(dev->of_node))
                dev->fwnode = NULL;

        of_node_put(dev->of_node);
        dev->of_node = NULL;

end:
        put_device(dev);
}
EXPORT_SYMBOL_GPL(device_remove_of_node);

/**
 * device_add_of_node - Add an of_node to an existing device
 * @dev: device whose device tree node is being added
 * @of_node: of_node to add
 *
 * Return: 0 on success or error code on failure.
 */
int device_add_of_node(struct device *dev, struct device_node *of_node)
{
        int ret;

        if (!of_node)
                return -EINVAL;

        dev = get_device(dev);
        if (!dev)
                return -EINVAL;

        if (dev->of_node) {
                dev_err(dev, "Cannot replace node %pOF with %pOF\n",
                        dev->of_node, of_node);
                ret = -EBUSY;
                goto end;
        }

        dev->of_node = of_node_get(of_node);

        if (!dev->fwnode)
                dev->fwnode = of_fwnode_handle(of_node);

        ret = 0;
end:
        put_device(dev);
        return ret;
}
EXPORT_SYMBOL_GPL(device_add_of_node);

/**
 * device_set_of_node_from_dev - reuse device-tree node of another device
 * @dev: device whose device-tree node is being set
 * @dev2: device whose device-tree node is being reused
 *
 * Takes another reference to the new device-tree node after first dropping
 * any reference held to the old node.
 */
void device_set_of_node_from_dev(struct device *dev, const struct device *dev2)
{
        of_node_put(dev->of_node);
        dev->of_node = of_node_get(dev2->of_node);
        dev->of_node_reused = true;
}
EXPORT_SYMBOL_GPL(device_set_of_node_from_dev);

void device_set_node(struct device *dev, struct fwnode_handle *fwnode)
{
        dev->fwnode = fwnode;
        dev->of_node = to_of_node(fwnode);
}
EXPORT_SYMBOL_GPL(device_set_node);

/**
 * get_dev_from_fwnode - Obtain a reference count of the struct device the
 * struct fwnode_handle is associated with.
 * @fwnode: The pointer to the struct fwnode_handle to obtain the struct device
 * reference count of.
 *
 * This function obtains a reference count of the device the device pointer
 * embedded in the struct fwnode_handle points to.
 *
 * Note that the struct device pointer embedded in struct fwnode_handle does
 * *not* have a reference count of the struct device itself.
 *
 * Hence, it is a UAF (and thus a bug) to call this function if the caller can't
 * guarantee that the last reference count of the corresponding struct device is
 * not dropped concurrently.
 *
 * This is possible since struct fwnode_handle has its own reference count and
 * hence can out-live the struct device it is associated with.
 */
struct device *get_dev_from_fwnode(struct fwnode_handle *fwnode)
{
        return get_device((fwnode)->dev);
}
EXPORT_SYMBOL_GPL(get_dev_from_fwnode);

int device_match_name(struct device *dev, const void *name)
{
        return sysfs_streq(dev_name(dev), name);
}
EXPORT_SYMBOL_GPL(device_match_name);

int device_match_type(struct device *dev, const void *type)
{
        return dev->type == type;
}
EXPORT_SYMBOL_GPL(device_match_type);

int device_match_of_node(struct device *dev, const void *np)
{
        return np && dev->of_node == np;
}
EXPORT_SYMBOL_GPL(device_match_of_node);

int device_match_fwnode(struct device *dev, const void *fwnode)
{
        return fwnode && dev_fwnode(dev) == fwnode;
}
EXPORT_SYMBOL_GPL(device_match_fwnode);

int device_match_devt(struct device *dev, const void *pdevt)
{
        return dev->devt == *(dev_t *)pdevt;
}
EXPORT_SYMBOL_GPL(device_match_devt);

int device_match_acpi_dev(struct device *dev, const void *adev)
{
        return adev && ACPI_COMPANION(dev) == adev;
}
EXPORT_SYMBOL(device_match_acpi_dev);

int device_match_acpi_handle(struct device *dev, const void *handle)
{
        return handle && ACPI_HANDLE(dev) == handle;
}
EXPORT_SYMBOL(device_match_acpi_handle);

int device_match_any(struct device *dev, const void *unused)
{
        return 1;
}
EXPORT_SYMBOL_GPL(device_match_any);




































































































   10 




    1 
    1 
    1 
    2 




    1 













    4 
















    3 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/unaligned.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>

struct nft_byteorder {
        u8                        sreg;
        u8                        dreg;
        enum nft_byteorder_ops        op:8;
        u8                        len;
        u8                        size;
};

void nft_byteorder_eval(const struct nft_expr *expr,
                        struct nft_regs *regs,
                        const struct nft_pktinfo *pkt)
{
        const struct nft_byteorder *priv = nft_expr_priv(expr);
        u32 *src = &regs->data[priv->sreg];
        u32 *dst = &regs->data[priv->dreg];
        u16 *s16, *d16;
        unsigned int i;

        s16 = (void *)src;
        d16 = (void *)dst;

        switch (priv->size) {
        case 8: {
                u64 *dst64 = (void *)dst;
                u64 src64;

                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 8; i++) {
                                src64 = nft_reg_load64(&src[i]);
                                nft_reg_store64(&dst64[i],
                                                be64_to_cpu((__force __be64)src64));
                        }
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 8; i++) {
                                src64 = (__force __u64)
                                        cpu_to_be64(nft_reg_load64(&src[i]));
                                nft_reg_store64(&dst64[i], src64);
                        }
                        break;
                }
                break;
        }
        case 4:
                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 4; i++)
                                dst[i] = ntohl((__force __be32)src[i]);
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 4; i++)
                                dst[i] = (__force __u32)htonl(src[i]);
                        break;
                }
                break;
        case 2:
                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 2; i++)
                                d16[i] = ntohs((__force __be16)s16[i]);
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 2; i++)
                                d16[i] = (__force __u16)htons(s16[i]);
                        break;
                }
                break;
        }
}

static const struct nla_policy nft_byteorder_policy[NFTA_BYTEORDER_MAX + 1] = {
        [NFTA_BYTEORDER_SREG]        = { .type = NLA_U32 },
        [NFTA_BYTEORDER_DREG]        = { .type = NLA_U32 },
        [NFTA_BYTEORDER_OP]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_BYTEORDER_LEN]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_BYTEORDER_SIZE]        = NLA_POLICY_MAX(NLA_BE32, 255),
};

static int nft_byteorder_init(const struct nft_ctx *ctx,
                              const struct nft_expr *expr,
                              const struct nlattr * const tb[])
{
        struct nft_byteorder *priv = nft_expr_priv(expr);
        u32 size, len;
        int err;

        if (tb[NFTA_BYTEORDER_SREG] == NULL ||
            tb[NFTA_BYTEORDER_DREG] == NULL ||
            tb[NFTA_BYTEORDER_LEN] == NULL ||
            tb[NFTA_BYTEORDER_SIZE] == NULL ||
            tb[NFTA_BYTEORDER_OP] == NULL)
                return -EINVAL;

        priv->op = ntohl(nla_get_be32(tb[NFTA_BYTEORDER_OP]));
        switch (priv->op) {
        case NFT_BYTEORDER_NTOH:
        case NFT_BYTEORDER_HTON:
                break;
        default:
                return -EINVAL;
        }

        err = nft_parse_u32_check(tb[NFTA_BYTEORDER_SIZE], U8_MAX, &size);
        if (err < 0)
                return err;

        priv->size = size;

        switch (priv->size) {
        case 2:
        case 4:
        case 8:
                break;
        default:
                return -EINVAL;
        }

        err = nft_parse_u32_check(tb[NFTA_BYTEORDER_LEN], U8_MAX, &len);
        if (err < 0)
                return err;

        priv->len = len;

        err = nft_parse_register_load(ctx, tb[NFTA_BYTEORDER_SREG], &priv->sreg,
                                      priv->len);
        if (err < 0)
                return err;

        return nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG],
                                        &priv->dreg, NULL, NFT_DATA_VALUE,
                                        priv->len);
}

static int nft_byteorder_dump(struct sk_buff *skb,
                              const struct nft_expr *expr, bool reset)
{
        const struct nft_byteorder *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_BYTEORDER_SREG, priv->sreg))
                goto nla_put_failure;
        if (nft_dump_register(skb, NFTA_BYTEORDER_DREG, priv->dreg))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_OP, htonl(priv->op)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_LEN, htonl(priv->len)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_SIZE, htonl(priv->size)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static bool nft_byteorder_reduce(struct nft_regs_track *track,
                                 const struct nft_expr *expr)
{
        struct nft_byteorder *priv = nft_expr_priv(expr);

        nft_reg_track_cancel(track, priv->dreg, priv->len);

        return false;
}

static const struct nft_expr_ops nft_byteorder_ops = {
        .type                = &nft_byteorder_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_byteorder)),
        .eval                = nft_byteorder_eval,
        .init                = nft_byteorder_init,
        .dump                = nft_byteorder_dump,
        .reduce                = nft_byteorder_reduce,
};

struct nft_expr_type nft_byteorder_type __read_mostly = {
        .name                = "byteorder",
        .ops                = &nft_byteorder_ops,
        .policy                = nft_byteorder_policy,
        .maxattr        = NFTA_BYTEORDER_MAX,
        .owner                = THIS_MODULE,
};



























































































  159 





  107 
   63 

  160 

    5 





  156 







    5 






  102 
  153 


   99 
   63 





















  232 







   24 

    1 









  196 
   12 








    6 





   21 

    1 
    1 






  162 
   39 




    1 




  199 



    1 













   21 


   21 

    6 



   15 























  199 





    2 


    6 






  198 



    6 

















   14 









    2 

















    8 
















  156 
  157 













































































































  176 






   39 


  143 

   20 






   10 
























   28 









   28 


    1 












    8 




    4 
   17 

    4 
   17 

   12 















    9 


    6 
    3 








    1 



    1 

   13 









   26 











   26 

   26 




    6 

   23 


   17 
    6 

   16 
   14 
   24 

   23 






   25 






   26 
   26 
   25 
   24 



   26 
   25 


   14 





   26 






   29 











   29 














   22 




    1 











    6 








    3 












    3 


















    1 

    1 











    1 















    2 






























    1 
    1 
    1 
    1 





    1 

    1 
    1 









    1 







    2 
















    1 




    1 

    1 







    2 











   15 









  119 






  120 


    1 







   34 


    2 


  117 






   93 








    1 



   26 



   28 

   40 



   91 




   93 


  118 





























  155 



























  118 

   40 



  157 





















  157 


















































  156 


  156 



  155 






   11 
  107 







  107 




 1106 







 1093 

  107 


  261 








































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
// SPDX-License-Identifier: GPL-2.0-only

#include <net/netdev_lock.h>
#include <net/netdev_queues.h>
#include <net/sock.h>
#include <linux/ethtool_netlink.h>
#include <linux/phy_link_topology.h>
#include <linux/pm_runtime.h>
#include "netlink.h"
#include "module_fw.h"

static struct genl_family ethtool_genl_family;

static bool ethnl_ok __read_mostly;
static u32 ethnl_bcast_seq;

#define ETHTOOL_FLAGS_BASIC (ETHTOOL_FLAG_COMPACT_BITSETS |        \
                             ETHTOOL_FLAG_OMIT_REPLY)
#define ETHTOOL_FLAGS_STATS (ETHTOOL_FLAGS_BASIC | ETHTOOL_FLAG_STATS)

const struct nla_policy ethnl_header_policy[] = {
        [ETHTOOL_A_HEADER_DEV_INDEX]        = { .type = NLA_U32 },
        [ETHTOOL_A_HEADER_DEV_NAME]        = { .type = NLA_NUL_STRING,
                                            .len = ALTIFNAMSIZ - 1 },
        [ETHTOOL_A_HEADER_FLAGS]        = NLA_POLICY_MASK(NLA_U32,
                                                          ETHTOOL_FLAGS_BASIC),
};

const struct nla_policy ethnl_header_policy_stats[] = {
        [ETHTOOL_A_HEADER_DEV_INDEX]        = { .type = NLA_U32 },
        [ETHTOOL_A_HEADER_DEV_NAME]        = { .type = NLA_NUL_STRING,
                                            .len = ALTIFNAMSIZ - 1 },
        [ETHTOOL_A_HEADER_FLAGS]        = NLA_POLICY_MASK(NLA_U32,
                                                          ETHTOOL_FLAGS_STATS),
};

const struct nla_policy ethnl_header_policy_phy[] = {
        [ETHTOOL_A_HEADER_DEV_INDEX]        = { .type = NLA_U32 },
        [ETHTOOL_A_HEADER_DEV_NAME]        = { .type = NLA_NUL_STRING,
                                            .len = ALTIFNAMSIZ - 1 },
        [ETHTOOL_A_HEADER_FLAGS]        = NLA_POLICY_MASK(NLA_U32,
                                                          ETHTOOL_FLAGS_BASIC),
        [ETHTOOL_A_HEADER_PHY_INDEX]                = NLA_POLICY_MIN(NLA_U32, 1),
};

const struct nla_policy ethnl_header_policy_phy_stats[] = {
        [ETHTOOL_A_HEADER_DEV_INDEX]        = { .type = NLA_U32 },
        [ETHTOOL_A_HEADER_DEV_NAME]        = { .type = NLA_NUL_STRING,
                                            .len = ALTIFNAMSIZ - 1 },
        [ETHTOOL_A_HEADER_FLAGS]        = NLA_POLICY_MASK(NLA_U32,
                                                          ETHTOOL_FLAGS_STATS),
        [ETHTOOL_A_HEADER_PHY_INDEX]                = NLA_POLICY_MIN(NLA_U32, 1),
};

int ethnl_sock_priv_set(struct sk_buff *skb, struct net_device *dev, u32 portid,
                        enum ethnl_sock_type type)
{
        struct ethnl_sock_priv *sk_priv;

        sk_priv = genl_sk_priv_get(&ethtool_genl_family, NETLINK_CB(skb).sk);
        if (IS_ERR(sk_priv))
                return PTR_ERR(sk_priv);

        sk_priv->dev = dev;
        sk_priv->portid = portid;
        sk_priv->type = type;

        return 0;
}

static void ethnl_sock_priv_destroy(void *priv)
{
        struct ethnl_sock_priv *sk_priv = priv;

        switch (sk_priv->type) {
        case ETHTOOL_SOCK_TYPE_MODULE_FW_FLASH:
                ethnl_module_fw_flash_sock_destroy(sk_priv);
                break;
        default:
                break;
        }
}

u32 ethnl_bcast_seq_next(void)
{
        ASSERT_RTNL();
        return ++ethnl_bcast_seq;
}

int ethnl_ops_begin(struct net_device *dev)
{
        int ret;

        if (!dev)
                return -ENODEV;

        if (dev->dev.parent)
                pm_runtime_get_sync(dev->dev.parent);

        netdev_ops_assert_locked(dev);

        if (!netif_device_present(dev) ||
            dev->reg_state >= NETREG_UNREGISTERING) {
                ret = -ENODEV;
                goto err;
        }

        if (dev->ethtool_ops->begin) {
                ret = dev->ethtool_ops->begin(dev);
                if (ret)
                        goto err;
        }

        return 0;
err:
        if (dev->dev.parent)
                pm_runtime_put(dev->dev.parent);

        return ret;
}

void ethnl_ops_complete(struct net_device *dev)
{
        if (dev->ethtool_ops->complete)
                dev->ethtool_ops->complete(dev);

        if (dev->dev.parent)
                pm_runtime_put(dev->dev.parent);
}

/**
 * ethnl_parse_header_dev_get() - parse request header
 * @req_info:    structure to put results into
 * @header:      nest attribute with request header
 * @net:         request netns
 * @extack:      netlink extack for error reporting
 * @require_dev: fail if no device identified in header
 *
 * Parse request header in nested attribute @nest and puts results into
 * the structure pointed to by @req_info. Extack from @info is used for error
 * reporting. If req_info->dev is not null on return, reference to it has
 * been taken. If error is returned, *req_info is null initialized and no
 * reference is held.
 *
 * Return: 0 on success or negative error code
 */
int ethnl_parse_header_dev_get(struct ethnl_req_info *req_info,
                               const struct nlattr *header, struct net *net,
                               struct netlink_ext_ack *extack, bool require_dev)
{
        struct nlattr *tb[ARRAY_SIZE(ethnl_header_policy_phy)];
        const struct nlattr *devname_attr;
        struct net_device *dev = NULL;
        u32 flags = 0;
        int ret;

        if (!header) {
                if (!require_dev)
                        return 0;
                NL_SET_ERR_MSG(extack, "request header missing");
                return -EINVAL;
        }
        /* No validation here, command policy should have a nested policy set
         * for the header, therefore validation should have already been done.
         */
        ret = nla_parse_nested(tb, ARRAY_SIZE(ethnl_header_policy_phy) - 1, header,
                               NULL, extack);
        if (ret < 0)
                return ret;
        if (tb[ETHTOOL_A_HEADER_FLAGS])
                flags = nla_get_u32(tb[ETHTOOL_A_HEADER_FLAGS]);

        devname_attr = tb[ETHTOOL_A_HEADER_DEV_NAME];
        if (tb[ETHTOOL_A_HEADER_DEV_INDEX]) {
                u32 ifindex = nla_get_u32(tb[ETHTOOL_A_HEADER_DEV_INDEX]);

                dev = netdev_get_by_index(net, ifindex, &req_info->dev_tracker,
                                          GFP_KERNEL);
                if (!dev) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            tb[ETHTOOL_A_HEADER_DEV_INDEX],
                                            "no device matches ifindex");
                        return -ENODEV;
                }
                /* if both ifindex and ifname are passed, they must match */
                if (devname_attr &&
                    strncmp(dev->name, nla_data(devname_attr), IFNAMSIZ)) {
                        netdev_put(dev, &req_info->dev_tracker);
                        NL_SET_ERR_MSG_ATTR(extack, header,
                                            "ifindex and name do not match");
                        return -ENODEV;
                }
        } else if (devname_attr) {
                dev = netdev_get_by_name(net, nla_data(devname_attr),
                                         &req_info->dev_tracker, GFP_KERNEL);
                if (!dev) {
                        NL_SET_ERR_MSG_ATTR(extack, devname_attr,
                                            "no device matches name");
                        return -ENODEV;
                }
        } else if (require_dev) {
                NL_SET_ERR_MSG_ATTR(extack, header,
                                    "neither ifindex nor name specified");
                return -EINVAL;
        }

        if (tb[ETHTOOL_A_HEADER_PHY_INDEX]) {
                if (dev) {
                        req_info->phy_index = nla_get_u32(tb[ETHTOOL_A_HEADER_PHY_INDEX]);
                } else {
                        NL_SET_ERR_MSG_ATTR(extack, header,
                                            "phy_index set without a netdev");
                        return -EINVAL;
                }
        }

        req_info->dev = dev;
        req_info->flags = flags;
        return 0;
}

struct phy_device *ethnl_req_get_phydev(const struct ethnl_req_info *req_info,
                                        struct nlattr **tb, unsigned int header,
                                        struct netlink_ext_ack *extack)
{
        struct phy_device *phydev;

        ASSERT_RTNL();

        if (!req_info->dev)
                return NULL;

        if (!req_info->phy_index)
                return req_info->dev->phydev;

        phydev = phy_link_topo_get_phy(req_info->dev, req_info->phy_index);
        if (!phydev && tb) {
                NL_SET_ERR_MSG_ATTR(extack, tb[header],
                                    "no phy matching phyindex");
                return ERR_PTR(-ENODEV);
        }

        return phydev;
}

/**
 * ethnl_fill_reply_header() - Put common header into a reply message
 * @skb:      skb with the message
 * @dev:      network device to describe in header
 * @attrtype: attribute type to use for the nest
 *
 * Create a nested attribute with attributes describing given network device.
 *
 * Return: 0 on success, error value (-EMSGSIZE only) on error
 */
int ethnl_fill_reply_header(struct sk_buff *skb, struct net_device *dev,
                            u16 attrtype)
{
        struct nlattr *nest;

        if (!dev)
                return 0;
        nest = nla_nest_start(skb, attrtype);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u32(skb, ETHTOOL_A_HEADER_DEV_INDEX, (u32)dev->ifindex) ||
            nla_put_string(skb, ETHTOOL_A_HEADER_DEV_NAME, dev->name))
                goto nla_put_failure;
        /* If more attributes are put into reply header, ethnl_header_size()
         * must be updated to account for them.
         */

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

/**
 * ethnl_reply_init() - Create skb for a reply and fill device identification
 * @payload:      payload length (without netlink and genetlink header)
 * @dev:          device the reply is about (may be null)
 * @cmd:          ETHTOOL_MSG_* message type for reply
 * @hdr_attrtype: attribute type for common header
 * @info:         genetlink info of the received packet we respond to
 * @ehdrp:        place to store payload pointer returned by genlmsg_new()
 *
 * Return: pointer to allocated skb on success, NULL on error
 */
struct sk_buff *ethnl_reply_init(size_t payload, struct net_device *dev, u8 cmd,
                                 u16 hdr_attrtype, struct genl_info *info,
                                 void **ehdrp)
{
        struct sk_buff *skb;

        skb = genlmsg_new(payload, GFP_KERNEL);
        if (!skb)
                goto err;
        *ehdrp = genlmsg_put_reply(skb, info, &ethtool_genl_family, 0, cmd);
        if (!*ehdrp)
                goto err_free;

        if (dev) {
                int ret;

                ret = ethnl_fill_reply_header(skb, dev, hdr_attrtype);
                if (ret < 0)
                        goto err_free;
        }
        return skb;

err_free:
        nlmsg_free(skb);
err:
        if (info)
                GENL_SET_ERR_MSG(info, "failed to setup reply message");
        return NULL;
}

void *ethnl_dump_put(struct sk_buff *skb, struct netlink_callback *cb, u8 cmd)
{
        return genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
                           &ethtool_genl_family, 0, cmd);
}

void *ethnl_bcastmsg_put(struct sk_buff *skb, u8 cmd)
{
        return genlmsg_put(skb, 0, ++ethnl_bcast_seq, &ethtool_genl_family, 0,
                           cmd);
}

void *ethnl_unicast_put(struct sk_buff *skb, u32 portid, u32 seq, u8 cmd)
{
        return genlmsg_put(skb, portid, seq, &ethtool_genl_family, 0, cmd);
}

int ethnl_multicast(struct sk_buff *skb, struct net_device *dev)
{
        return genlmsg_multicast_netns(&ethtool_genl_family, dev_net(dev), skb,
                                       0, ETHNL_MCGRP_MONITOR, GFP_KERNEL);
}

/* GET request helpers */

/**
 * struct ethnl_dump_ctx - context structure for generic dumpit() callback
 * @ops:        request ops of currently processed message type
 * @req_info:   parsed request header of processed request
 * @reply_data: data needed to compose the reply
 * @pos_ifindex: saved iteration position - ifindex
 *
 * These parameters are kept in struct netlink_callback as context preserved
 * between iterations. They are initialized by ethnl_default_start() and used
 * in ethnl_default_dumpit() and ethnl_default_done().
 */
struct ethnl_dump_ctx {
        const struct ethnl_request_ops        *ops;
        struct ethnl_req_info                *req_info;
        struct ethnl_reply_data                *reply_data;
        unsigned long                        pos_ifindex;
};

/**
 * struct ethnl_perphy_dump_ctx - context for dumpit() PHY-aware callbacks
 * @ethnl_ctx: generic ethnl context
 * @ifindex: For Filtered DUMP requests, the ifindex of the targeted netdev
 * @pos_phyindex: iterator position for multi-msg DUMP
 */
struct ethnl_perphy_dump_ctx {
        struct ethnl_dump_ctx        ethnl_ctx;
        unsigned int                ifindex;
        unsigned long                pos_phyindex;
};

static const struct ethnl_request_ops *
ethnl_default_requests[__ETHTOOL_MSG_USER_CNT] = {
        [ETHTOOL_MSG_STRSET_GET]        = &ethnl_strset_request_ops,
        [ETHTOOL_MSG_LINKINFO_GET]        = &ethnl_linkinfo_request_ops,
        [ETHTOOL_MSG_LINKINFO_SET]        = &ethnl_linkinfo_request_ops,
        [ETHTOOL_MSG_LINKMODES_GET]        = &ethnl_linkmodes_request_ops,
        [ETHTOOL_MSG_LINKMODES_SET]        = &ethnl_linkmodes_request_ops,
        [ETHTOOL_MSG_LINKSTATE_GET]        = &ethnl_linkstate_request_ops,
        [ETHTOOL_MSG_DEBUG_GET]                = &ethnl_debug_request_ops,
        [ETHTOOL_MSG_DEBUG_SET]                = &ethnl_debug_request_ops,
        [ETHTOOL_MSG_WOL_GET]                = &ethnl_wol_request_ops,
        [ETHTOOL_MSG_WOL_SET]                = &ethnl_wol_request_ops,
        [ETHTOOL_MSG_FEATURES_GET]        = &ethnl_features_request_ops,
        [ETHTOOL_MSG_PRIVFLAGS_GET]        = &ethnl_privflags_request_ops,
        [ETHTOOL_MSG_PRIVFLAGS_SET]        = &ethnl_privflags_request_ops,
        [ETHTOOL_MSG_RINGS_GET]                = &ethnl_rings_request_ops,
        [ETHTOOL_MSG_RINGS_SET]                = &ethnl_rings_request_ops,
        [ETHTOOL_MSG_CHANNELS_GET]        = &ethnl_channels_request_ops,
        [ETHTOOL_MSG_CHANNELS_SET]        = &ethnl_channels_request_ops,
        [ETHTOOL_MSG_COALESCE_GET]        = &ethnl_coalesce_request_ops,
        [ETHTOOL_MSG_COALESCE_SET]        = &ethnl_coalesce_request_ops,
        [ETHTOOL_MSG_PAUSE_GET]                = &ethnl_pause_request_ops,
        [ETHTOOL_MSG_PAUSE_SET]                = &ethnl_pause_request_ops,
        [ETHTOOL_MSG_EEE_GET]                = &ethnl_eee_request_ops,
        [ETHTOOL_MSG_EEE_SET]                = &ethnl_eee_request_ops,
        [ETHTOOL_MSG_FEC_GET]                = &ethnl_fec_request_ops,
        [ETHTOOL_MSG_FEC_SET]                = &ethnl_fec_request_ops,
        [ETHTOOL_MSG_TSINFO_GET]        = &ethnl_tsinfo_request_ops,
        [ETHTOOL_MSG_MODULE_EEPROM_GET]        = &ethnl_module_eeprom_request_ops,
        [ETHTOOL_MSG_STATS_GET]                = &ethnl_stats_request_ops,
        [ETHTOOL_MSG_PHC_VCLOCKS_GET]        = &ethnl_phc_vclocks_request_ops,
        [ETHTOOL_MSG_MODULE_GET]        = &ethnl_module_request_ops,
        [ETHTOOL_MSG_MODULE_SET]        = &ethnl_module_request_ops,
        [ETHTOOL_MSG_PSE_GET]                = &ethnl_pse_request_ops,
        [ETHTOOL_MSG_PSE_SET]                = &ethnl_pse_request_ops,
        [ETHTOOL_MSG_RSS_GET]                = &ethnl_rss_request_ops,
        [ETHTOOL_MSG_RSS_SET]                = &ethnl_rss_request_ops,
        [ETHTOOL_MSG_PLCA_GET_CFG]        = &ethnl_plca_cfg_request_ops,
        [ETHTOOL_MSG_PLCA_SET_CFG]        = &ethnl_plca_cfg_request_ops,
        [ETHTOOL_MSG_PLCA_GET_STATUS]        = &ethnl_plca_status_request_ops,
        [ETHTOOL_MSG_MM_GET]                = &ethnl_mm_request_ops,
        [ETHTOOL_MSG_MM_SET]                = &ethnl_mm_request_ops,
        [ETHTOOL_MSG_TSCONFIG_GET]        = &ethnl_tsconfig_request_ops,
        [ETHTOOL_MSG_TSCONFIG_SET]        = &ethnl_tsconfig_request_ops,
        [ETHTOOL_MSG_PHY_GET]                = &ethnl_phy_request_ops,
};

static struct ethnl_dump_ctx *ethnl_dump_context(struct netlink_callback *cb)
{
        return (struct ethnl_dump_ctx *)cb->ctx;
}

static struct ethnl_perphy_dump_ctx *
ethnl_perphy_dump_context(struct netlink_callback *cb)
{
        return (struct ethnl_perphy_dump_ctx *)cb->ctx;
}

/**
 * ethnl_default_parse() - Parse request message
 * @req_info:    pointer to structure to put data into
 * @info:         genl_info from the request
 * @request_ops: struct request_ops for request type
 * @require_dev: fail if no device identified in header
 *
 * Parse universal request header and call request specific ->parse_request()
 * callback (if defined) to parse the rest of the message.
 *
 * Return: 0 on success or negative error code
 */
static int ethnl_default_parse(struct ethnl_req_info *req_info,
                               const struct genl_info *info,
                               const struct ethnl_request_ops *request_ops,
                               bool require_dev)
{
        struct nlattr **tb = info->attrs;
        int ret;

        ret = ethnl_parse_header_dev_get(req_info, tb[request_ops->hdr_attr],
                                         genl_info_net(info), info->extack,
                                         require_dev);
        if (ret < 0)
                return ret;

        if (request_ops->parse_request) {
                ret = request_ops->parse_request(req_info, tb, info->extack);
                if (ret < 0)
                        goto err_dev;
        }

        return 0;

err_dev:
        netdev_put(req_info->dev, &req_info->dev_tracker);
        req_info->dev = NULL;
        return ret;
}

/**
 * ethnl_init_reply_data() - Initialize reply data for GET request
 * @reply_data: pointer to embedded struct ethnl_reply_data
 * @ops:        instance of struct ethnl_request_ops describing the layout
 * @dev:        network device to initialize the reply for
 *
 * Fills the reply data part with zeros and sets the dev member. Must be called
 * before calling the ->fill_reply() callback (for each iteration when handling
 * dump requests).
 */
static void ethnl_init_reply_data(struct ethnl_reply_data *reply_data,
                                  const struct ethnl_request_ops *ops,
                                  struct net_device *dev)
{
        memset(reply_data, 0, ops->reply_data_size);
        reply_data->dev = dev;
}

/* default ->doit() handler for GET type requests */
static int ethnl_default_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct ethnl_reply_data *reply_data = NULL;
        struct ethnl_req_info *req_info = NULL;
        const u8 cmd = info->genlhdr->cmd;
        const struct ethnl_request_ops *ops;
        int hdr_len, reply_len;
        struct sk_buff *rskb;
        void *reply_payload;
        int ret;

        ops = ethnl_default_requests[cmd];
        if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", cmd))
                return -EOPNOTSUPP;
        if (GENL_REQ_ATTR_CHECK(info, ops->hdr_attr))
                return -EINVAL;

        req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
        if (!req_info)
                return -ENOMEM;
        reply_data = kmalloc(ops->reply_data_size, GFP_KERNEL);
        if (!reply_data) {
                kfree(req_info);
                return -ENOMEM;
        }

        ret = ethnl_default_parse(req_info, info, ops, !ops->allow_nodev_do);
        if (ret < 0)
                goto err_free;
        ethnl_init_reply_data(reply_data, ops, req_info->dev);

        rtnl_lock();
        if (req_info->dev)
                netdev_lock_ops(req_info->dev);
        ret = ops->prepare_data(req_info, reply_data, info);
        if (req_info->dev)
                netdev_unlock_ops(req_info->dev);
        rtnl_unlock();
        if (ret < 0)
                goto err_dev;
        ret = ops->reply_size(req_info, reply_data);
        if (ret < 0)
                goto err_cleanup;
        reply_len = ret;
        ret = -ENOMEM;
        rskb = ethnl_reply_init(reply_len + ethnl_reply_header_size(),
                                req_info->dev, ops->reply_cmd,
                                ops->hdr_attr, info, &reply_payload);
        if (!rskb)
                goto err_cleanup;
        hdr_len = rskb->len;
        ret = ops->fill_reply(rskb, req_info, reply_data);
        if (ret < 0)
                goto err_msg;
        WARN_ONCE(rskb->len - hdr_len > reply_len,
                  "ethnl cmd %d: calculated reply length %d, but consumed %d\n",
                  cmd, reply_len, rskb->len - hdr_len);
        if (ops->cleanup_data)
                ops->cleanup_data(reply_data);

        genlmsg_end(rskb, reply_payload);
        netdev_put(req_info->dev, &req_info->dev_tracker);
        kfree(reply_data);
        kfree(req_info);
        return genlmsg_reply(rskb, info);

err_msg:
        WARN_ONCE(ret == -EMSGSIZE, "calculated message payload length (%d) not sufficient\n", reply_len);
        nlmsg_free(rskb);
err_cleanup:
        if (ops->cleanup_data)
                ops->cleanup_data(reply_data);
err_dev:
        netdev_put(req_info->dev, &req_info->dev_tracker);
err_free:
        kfree(reply_data);
        kfree(req_info);
        return ret;
}

static int ethnl_default_dump_one(struct sk_buff *skb, struct net_device *dev,
                                  const struct ethnl_dump_ctx *ctx,
                                  const struct genl_info *info)
{
        void *ehdr;
        int ret;

        ehdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
                           &ethtool_genl_family, NLM_F_MULTI,
                           ctx->ops->reply_cmd);
        if (!ehdr)
                return -EMSGSIZE;

        ethnl_init_reply_data(ctx->reply_data, ctx->ops, dev);
        rtnl_lock();
        netdev_lock_ops(dev);
        ret = ctx->ops->prepare_data(ctx->req_info, ctx->reply_data, info);
        netdev_unlock_ops(dev);
        rtnl_unlock();
        if (ret < 0)
                goto out_cancel;
        ret = ethnl_fill_reply_header(skb, dev, ctx->ops->hdr_attr);
        if (ret < 0)
                goto out;
        ret = ctx->ops->fill_reply(skb, ctx->req_info, ctx->reply_data);

out:
        if (ctx->ops->cleanup_data)
                ctx->ops->cleanup_data(ctx->reply_data);
out_cancel:
        ctx->reply_data->dev = NULL;
        if (ret < 0)
                genlmsg_cancel(skb, ehdr);
        else
                genlmsg_end(skb, ehdr);
        return ret;
}

/* Default ->dumpit() handler for GET requests. */
static int ethnl_default_dumpit(struct sk_buff *skb,
                                struct netlink_callback *cb)
{
        struct ethnl_dump_ctx *ctx = ethnl_dump_context(cb);
        struct net *net = sock_net(skb->sk);
        netdevice_tracker dev_tracker;
        struct net_device *dev;
        int ret = 0;

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ctx->pos_ifindex) {
                netdev_hold(dev, &dev_tracker, GFP_ATOMIC);
                rcu_read_unlock();

                ret = ethnl_default_dump_one(skb, dev, ctx, genl_info_dump(cb));

                rcu_read_lock();
                netdev_put(dev, &dev_tracker);

                if (ret < 0 && ret != -EOPNOTSUPP) {
                        if (likely(skb->len))
                                ret = skb->len;
                        break;
                }
                ret = 0;
        }
        rcu_read_unlock();

        return ret;
}

/* generic ->start() handler for GET requests */
static int ethnl_default_start(struct netlink_callback *cb)
{
        const struct genl_dumpit_info *info = genl_dumpit_info(cb);
        struct ethnl_dump_ctx *ctx = ethnl_dump_context(cb);
        struct ethnl_reply_data *reply_data;
        const struct ethnl_request_ops *ops;
        struct ethnl_req_info *req_info;
        struct genlmsghdr *ghdr;
        int ret;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));

        ghdr = nlmsg_data(cb->nlh);
        ops = ethnl_default_requests[ghdr->cmd];
        if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", ghdr->cmd))
                return -EOPNOTSUPP;
        req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
        if (!req_info)
                return -ENOMEM;
        reply_data = kmalloc(ops->reply_data_size, GFP_KERNEL);
        if (!reply_data) {
                ret = -ENOMEM;
                goto free_req_info;
        }

        ret = ethnl_default_parse(req_info, &info->info, ops, false);
        if (ret < 0)
                goto free_reply_data;
        if (req_info->dev) {
                /* We ignore device specification in dump requests but as the
                 * same parser as for non-dump (doit) requests is used, it
                 * would take reference to the device if it finds one
                 */
                netdev_put(req_info->dev, &req_info->dev_tracker);
                req_info->dev = NULL;
        }

        ctx->ops = ops;
        ctx->req_info = req_info;
        ctx->reply_data = reply_data;
        ctx->pos_ifindex = 0;

        return 0;

free_reply_data:
        kfree(reply_data);
free_req_info:
        kfree(req_info);

        return ret;
}

/* per-PHY ->start() handler for GET requests */
static int ethnl_perphy_start(struct netlink_callback *cb)
{
        struct ethnl_perphy_dump_ctx *phy_ctx = ethnl_perphy_dump_context(cb);
        const struct genl_dumpit_info *info = genl_dumpit_info(cb);
        struct ethnl_dump_ctx *ctx = &phy_ctx->ethnl_ctx;
        struct ethnl_reply_data *reply_data;
        const struct ethnl_request_ops *ops;
        struct ethnl_req_info *req_info;
        struct genlmsghdr *ghdr;
        int ret;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));

        ghdr = nlmsg_data(cb->nlh);
        ops = ethnl_default_requests[ghdr->cmd];
        if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", ghdr->cmd))
                return -EOPNOTSUPP;
        req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
        if (!req_info)
                return -ENOMEM;
        reply_data = kmalloc(ops->reply_data_size, GFP_KERNEL);
        if (!reply_data) {
                ret = -ENOMEM;
                goto free_req_info;
        }

        /* Unlike per-dev dump, don't ignore dev. The dump handler
         * will notice it and dump PHYs from given dev. We only keep track of
         * the dev's ifindex, .dumpit() will grab and release the netdev itself.
         */
        ret = ethnl_default_parse(req_info, &info->info, ops, false);
        if (ret < 0)
                goto free_reply_data;
        if (req_info->dev) {
                phy_ctx->ifindex = req_info->dev->ifindex;
                netdev_put(req_info->dev, &req_info->dev_tracker);
                req_info->dev = NULL;
        }

        ctx->ops = ops;
        ctx->req_info = req_info;
        ctx->reply_data = reply_data;
        ctx->pos_ifindex = 0;

        return 0;

free_reply_data:
        kfree(reply_data);
free_req_info:
        kfree(req_info);

        return ret;
}

static int ethnl_perphy_dump_one_dev(struct sk_buff *skb,
                                     struct ethnl_perphy_dump_ctx *ctx,
                                     const struct genl_info *info)
{
        struct ethnl_dump_ctx *ethnl_ctx = &ctx->ethnl_ctx;
        struct net_device *dev = ethnl_ctx->req_info->dev;
        struct phy_device_node *pdn;
        int ret;

        if (!dev->link_topo)
                return 0;

        xa_for_each_start(&dev->link_topo->phys, ctx->pos_phyindex, pdn,
                          ctx->pos_phyindex) {
                ethnl_ctx->req_info->phy_index = ctx->pos_phyindex;

                /* We can re-use the original dump_one as ->prepare_data in
                 * commands use ethnl_req_get_phydev(), which gets the PHY from
                 * the req_info->phy_index
                 */
                ret = ethnl_default_dump_one(skb, dev, ethnl_ctx, info);
                if (ret)
                        return ret;
        }

        ctx->pos_phyindex = 0;

        return 0;
}

static int ethnl_perphy_dump_all_dev(struct sk_buff *skb,
                                     struct ethnl_perphy_dump_ctx *ctx,
                                     const struct genl_info *info)
{
        struct ethnl_dump_ctx *ethnl_ctx = &ctx->ethnl_ctx;
        struct net *net = sock_net(skb->sk);
        netdevice_tracker dev_tracker;
        struct net_device *dev;
        int ret = 0;

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ethnl_ctx->pos_ifindex) {
                netdev_hold(dev, &dev_tracker, GFP_ATOMIC);
                rcu_read_unlock();

                /* per-PHY commands use ethnl_req_get_phydev(), which needs the
                 * net_device in the req_info
                 */
                ethnl_ctx->req_info->dev = dev;
                ret = ethnl_perphy_dump_one_dev(skb, ctx, info);

                rcu_read_lock();
                netdev_put(dev, &dev_tracker);
                ethnl_ctx->req_info->dev = NULL;

                if (ret < 0 && ret != -EOPNOTSUPP) {
                        if (likely(skb->len))
                                ret = skb->len;
                        break;
                }
                ret = 0;
        }
        rcu_read_unlock();

        return ret;
}

/* per-PHY ->dumpit() handler for GET requests. */
static int ethnl_perphy_dumpit(struct sk_buff *skb,
                               struct netlink_callback *cb)
{
        struct ethnl_perphy_dump_ctx *ctx = ethnl_perphy_dump_context(cb);
        const struct genl_dumpit_info *info = genl_dumpit_info(cb);
        struct ethnl_dump_ctx *ethnl_ctx = &ctx->ethnl_ctx;
        int ret = 0;

        if (ctx->ifindex) {
                netdevice_tracker dev_tracker;
                struct net_device *dev;

                dev = netdev_get_by_index(genl_info_net(&info->info),
                                          ctx->ifindex, &dev_tracker,
                                          GFP_KERNEL);
                if (!dev)
                        return -ENODEV;

                ethnl_ctx->req_info->dev = dev;
                ret = ethnl_perphy_dump_one_dev(skb, ctx, genl_info_dump(cb));

                if (ret < 0 && ret != -EOPNOTSUPP && likely(skb->len))
                        ret = skb->len;

                netdev_put(dev, &dev_tracker);
        } else {
                ret = ethnl_perphy_dump_all_dev(skb, ctx, genl_info_dump(cb));
        }

        return ret;
}

/* per-PHY ->done() handler for GET requests */
static int ethnl_perphy_done(struct netlink_callback *cb)
{
        struct ethnl_perphy_dump_ctx *ctx = ethnl_perphy_dump_context(cb);
        struct ethnl_dump_ctx *ethnl_ctx = &ctx->ethnl_ctx;

        kfree(ethnl_ctx->reply_data);
        kfree(ethnl_ctx->req_info);

        return 0;
}

/* default ->done() handler for GET requests */
static int ethnl_default_done(struct netlink_callback *cb)
{
        struct ethnl_dump_ctx *ctx = ethnl_dump_context(cb);

        kfree(ctx->reply_data);
        kfree(ctx->req_info);

        return 0;
}

static int ethnl_default_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        const struct ethnl_request_ops *ops;
        const u8 cmd = info->genlhdr->cmd;
        struct ethnl_req_info *req_info;
        struct net_device *dev;
        int ret;

        ops = ethnl_default_requests[cmd];
        if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", cmd))
                return -EOPNOTSUPP;
        if (GENL_REQ_ATTR_CHECK(info, ops->hdr_attr))
                return -EINVAL;

        req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
        if (!req_info)
                return -ENOMEM;

        ret = ethnl_default_parse(req_info, info,  ops, true);
        if (ret < 0)
                goto out_free_req;

        if (ops->set_validate) {
                ret = ops->set_validate(req_info, info);
                /* 0 means nothing to do */
                if (ret <= 0)
                        goto out_dev;
        }

        dev = req_info->dev;

        rtnl_lock();
        netdev_lock_ops(dev);
        dev->cfg_pending = kmemdup(dev->cfg, sizeof(*dev->cfg),
                                   GFP_KERNEL_ACCOUNT);
        if (!dev->cfg_pending) {
                ret = -ENOMEM;
                goto out_tie_cfg;
        }

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto out_free_cfg;

        ret = ops->set(req_info, info);
        if (ret < 0)
                goto out_ops;

        swap(dev->cfg, dev->cfg_pending);
        if (!ret)
                goto out_ops;
        ethnl_notify(dev, ops->set_ntf_cmd, req_info);

        ret = 0;
out_ops:
        ethnl_ops_complete(dev);
out_free_cfg:
        kfree(dev->cfg_pending);
out_tie_cfg:
        dev->cfg_pending = dev->cfg;
        netdev_unlock_ops(dev);
        rtnl_unlock();
out_dev:
        ethnl_parse_header_dev_put(req_info);
out_free_req:
        kfree(req_info);
        return ret;
}

static const struct ethnl_request_ops *
ethnl_default_notify_ops[ETHTOOL_MSG_KERNEL_MAX + 1] = {
        [ETHTOOL_MSG_LINKINFO_NTF]        = &ethnl_linkinfo_request_ops,
        [ETHTOOL_MSG_LINKMODES_NTF]        = &ethnl_linkmodes_request_ops,
        [ETHTOOL_MSG_DEBUG_NTF]                = &ethnl_debug_request_ops,
        [ETHTOOL_MSG_WOL_NTF]                = &ethnl_wol_request_ops,
        [ETHTOOL_MSG_FEATURES_NTF]        = &ethnl_features_request_ops,
        [ETHTOOL_MSG_PRIVFLAGS_NTF]        = &ethnl_privflags_request_ops,
        [ETHTOOL_MSG_RINGS_NTF]                = &ethnl_rings_request_ops,
        [ETHTOOL_MSG_CHANNELS_NTF]        = &ethnl_channels_request_ops,
        [ETHTOOL_MSG_COALESCE_NTF]        = &ethnl_coalesce_request_ops,
        [ETHTOOL_MSG_PAUSE_NTF]                = &ethnl_pause_request_ops,
        [ETHTOOL_MSG_EEE_NTF]                = &ethnl_eee_request_ops,
        [ETHTOOL_MSG_FEC_NTF]                = &ethnl_fec_request_ops,
        [ETHTOOL_MSG_MODULE_NTF]        = &ethnl_module_request_ops,
        [ETHTOOL_MSG_PLCA_NTF]                = &ethnl_plca_cfg_request_ops,
        [ETHTOOL_MSG_MM_NTF]                = &ethnl_mm_request_ops,
        [ETHTOOL_MSG_RSS_NTF]                = &ethnl_rss_request_ops,
        [ETHTOOL_MSG_RSS_CREATE_NTF]        = &ethnl_rss_request_ops,
};

/* default notification handler */
static void ethnl_default_notify(struct net_device *dev, unsigned int cmd,
                                 const struct ethnl_req_info *orig_req_info)
{
        struct ethnl_reply_data *reply_data;
        const struct ethnl_request_ops *ops;
        struct ethnl_req_info *req_info;
        struct genl_info info;
        struct sk_buff *skb;
        void *reply_payload;
        int reply_len;
        int ret;

        genl_info_init_ntf(&info, &ethtool_genl_family, cmd);

        if (WARN_ONCE(cmd > ETHTOOL_MSG_KERNEL_MAX ||
                      !ethnl_default_notify_ops[cmd],
                      "unexpected notification type %u\n", cmd))
                return;
        ops = ethnl_default_notify_ops[cmd];
        req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
        if (!req_info)
                return;
        reply_data = kmalloc(ops->reply_data_size, GFP_KERNEL);
        if (!reply_data) {
                kfree(req_info);
                return;
        }

        req_info->dev = dev;
        req_info->flags |= ETHTOOL_FLAG_COMPACT_BITSETS;
        if (orig_req_info) {
                req_info->phy_index = orig_req_info->phy_index;
                memcpy(&req_info[1], &orig_req_info[1],
                       ops->req_info_size - sizeof(*req_info));
        }

        netdev_ops_assert_locked(dev);

        ethnl_init_reply_data(reply_data, ops, dev);
        ret = ops->prepare_data(req_info, reply_data, &info);
        if (ret < 0)
                goto err_rep;
        ret = ops->reply_size(req_info, reply_data);
        if (ret < 0)
                goto err_cleanup;
        reply_len = ret + ethnl_reply_header_size();
        skb = genlmsg_new(reply_len, GFP_KERNEL);
        if (!skb)
                goto err_cleanup;
        reply_payload = ethnl_bcastmsg_put(skb, cmd);
        if (!reply_payload)
                goto err_skb;
        ret = ethnl_fill_reply_header(skb, dev, ops->hdr_attr);
        if (ret < 0)
                goto err_msg;
        ret = ops->fill_reply(skb, req_info, reply_data);
        if (ret < 0)
                goto err_msg;
        if (ops->cleanup_data)
                ops->cleanup_data(reply_data);

        genlmsg_end(skb, reply_payload);
        kfree(reply_data);
        kfree(req_info);
        ethnl_multicast(skb, dev);
        return;

err_msg:
        WARN_ONCE(ret == -EMSGSIZE,
                  "calculated message payload length (%d) not sufficient\n",
                  reply_len);
err_skb:
        nlmsg_free(skb);
err_cleanup:
        if (ops->cleanup_data)
                ops->cleanup_data(reply_data);
err_rep:
        kfree(reply_data);
        kfree(req_info);
        return;
}

/* notifications */

typedef void (*ethnl_notify_handler_t)(struct net_device *dev, unsigned int cmd,
                                       const struct ethnl_req_info *req_info);

static const ethnl_notify_handler_t ethnl_notify_handlers[] = {
        [ETHTOOL_MSG_LINKINFO_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_LINKMODES_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_DEBUG_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_WOL_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_FEATURES_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_PRIVFLAGS_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_RINGS_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_CHANNELS_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_COALESCE_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_PAUSE_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_EEE_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_FEC_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_MODULE_NTF]        = ethnl_default_notify,
        [ETHTOOL_MSG_PLCA_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_MM_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_RSS_NTF]                = ethnl_default_notify,
        [ETHTOOL_MSG_RSS_CREATE_NTF]        = ethnl_default_notify,
};

void ethnl_notify(struct net_device *dev, unsigned int cmd,
                  const struct ethnl_req_info *req_info)
{
        if (unlikely(!ethnl_ok))
                return;
        ASSERT_RTNL();

        if (likely(cmd < ARRAY_SIZE(ethnl_notify_handlers) &&
                   ethnl_notify_handlers[cmd]))
                ethnl_notify_handlers[cmd](dev, cmd, req_info);
        else
                WARN_ONCE(1, "notification %u not implemented (dev=%s)\n",
                          cmd, netdev_name(dev));
}

void ethtool_notify(struct net_device *dev, unsigned int cmd)
{
        ethnl_notify(dev, cmd, NULL);
}
EXPORT_SYMBOL(ethtool_notify);

static void ethnl_notify_features(struct netdev_notifier_info *info)
{
        struct net_device *dev = netdev_notifier_info_to_dev(info);

        ethtool_notify(dev, ETHTOOL_MSG_FEATURES_NTF);
}

static int ethnl_netdev_event(struct notifier_block *this, unsigned long event,
                              void *ptr)
{
        struct netdev_notifier_info *info = ptr;
        struct netlink_ext_ack *extack;
        struct net_device *dev;

        dev = netdev_notifier_info_to_dev(info);
        extack = netdev_notifier_info_to_extack(info);

        switch (event) {
        case NETDEV_FEAT_CHANGE:
                ethnl_notify_features(ptr);
                break;
        case NETDEV_PRE_UP:
                if (dev->ethtool->module_fw_flash_in_progress) {
                        NL_SET_ERR_MSG(extack, "Can't set port up while flashing module firmware");
                        return NOTIFY_BAD;
                }
        }

        return NOTIFY_DONE;
}

static struct notifier_block ethnl_netdev_notifier = {
        .notifier_call = ethnl_netdev_event,
};

/* genetlink setup */

static const struct genl_ops ethtool_genl_ops[] = {
        {
                .cmd        = ETHTOOL_MSG_STRSET_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_strset_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_strset_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_LINKINFO_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_linkinfo_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_linkinfo_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_LINKINFO_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_linkinfo_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_linkinfo_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_LINKMODES_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_linkmodes_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_linkmodes_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_LINKMODES_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_linkmodes_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_linkmodes_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_LINKSTATE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_linkstate_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_linkstate_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_DEBUG_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_debug_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_debug_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_DEBUG_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_debug_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_debug_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_WOL_GET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_wol_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_wol_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_WOL_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_wol_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_wol_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_FEATURES_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_features_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_features_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_FEATURES_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_set_features,
                .policy = ethnl_features_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_features_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PRIVFLAGS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_privflags_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_privflags_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PRIVFLAGS_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_privflags_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_privflags_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RINGS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_rings_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_rings_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RINGS_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_rings_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_rings_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_CHANNELS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_channels_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_channels_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_CHANNELS_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_channels_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_channels_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_COALESCE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_coalesce_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_coalesce_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_COALESCE_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_coalesce_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_coalesce_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PAUSE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_pause_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_pause_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PAUSE_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_pause_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_pause_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_EEE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_eee_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_eee_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_EEE_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_eee_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_eee_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_TSINFO_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_tsinfo_start,
                .dumpit        = ethnl_tsinfo_dumpit,
                .done        = ethnl_tsinfo_done,
                .policy = ethnl_tsinfo_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_tsinfo_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_CABLE_TEST_ACT,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_act_cable_test,
                .policy = ethnl_cable_test_act_policy,
                .maxattr = ARRAY_SIZE(ethnl_cable_test_act_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_CABLE_TEST_TDR_ACT,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_act_cable_test_tdr,
                .policy = ethnl_cable_test_tdr_act_policy,
                .maxattr = ARRAY_SIZE(ethnl_cable_test_tdr_act_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_TUNNEL_INFO_GET,
                .doit        = ethnl_tunnel_info_doit,
                .start        = ethnl_tunnel_info_start,
                .dumpit        = ethnl_tunnel_info_dumpit,
                .policy = ethnl_tunnel_info_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_tunnel_info_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_FEC_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_fec_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_fec_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_FEC_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_fec_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_fec_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MODULE_EEPROM_GET,
                .flags  = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_module_eeprom_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_module_eeprom_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_STATS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_stats_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_stats_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PHC_VCLOCKS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_phc_vclocks_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_phc_vclocks_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MODULE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_module_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_module_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MODULE_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_module_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_module_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PSE_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_perphy_start,
                .dumpit        = ethnl_perphy_dumpit,
                .done        = ethnl_perphy_done,
                .policy = ethnl_pse_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_pse_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PSE_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_pse_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_pse_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RSS_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_rss_dump_start,
                .dumpit        = ethnl_rss_dumpit,
                .policy = ethnl_rss_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_rss_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PLCA_GET_CFG,
                .doit        = ethnl_default_doit,
                .start        = ethnl_perphy_start,
                .dumpit        = ethnl_perphy_dumpit,
                .done        = ethnl_perphy_done,
                .policy = ethnl_plca_get_cfg_policy,
                .maxattr = ARRAY_SIZE(ethnl_plca_get_cfg_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PLCA_SET_CFG,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_plca_set_cfg_policy,
                .maxattr = ARRAY_SIZE(ethnl_plca_set_cfg_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PLCA_GET_STATUS,
                .doit        = ethnl_default_doit,
                .start        = ethnl_perphy_start,
                .dumpit        = ethnl_perphy_dumpit,
                .done        = ethnl_perphy_done,
                .policy = ethnl_plca_get_status_policy,
                .maxattr = ARRAY_SIZE(ethnl_plca_get_status_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MM_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_mm_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_mm_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MM_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_mm_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_mm_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_MODULE_FW_FLASH_ACT,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_act_module_fw_flash,
                .policy        = ethnl_module_fw_flash_act_policy,
                .maxattr = ARRAY_SIZE(ethnl_module_fw_flash_act_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_PHY_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_perphy_start,
                .dumpit        = ethnl_perphy_dumpit,
                .done        = ethnl_perphy_done,
                .policy = ethnl_phy_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_phy_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_TSCONFIG_GET,
                .doit        = ethnl_default_doit,
                .start        = ethnl_default_start,
                .dumpit        = ethnl_default_dumpit,
                .done        = ethnl_default_done,
                .policy = ethnl_tsconfig_get_policy,
                .maxattr = ARRAY_SIZE(ethnl_tsconfig_get_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_TSCONFIG_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_tsconfig_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_tsconfig_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RSS_SET,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_default_set_doit,
                .policy = ethnl_rss_set_policy,
                .maxattr = ARRAY_SIZE(ethnl_rss_set_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RSS_CREATE_ACT,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_rss_create_doit,
                .policy        = ethnl_rss_create_policy,
                .maxattr = ARRAY_SIZE(ethnl_rss_create_policy) - 1,
        },
        {
                .cmd        = ETHTOOL_MSG_RSS_DELETE_ACT,
                .flags        = GENL_UNS_ADMIN_PERM,
                .doit        = ethnl_rss_delete_doit,
                .policy        = ethnl_rss_delete_policy,
                .maxattr = ARRAY_SIZE(ethnl_rss_delete_policy) - 1,
        },
};

static const struct genl_multicast_group ethtool_nl_mcgrps[] = {
        [ETHNL_MCGRP_MONITOR] = { .name = ETHTOOL_MCGRP_MONITOR_NAME },
};

static struct genl_family ethtool_genl_family __ro_after_init = {
        .name                = ETHTOOL_GENL_NAME,
        .version        = ETHTOOL_GENL_VERSION,
        .netnsok        = true,
        .parallel_ops        = true,
        .ops                = ethtool_genl_ops,
        .n_ops                = ARRAY_SIZE(ethtool_genl_ops),
        .resv_start_op        = ETHTOOL_MSG_MODULE_GET + 1,
        .mcgrps                = ethtool_nl_mcgrps,
        .n_mcgrps        = ARRAY_SIZE(ethtool_nl_mcgrps),
        .sock_priv_size                = sizeof(struct ethnl_sock_priv),
        .sock_priv_destroy        = ethnl_sock_priv_destroy,
};

/* module setup */

static int __init ethnl_init(void)
{
        int ret;

        ret = genl_register_family(&ethtool_genl_family);
        if (WARN(ret < 0, "ethtool: genetlink family registration failed"))
                return ret;
        ethnl_ok = true;

        ret = register_netdevice_notifier(&ethnl_netdev_notifier);
        WARN(ret < 0, "ethtool: net device notifier registration failed");
        return ret;
}

subsys_initcall(ethnl_init);























































































   19 



   20 


   20 






















































































































   20 
































































































































































































































































   10 














    1 









   10 















   20 

   20 


















   22 










    3 





   19 




    1 







   19 





   20 

   19 
    1 









   23 











   23 










   22 



















   22 









   23 
   23 


   23 





   22 







   23 

































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2005,2006,2007,2008 IBM Corporation
 *
 * Authors:
 * Mimi Zohar <zohar@us.ibm.com>
 * Kylene Hall <kjhall@us.ibm.com>
 *
 * File: ima_crypto.c
 *        Calculates md5/sha1 file hash, template hash, boot-aggreate hash
 */

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/ratelimit.h>
#include <linux/file.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <crypto/hash.h>

#include "ima.h"

/* minimum file size for ahash use */
static unsigned long ima_ahash_minsize;
module_param_named(ahash_minsize, ima_ahash_minsize, ulong, 0644);
MODULE_PARM_DESC(ahash_minsize, "Minimum file size for ahash use");

/* default is 0 - 1 page. */
static int ima_maxorder;
static unsigned int ima_bufsize = PAGE_SIZE;

static int param_set_bufsize(const char *val, const struct kernel_param *kp)
{
        unsigned long long size;
        int order;

        size = memparse(val, NULL);
        order = get_order(size);
        if (order > MAX_PAGE_ORDER)
                return -EINVAL;
        ima_maxorder = order;
        ima_bufsize = PAGE_SIZE << order;
        return 0;
}

static const struct kernel_param_ops param_ops_bufsize = {
        .set = param_set_bufsize,
        .get = param_get_uint,
};
#define param_check_bufsize(name, p) __param_check(name, p, unsigned int)

module_param_named(ahash_bufsize, ima_bufsize, bufsize, 0644);
MODULE_PARM_DESC(ahash_bufsize, "Maximum ahash buffer size");

static struct crypto_shash *ima_shash_tfm;
static struct crypto_ahash *ima_ahash_tfm;

int ima_sha1_idx __ro_after_init;
int ima_hash_algo_idx __ro_after_init;
/*
 * Additional number of slots reserved, as needed, for SHA1
 * and IMA default algo.
 */
int ima_extra_slots __ro_after_init;

struct ima_algo_desc *ima_algo_array __ro_after_init;

static int __init ima_init_ima_crypto(void)
{
        long rc;

        ima_shash_tfm = crypto_alloc_shash(hash_algo_name[ima_hash_algo], 0, 0);
        if (IS_ERR(ima_shash_tfm)) {
                rc = PTR_ERR(ima_shash_tfm);
                pr_err("Can not allocate %s (reason: %ld)\n",
                       hash_algo_name[ima_hash_algo], rc);
                return rc;
        }
        pr_info("Allocated hash algorithm: %s\n",
                hash_algo_name[ima_hash_algo]);
        return 0;
}

static struct crypto_shash *ima_alloc_tfm(enum hash_algo algo)
{
        struct crypto_shash *tfm = ima_shash_tfm;
        int rc, i;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo == ima_hash_algo)
                return tfm;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm && ima_algo_array[i].algo == algo)
                        return ima_algo_array[i].tfm;

        tfm = crypto_alloc_shash(hash_algo_name[algo], 0, 0);
        if (IS_ERR(tfm)) {
                rc = PTR_ERR(tfm);
                pr_err("Can not allocate %s (reason: %d)\n",
                       hash_algo_name[algo], rc);
        }
        return tfm;
}

int __init ima_init_crypto(void)
{
        enum hash_algo algo;
        long rc;
        int i;

        rc = ima_init_ima_crypto();
        if (rc)
                return rc;

        ima_sha1_idx = -1;
        ima_hash_algo_idx = -1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (algo == HASH_ALGO_SHA1)
                        ima_sha1_idx = i;

                if (algo == ima_hash_algo)
                        ima_hash_algo_idx = i;
        }

        if (ima_sha1_idx < 0) {
                ima_sha1_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;
                if (ima_hash_algo == HASH_ALGO_SHA1)
                        ima_hash_algo_idx = ima_sha1_idx;
        }

        if (ima_hash_algo_idx < 0)
                ima_hash_algo_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;

        ima_algo_array = kcalloc(NR_BANKS(ima_tpm_chip) + ima_extra_slots,
                                 sizeof(*ima_algo_array), GFP_KERNEL);
        if (!ima_algo_array) {
                rc = -ENOMEM;
                goto out;
        }

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                ima_algo_array[i].algo = algo;

                /* unknown TPM algorithm */
                if (algo == HASH_ALGO__LAST)
                        continue;

                if (algo == ima_hash_algo) {
                        ima_algo_array[i].tfm = ima_shash_tfm;
                        continue;
                }

                ima_algo_array[i].tfm = ima_alloc_tfm(algo);
                if (IS_ERR(ima_algo_array[i].tfm)) {
                        if (algo == HASH_ALGO_SHA1) {
                                rc = PTR_ERR(ima_algo_array[i].tfm);
                                ima_algo_array[i].tfm = NULL;
                                goto out_array;
                        }

                        ima_algo_array[i].tfm = NULL;
                }
        }

        if (ima_sha1_idx >= NR_BANKS(ima_tpm_chip)) {
                if (ima_hash_algo == HASH_ALGO_SHA1) {
                        ima_algo_array[ima_sha1_idx].tfm = ima_shash_tfm;
                } else {
                        ima_algo_array[ima_sha1_idx].tfm =
                                                ima_alloc_tfm(HASH_ALGO_SHA1);
                        if (IS_ERR(ima_algo_array[ima_sha1_idx].tfm)) {
                                rc = PTR_ERR(ima_algo_array[ima_sha1_idx].tfm);
                                goto out_array;
                        }
                }

                ima_algo_array[ima_sha1_idx].algo = HASH_ALGO_SHA1;
        }

        if (ima_hash_algo_idx >= NR_BANKS(ima_tpm_chip) &&
            ima_hash_algo_idx != ima_sha1_idx) {
                ima_algo_array[ima_hash_algo_idx].tfm = ima_shash_tfm;
                ima_algo_array[ima_hash_algo_idx].algo = ima_hash_algo;
        }

        return 0;
out_array:
        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (!ima_algo_array[i].tfm ||
                    ima_algo_array[i].tfm == ima_shash_tfm)
                        continue;

                crypto_free_shash(ima_algo_array[i].tfm);
        }
        kfree(ima_algo_array);
out:
        crypto_free_shash(ima_shash_tfm);
        return rc;
}

static void ima_free_tfm(struct crypto_shash *tfm)
{
        int i;

        if (tfm == ima_shash_tfm)
                return;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm == tfm)
                        return;

        crypto_free_shash(tfm);
}

/**
 * ima_alloc_pages() - Allocate contiguous pages.
 * @max_size:       Maximum amount of memory to allocate.
 * @allocated_size: Returned size of actual allocation.
 * @last_warn:      Should the min_size allocation warn or not.
 *
 * Tries to do opportunistic allocation for memory first trying to allocate
 * max_size amount of memory and then splitting that until zero order is
 * reached. Allocation is tried without generating allocation warnings unless
 * last_warn is set. Last_warn set affects only last allocation of zero order.
 *
 * By default, ima_maxorder is 0 and it is equivalent to kmalloc(GFP_KERNEL)
 *
 * Return pointer to allocated memory, or NULL on failure.
 */
static void *ima_alloc_pages(loff_t max_size, size_t *allocated_size,
                             int last_warn)
{
        void *ptr;
        int order = ima_maxorder;
        gfp_t gfp_mask = __GFP_RECLAIM | __GFP_NOWARN | __GFP_NORETRY;

        if (order)
                order = min(get_order(max_size), order);

        for (; order; order--) {
                ptr = (void *)__get_free_pages(gfp_mask, order);
                if (ptr) {
                        *allocated_size = PAGE_SIZE << order;
                        return ptr;
                }
        }

        /* order is zero - one page */

        gfp_mask = GFP_KERNEL;

        if (!last_warn)
                gfp_mask |= __GFP_NOWARN;

        ptr = (void *)__get_free_pages(gfp_mask, 0);
        if (ptr) {
                *allocated_size = PAGE_SIZE;
                return ptr;
        }

        *allocated_size = 0;
        return NULL;
}

/**
 * ima_free_pages() - Free pages allocated by ima_alloc_pages().
 * @ptr:  Pointer to allocated pages.
 * @size: Size of allocated buffer.
 */
static void ima_free_pages(void *ptr, size_t size)
{
        if (!ptr)
                return;
        free_pages((unsigned long)ptr, get_order(size));
}

static struct crypto_ahash *ima_alloc_atfm(enum hash_algo algo)
{
        struct crypto_ahash *tfm = ima_ahash_tfm;
        int rc;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo != ima_hash_algo || !tfm) {
                tfm = crypto_alloc_ahash(hash_algo_name[algo], 0, 0);
                if (!IS_ERR(tfm)) {
                        if (algo == ima_hash_algo)
                                ima_ahash_tfm = tfm;
                } else {
                        rc = PTR_ERR(tfm);
                        pr_err("Can not allocate %s (reason: %d)\n",
                               hash_algo_name[algo], rc);
                }
        }
        return tfm;
}

static void ima_free_atfm(struct crypto_ahash *tfm)
{
        if (tfm != ima_ahash_tfm)
                crypto_free_ahash(tfm);
}

static inline int ahash_wait(int err, struct crypto_wait *wait)
{

        err = crypto_wait_req(err, wait);

        if (err)
                pr_crit_ratelimited("ahash calculation failed: err: %d\n", err);

        return err;
}

static int ima_calc_file_hash_atfm(struct file *file,
                                   struct ima_digest_data *hash,
                                   struct crypto_ahash *tfm)
{
        loff_t i_size, offset;
        char *rbuf[2] = { NULL, };
        int rc, rbuf_len, active = 0, ahash_rc = 0;
        struct ahash_request *req;
        struct scatterlist sg[1];
        struct crypto_wait wait;
        size_t rbuf_size[2];

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out1;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out2;

        /*
         * Try to allocate maximum size of memory.
         * Fail if even a single page cannot be allocated.
         */
        rbuf[0] = ima_alloc_pages(i_size, &rbuf_size[0], 1);
        if (!rbuf[0]) {
                rc = -ENOMEM;
                goto out1;
        }

        /* Only allocate one buffer if that is enough. */
        if (i_size > rbuf_size[0]) {
                /*
                 * Try to allocate secondary buffer. If that fails fallback to
                 * using single buffering. Use previous memory allocation size
                 * as baseline for possible allocation size.
                 */
                rbuf[1] = ima_alloc_pages(i_size - rbuf_size[0],
                                          &rbuf_size[1], 0);
        }

        for (offset = 0; offset < i_size; offset += rbuf_len) {
                if (!rbuf[1] && offset) {
                        /* Not using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }
                /* read buffer */
                rbuf_len = min_t(loff_t, i_size - offset, rbuf_size[active]);
                rc = integrity_kernel_read(file, offset, rbuf[active],
                                           rbuf_len);
                if (rc != rbuf_len) {
                        if (rc >= 0)
                                rc = -EINVAL;
                        /*
                         * Forward current rc, do not overwrite with return value
                         * from ahash_wait()
                         */
                        ahash_wait(ahash_rc, &wait);
                        goto out3;
                }

                if (rbuf[1] && offset) {
                        /* Using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }

                sg_init_one(&sg[0], rbuf[active], rbuf_len);
                ahash_request_set_crypt(req, sg, NULL, rbuf_len);

                ahash_rc = crypto_ahash_update(req);

                if (rbuf[1])
                        active = !active; /* swap buffers, if we use two */
        }
        /* wait for the last update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
out3:
        ima_free_pages(rbuf[0], rbuf_size[0]);
        ima_free_pages(rbuf[1], rbuf_size[1]);
out2:
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out1:
        ahash_request_free(req);
        return rc;
}

static int ima_calc_file_ahash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_atfm(file, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int ima_calc_file_hash_tfm(struct file *file,
                                  struct ima_digest_data *hash,
                                  struct crypto_shash *tfm)
{
        loff_t i_size, offset = 0;
        char *rbuf;
        int rc;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out;

        rbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
        if (!rbuf)
                return -ENOMEM;

        while (offset < i_size) {
                int rbuf_len;

                rbuf_len = integrity_kernel_read(file, offset, rbuf, PAGE_SIZE);
                if (rbuf_len < 0) {
                        rc = rbuf_len;
                        break;
                }
                if (rbuf_len == 0) {        /* unexpected EOF */
                        rc = -EINVAL;
                        break;
                }
                offset += rbuf_len;

                rc = crypto_shash_update(shash, rbuf, rbuf_len);
                if (rc)
                        break;
        }
        kfree(rbuf);
out:
        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int ima_calc_file_shash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_tfm(file, hash, tfm);

        ima_free_tfm(tfm);

        return rc;
}

/*
 * ima_calc_file_hash - calculate file hash
 *
 * Asynchronous hash (ahash) allows using HW acceleration for calculating
 * a hash. ahash performance varies for different data sizes on different
 * crypto accelerators. shash performance might be better for smaller files.
 * The 'ima.ahash_minsize' module parameter allows specifying the best
 * minimum file size for using ahash on the system.
 *
 * If the ima.ahash_minsize parameter is not specified, this function uses
 * shash for the hash calculation.  If ahash fails, it falls back to using
 * shash.
 */
int ima_calc_file_hash(struct file *file, struct ima_digest_data *hash)
{
        loff_t i_size;
        int rc;
        struct file *f = file;
        bool new_file_instance = false;

        /*
         * For consistency, fail file's opened with the O_DIRECT flag on
         * filesystems mounted with/without DAX option.
         */
        if (file->f_flags & O_DIRECT) {
                hash->length = hash_digest_size[ima_hash_algo];
                hash->algo = ima_hash_algo;
                return -EINVAL;
        }

        /* Open a new file instance in O_RDONLY if we cannot read */
        if (!(file->f_mode & FMODE_READ)) {
                int flags = file->f_flags & ~(O_WRONLY | O_APPEND |
                                O_TRUNC | O_CREAT | O_NOCTTY | O_EXCL);
                flags |= O_RDONLY;
                f = dentry_open(&file->f_path, flags, file->f_cred);
                if (IS_ERR(f))
                        return PTR_ERR(f);

                new_file_instance = true;
        }

        i_size = i_size_read(file_inode(f));

        if (ima_ahash_minsize && i_size >= ima_ahash_minsize) {
                rc = ima_calc_file_ahash(f, hash);
                if (!rc)
                        goto out;
        }

        rc = ima_calc_file_shash(f, hash);
out:
        if (new_file_instance)
                fput(f);
        return rc;
}

/*
 * Calculate the hash of template data
 */
static int ima_calc_field_array_hash_tfm(struct ima_field_data *field_data,
                                         struct ima_template_entry *entry,
                                         int tfm_idx)
{
        SHASH_DESC_ON_STACK(shash, ima_algo_array[tfm_idx].tfm);
        struct ima_template_desc *td = entry->template_desc;
        int num_fields = entry->template_desc->num_fields;
        int rc, i;

        shash->tfm = ima_algo_array[tfm_idx].tfm;

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        for (i = 0; i < num_fields; i++) {
                u8 buffer[IMA_EVENT_NAME_LEN_MAX + 1] = { 0 };
                u8 *data_to_hash = field_data[i].data;
                u32 datalen = field_data[i].len;
                u32 datalen_to_hash = !ima_canonical_fmt ?
                                datalen : (__force u32)cpu_to_le32(datalen);

                if (strcmp(td->name, IMA_TEMPLATE_IMA_NAME) != 0) {
                        rc = crypto_shash_update(shash,
                                                (const u8 *) &datalen_to_hash,
                                                sizeof(datalen_to_hash));
                        if (rc)
                                break;
                } else if (strcmp(td->fields[i]->field_id, "n") == 0) {
                        memcpy(buffer, data_to_hash, datalen);
                        data_to_hash = buffer;
                        datalen = IMA_EVENT_NAME_LEN_MAX + 1;
                }
                rc = crypto_shash_update(shash, data_to_hash, datalen);
                if (rc)
                        break;
        }

        if (!rc)
                rc = crypto_shash_final(shash, entry->digests[tfm_idx].digest);

        return rc;
}

int ima_calc_field_array_hash(struct ima_field_data *field_data,
                              struct ima_template_entry *entry)
{
        u16 alg_id;
        int rc, i;

        rc = ima_calc_field_array_hash_tfm(field_data, entry, ima_sha1_idx);
        if (rc)
                return rc;

        entry->digests[ima_sha1_idx].alg_id = TPM_ALG_SHA1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (i == ima_sha1_idx)
                        continue;

                if (i < NR_BANKS(ima_tpm_chip)) {
                        alg_id = ima_tpm_chip->allocated_banks[i].alg_id;
                        entry->digests[i].alg_id = alg_id;
                }

                /* for unmapped TPM algorithms digest is still a padded SHA1 */
                if (!ima_algo_array[i].tfm) {
                        memcpy(entry->digests[i].digest,
                               entry->digests[ima_sha1_idx].digest,
                               TPM_DIGEST_SIZE);
                        continue;
                }

                rc = ima_calc_field_array_hash_tfm(field_data, entry, i);
                if (rc)
                        return rc;
        }
        return rc;
}

static int calc_buffer_ahash_atfm(const void *buf, loff_t len,
                                  struct ima_digest_data *hash,
                                  struct crypto_ahash *tfm)
{
        struct ahash_request *req;
        struct scatterlist sg;
        struct crypto_wait wait;
        int rc, ahash_rc = 0;

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out;

        sg_init_one(&sg, buf, len);
        ahash_request_set_crypt(req, &sg, NULL, len);

        ahash_rc = crypto_ahash_update(req);

        /* wait for the update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out:
        ahash_request_free(req);
        return rc;
}

static int calc_buffer_ahash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_ahash_atfm(buf, len, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int calc_buffer_shash_tfm(const void *buf, loff_t size,
                                struct ima_digest_data *hash,
                                struct crypto_shash *tfm)
{
        SHASH_DESC_ON_STACK(shash, tfm);
        unsigned int len;
        int rc;

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        while (size) {
                len = size < PAGE_SIZE ? size : PAGE_SIZE;
                rc = crypto_shash_update(shash, buf, len);
                if (rc)
                        break;
                buf += len;
                size -= len;
        }

        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int calc_buffer_shash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_shash_tfm(buf, len, hash, tfm);

        ima_free_tfm(tfm);
        return rc;
}

int ima_calc_buffer_hash(const void *buf, loff_t len,
                         struct ima_digest_data *hash)
{
        int rc;

        if (ima_ahash_minsize && len >= ima_ahash_minsize) {
                rc = calc_buffer_ahash(buf, len, hash);
                if (!rc)
                        return 0;
        }

        return calc_buffer_shash(buf, len, hash);
}

static void ima_pcrread(u32 idx, struct tpm_digest *d)
{
        if (!ima_tpm_chip)
                return;

        if (tpm_pcr_read(ima_tpm_chip, idx, d) != 0)
                pr_err("Error Communicating to TPM chip\n");
}

/*
 * The boot_aggregate is a cumulative hash over TPM registers 0 - 7.  With
 * TPM 1.2 the boot_aggregate was based on reading the SHA1 PCRs, but with
 * TPM 2.0 hash agility, TPM chips could support multiple TPM PCR banks,
 * allowing firmware to configure and enable different banks.
 *
 * Knowing which TPM bank is read to calculate the boot_aggregate digest
 * needs to be conveyed to a verifier.  For this reason, use the same
 * hash algorithm for reading the TPM PCRs as for calculating the boot
 * aggregate digest as stored in the measurement list.
 */
static int ima_calc_boot_aggregate_tfm(char *digest, u16 alg_id,
                                       struct crypto_shash *tfm)
{
        struct tpm_digest d = { .alg_id = alg_id, .digest = {0} };
        int rc;
        u32 i;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        pr_devel("calculating the boot-aggregate based on TPM bank: %04x\n",
                 d.alg_id);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        /* cumulative digest over TPM registers 0-7 */
        for (i = TPM_PCR0; i < TPM_PCR8; i++) {
                ima_pcrread(i, &d);
                /* now accumulate with current aggregate */
                rc = crypto_shash_update(shash, d.digest,
                                         crypto_shash_digestsize(tfm));
                if (rc != 0)
                        return rc;
        }
        /*
         * Extend cumulative digest over TPM registers 8-9, which contain
         * measurement for the kernel command line (reg. 8) and image (reg. 9)
         * in a typical PCR allocation. Registers 8-9 are only included in
         * non-SHA1 boot_aggregate digests to avoid ambiguity.
         */
        if (alg_id != TPM_ALG_SHA1) {
                for (i = TPM_PCR8; i < TPM_PCR10; i++) {
                        ima_pcrread(i, &d);
                        rc = crypto_shash_update(shash, d.digest,
                                                crypto_shash_digestsize(tfm));
                }
        }
        if (!rc)
                crypto_shash_final(shash, digest);
        return rc;
}

int ima_calc_boot_aggregate(struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        u16 crypto_id, alg_id;
        int rc, i, bank_idx = -1;

        for (i = 0; i < ima_tpm_chip->nr_allocated_banks; i++) {
                crypto_id = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (crypto_id == hash->algo) {
                        bank_idx = i;
                        break;
                }

                if (crypto_id == HASH_ALGO_SHA256)
                        bank_idx = i;

                if (bank_idx == -1 && crypto_id == HASH_ALGO_SHA1)
                        bank_idx = i;
        }

        if (bank_idx == -1) {
                pr_err("No suitable TPM algorithm for boot aggregate\n");
                return 0;
        }

        hash->algo = ima_tpm_chip->allocated_banks[bank_idx].crypto_id;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        hash->length = crypto_shash_digestsize(tfm);
        alg_id = ima_tpm_chip->allocated_banks[bank_idx].alg_id;
        rc = ima_calc_boot_aggregate_tfm(hash->digest, alg_id, tfm);

        ima_free_tfm(tfm);

        return rc;
}
































































































   33 






  443 










  445 






  441 




   51 



   23 
  441 







  344 


    1 
  345 










    7 

    7 




    7 
    7 





    7 



    7 













    7 






    7 


    7 




  450 
  447 






  373 
   75 

   10 




  445 

    5 



  448 






  450 
  447 



  447 

  450 
  451 


    1 

    1 















   32 


















    1 





    2 


















  452 






   35 

   26 


  426 



  450 





  454 










   49 




















   49 


   48 
   49 







   53 
   52 












   11 










    2 
   40 





   39 






   41 
    1 




   41 


   11 




   51 






  416 



  415 
  420 



  415 



  418 






















  192 











  191 
  194 

  193 
   47 




  143 
    8 









    1 

    7 

  147 


  192 




  193 


  193 









  192 











  192 
  193 



    2 







  152 





   26 




   24 
   26 








   26 
   26 









   28 



   26 

    2 














   14 
   14 

















   14 






   14 
   14 
   14 







   12 
    8 




































   13 
   11 



   15 

























   15 







    9 
    9 










   15 








   15 






   15 





















   25 



   25 







   26 





























   12 

   14 




   14 



   14 


   14 




    2 




   14 











   11 




   13 




    2 














   11 



   11 



    5 





    2 




















   13 







   15 






    3 

















   79 



   10 





   69 
    4 













  221 




















  221 
  219 




  220 





    2 







   43 

   76 
    6 
  121 





   20 












   14 
    2 

    5 

    2 
    4 





    3 


  196 


    2 









  215 


    1 

    2 











  217 


  214 




   14 









  212 
   14 





















   12 
  213 

  213 


   20 
  196 


















   36 


  182 













  213 




  210 













  209 
   67 


    3 

















  213 

  190 
   26 




  183 
   36 










   23 
    3 



    2 

    1 
    1 














    2 







   19 
   19 





   18 


   18 


   19 






   18 




    2 






  174 
   48 



    1 

    6 


    6 
    5 
    1 





  209 








  213 

    2 






  173 

   46 

   46 




















  207 
    5 
    1 























   82 



    4 


   12 

   82 






   87 




















  213 










    2 





  212 

  212 
   14 



















    8 
  163 
  204 
   36 


  173 
   46 

    1 


   61 

  148 




  209 






  174 
   46 





  209 


  212 





  211 

  174 






    4 

   43 

    4 









  143 


  207 
  205 
   19 









   63 



    1 



   71 











   16 






   11 
   11 









  140 



    1 















    5 



  139 







    7 

















   14 



   14 





    1 






   11 


    2 




   13 



   11 




   12 



    6 








    7 









    6 



    1 
    6 


    7 
    1 
    6 
    1 
    7 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
// SPDX-License-Identifier: GPL-2.0-only
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                The Internet Protocol (IP) output module.
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                Donald Becker, <becker@super.org>
 *                Alan Cox, <Alan.Cox@linux.org>
 *                Richard Underwood
 *                Stefan Becker, <stefanb@yello.ping.de>
 *                Jorge Cwik, <jorge@laser.satlink.net>
 *                Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 *                Hirokazu Takahashi, <taka@valinux.co.jp>
 *
 *        See ip_input.c for original log
 *
 *        Fixes:
 *                Alan Cox        :        Missing nonblock feature in ip_build_xmit.
 *                Mike Kilburn        :        htons() missing in ip_build_xmit.
 *                Bradford Johnson:        Fix faulty handling of some frames when
 *                                        no route is found.
 *                Alexander Demenshin:        Missing sk/skb free in ip_queue_xmit
 *                                        (in case if packet not accepted by
 *                                        output firewall rules)
 *                Mike McLagan        :        Routing by source
 *                Alexey Kuznetsov:        use new route cache
 *                Andi Kleen:                Fix broken PMTU recovery and remove
 *                                        some redundant tests.
 *        Vitaly E. Lavrov        :        Transparent proxy revived after year coma.
 *                Andi Kleen        :         Replace ip_reply with ip_send_reply.
 *                Andi Kleen        :        Split fast and slow ip_build_xmit path
 *                                        for decreased register pressure on x86
 *                                        and more readability.
 *                Marc Boucher        :        When call_out_firewall returns FW_QUEUE,
 *                                        silently drop skb instead of failing with -EPERM.
 *                Detlev Wengorz        :        Copy protocol for fragments.
 *                Hirokazu Takahashi:        HW checksumming for outgoing UDP
 *                                        datagrams.
 *                Hirokazu Takahashi:        sendfile() on UDP works now.
 */

#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/highmem.h>
#include <linux/slab.h>

#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/proc_fs.h>
#include <linux/stat.h>
#include <linux/init.h>

#include <net/flow.h>
#include <net/snmp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
#include <net/xfrm.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/icmp.h>
#include <net/checksum.h>
#include <net/gso.h>
#include <net/inetpeer.h>
#include <net/lwtunnel.h>
#include <net/inet_dscp.h>
#include <linux/bpf-cgroup.h>
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_bridge.h>
#include <linux/netlink.h>
#include <linux/tcp.h>
#include <net/psp.h>

static int
ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
            unsigned int mtu,
            int (*output)(struct net *, struct sock *, struct sk_buff *));

/* Generate a checksum for an outgoing IP datagram. */
void ip_send_check(struct iphdr *iph)
{
        iph->check = 0;
        iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
}
EXPORT_SYMBOL(ip_send_check);

int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct iphdr *iph = ip_hdr(skb);

        IP_INC_STATS(net, IPSTATS_MIB_OUTREQUESTS);

        iph_set_totlen(iph, skb->len);
        ip_send_check(iph);

        /* if egress device is enslaved to an L3 master device pass the
         * skb to its handler for processing
         */
        skb = l3mdev_ip_out(sk, skb);
        if (unlikely(!skb))
                return 0;

        skb->protocol = htons(ETH_P_IP);

        return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
                       net, sk, skb, NULL, skb_dst_dev(skb),
                       dst_output);
}

int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        int err;

        err = __ip_local_out(net, sk, skb);
        if (likely(err == 1))
                err = dst_output(net, sk, skb);

        return err;
}
EXPORT_SYMBOL_GPL(ip_local_out);

static inline int ip_select_ttl(const struct inet_sock *inet,
                                const struct dst_entry *dst)
{
        int ttl = READ_ONCE(inet->uc_ttl);

        if (ttl < 0)
                ttl = ip4_dst_hoplimit(dst);
        return ttl;
}

/*
 *                Add an ip header to a skbuff and send it out.
 *
 */
int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,
                          __be32 saddr, __be32 daddr, struct ip_options_rcu *opt,
                          u8 tos)
{
        const struct inet_sock *inet = inet_sk(sk);
        struct rtable *rt = skb_rtable(skb);
        struct net *net = sock_net(sk);
        struct iphdr *iph;

        /* Build the IP header. */
        skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : 0));
        skb_reset_network_header(skb);
        iph = ip_hdr(skb);
        iph->version  = 4;
        iph->ihl      = 5;
        iph->tos      = tos;
        iph->ttl      = ip_select_ttl(inet, &rt->dst);
        iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);
        iph->saddr    = saddr;
        iph->protocol = sk->sk_protocol;
        /* Do not bother generating IPID for small packets (eg SYNACK) */
        if (skb->len <= IPV4_MIN_MTU || ip_dont_fragment(sk, &rt->dst)) {
                iph->frag_off = htons(IP_DF);
                iph->id = 0;
        } else {
                iph->frag_off = 0;
                /* TCP packets here are SYNACK with fat IPv4/TCP options.
                 * Avoid using the hashed IP ident generator.
                 */
                if (sk->sk_protocol == IPPROTO_TCP)
                        iph->id = (__force __be16)get_random_u16();
                else
                        __ip_select_ident(net, iph, 1);
        }

        if (opt && opt->opt.optlen) {
                iph->ihl += opt->opt.optlen>>2;
                ip_options_build(skb, &opt->opt, daddr, rt);
        }

        skb->priority = READ_ONCE(sk->sk_priority);
        if (!skb->mark)
                skb->mark = READ_ONCE(sk->sk_mark);

        /* Send it out. */
        return ip_local_out(net, skb->sk, skb);
}
EXPORT_SYMBOL_GPL(ip_build_and_send_pkt);

static int ip_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);
        struct rtable *rt = dst_rtable(dst);
        struct net_device *dev = dst_dev(dst);
        unsigned int hh_len = LL_RESERVED_SPACE(dev);
        struct neighbour *neigh;
        bool is_v6gw = false;

        if (rt->rt_type == RTN_MULTICAST) {
                IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTMCAST, skb->len);
        } else if (rt->rt_type == RTN_BROADCAST)
                IP_UPD_PO_STATS(net, IPSTATS_MIB_OUTBCAST, skb->len);

        /* OUTOCTETS should be counted after fragment */
        IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);

        if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
                skb = skb_expand_head(skb, hh_len);
                if (!skb)
                        return -ENOMEM;
        }

        if (lwtunnel_xmit_redirect(dst->lwtstate)) {
                int res = lwtunnel_xmit(skb);

                if (res != LWTUNNEL_XMIT_CONTINUE)
                        return res;
        }

        rcu_read_lock();
        neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
        if (!IS_ERR(neigh)) {
                int res;

                sock_confirm_neigh(skb, neigh);
                /* if crossing protocols, can not use the cached header */
                res = neigh_output(neigh, skb, is_v6gw);
                rcu_read_unlock();
                return res;
        }
        rcu_read_unlock();

        net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
                            __func__);
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL);
        return PTR_ERR(neigh);
}

static int ip_finish_output_gso(struct net *net, struct sock *sk,
                                struct sk_buff *skb, unsigned int mtu)
{
        struct sk_buff *segs, *nskb;
        netdev_features_t features;
        int ret = 0;

        /* common case: seglen is <= mtu
         */
        if (skb_gso_validate_network_len(skb, mtu))
                return ip_finish_output2(net, sk, skb);

        /* Slowpath -  GSO segment length exceeds the egress MTU.
         *
         * This can happen in several cases:
         *  - Forwarding of a TCP GRO skb, when DF flag is not set.
         *  - Forwarding of an skb that arrived on a virtualization interface
         *    (virtio-net/vhost/tap) with TSO/GSO size set by other network
         *    stack.
         *  - Local GSO skb transmitted on an NETIF_F_TSO tunnel stacked over an
         *    interface with a smaller MTU.
         *  - Arriving GRO skb (or GSO skb in a virtualized environment) that is
         *    bridged to a NETIF_F_TSO tunnel stacked over an interface with an
         *    insufficient MTU.
         */
        features = netif_skb_features(skb);
        BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET);
        segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
        if (IS_ERR_OR_NULL(segs)) {
                kfree_skb(skb);
                return -ENOMEM;
        }

        consume_skb(skb);

        skb_list_walk_safe(segs, segs, nskb) {
                int err;

                skb_mark_not_on_list(segs);
                err = ip_fragment(net, sk, segs, mtu, ip_finish_output2);

                if (err && ret == 0)
                        ret = err;
        }

        return ret;
}

static int __ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        unsigned int mtu;

#if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
        /* Policy lookup after SNAT yielded a new policy */
        if (skb_dst(skb)->xfrm) {
                IPCB(skb)->flags |= IPSKB_REROUTED;
                return dst_output(net, sk, skb);
        }
#endif
        mtu = ip_skb_dst_mtu(sk, skb);
        if (skb_is_gso(skb))
                return ip_finish_output_gso(net, sk, skb, mtu);

        if (skb->len > mtu || IPCB(skb)->frag_max_size)
                return ip_fragment(net, sk, skb, mtu, ip_finish_output2);

        return ip_finish_output2(net, sk, skb);
}

static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        int ret;

        ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
        switch (ret) {
        case NET_XMIT_SUCCESS:
                return __ip_finish_output(net, sk, skb);
        case NET_XMIT_CN:
                return __ip_finish_output(net, sk, skb) ? : ret;
        default:
                kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS);
                return ret;
        }
}

static int ip_mc_finish_output(struct net *net, struct sock *sk,
                               struct sk_buff *skb)
{
        struct rtable *new_rt;
        bool do_cn = false;
        int ret, err;

        ret = BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb);
        switch (ret) {
        case NET_XMIT_CN:
                do_cn = true;
                fallthrough;
        case NET_XMIT_SUCCESS:
                break;
        default:
                kfree_skb_reason(skb, SKB_DROP_REASON_BPF_CGROUP_EGRESS);
                return ret;
        }

        /* Reset rt_iif so that inet_iif() will return skb->skb_iif. Setting
         * this to non-zero causes ipi_ifindex in in_pktinfo to be overwritten,
         * see ipv4_pktinfo_prepare().
         */
        new_rt = rt_dst_clone(net->loopback_dev, skb_rtable(skb));
        if (new_rt) {
                new_rt->rt_iif = 0;
                skb_dst_drop(skb);
                skb_dst_set(skb, &new_rt->dst);
        }

        err = dev_loopback_xmit(net, sk, skb);
        return (do_cn && err) ? ret : err;
}

int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct rtable *rt = skb_rtable(skb);
        struct net_device *dev = rt->dst.dev;

        /*
         *        If the indicated interface is up and running, send the packet.
         */
        skb->dev = dev;
        skb->protocol = htons(ETH_P_IP);

        /*
         *        Multicasts are looped back for other local users
         */

        if (rt->rt_flags&RTCF_MULTICAST) {
                if (sk_mc_loop(sk)
#ifdef CONFIG_IP_MROUTE
                /* Small optimization: do not loopback not local frames,
                   which returned after forwarding; they will be  dropped
                   by ip_mr_input in any case.
                   Note, that local frames are looped back to be delivered
                   to local recipients.

                   This check is duplicated in ip_mr_input at the moment.
                 */
                    &&
                    ((rt->rt_flags & RTCF_LOCAL) ||
                     !(IPCB(skb)->flags & IPSKB_FORWARDED))
#endif
                   ) {
                        struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
                        if (newskb)
                                NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                                        net, sk, newskb, NULL, newskb->dev,
                                        ip_mc_finish_output);
                }

                /* Multicasts with ttl 0 must not go beyond the host */

                if (ip_hdr(skb)->ttl == 0) {
                        kfree_skb(skb);
                        return 0;
                }
        }

        if (rt->rt_flags&RTCF_BROADCAST) {
                struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
                if (newskb)
                        NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                                net, sk, newskb, NULL, newskb->dev,
                                ip_mc_finish_output);
        }

        return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                            net, sk, skb, NULL, skb->dev,
                            ip_finish_output,
                            !(IPCB(skb)->flags & IPSKB_REROUTED));
}

int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        struct net_device *dev, *indev = skb->dev;
        int ret_val;

        rcu_read_lock();
        dev = skb_dst_dev_rcu(skb);
        skb->dev = dev;
        skb->protocol = htons(ETH_P_IP);

        ret_val = NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                                net, sk, skb, indev, dev,
                                ip_finish_output,
                                !(IPCB(skb)->flags & IPSKB_REROUTED));
        rcu_read_unlock();
        return ret_val;
}
EXPORT_SYMBOL(ip_output);

/*
 * copy saddr and daddr, possibly using 64bit load/stores
 * Equivalent to :
 *   iph->saddr = fl4->saddr;
 *   iph->daddr = fl4->daddr;
 */
static void ip_copy_addrs(struct iphdr *iph, const struct flowi4 *fl4)
{
        BUILD_BUG_ON(offsetof(typeof(*fl4), daddr) !=
                     offsetof(typeof(*fl4), saddr) + sizeof(fl4->saddr));

        iph->saddr = fl4->saddr;
        iph->daddr = fl4->daddr;
}

/* Note: skb->sk can be different from sk, in case of tunnels */
int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
                    __u8 tos)
{
        struct inet_sock *inet = inet_sk(sk);
        struct net *net = sock_net(sk);
        struct ip_options_rcu *inet_opt;
        struct flowi4 *fl4;
        struct rtable *rt;
        struct iphdr *iph;
        int res;

        /* Skip all of this if the packet is already routed,
         * f.e. by something like SCTP.
         */
        rcu_read_lock();
        inet_opt = rcu_dereference(inet->inet_opt);
        fl4 = &fl->u.ip4;
        rt = skb_rtable(skb);
        if (rt)
                goto packet_routed;

        /* Make sure we can route this packet. */
        rt = dst_rtable(__sk_dst_check(sk, 0));
        if (!rt) {
                inet_sk_init_flowi4(inet, fl4);

                /* sctp_v4_xmit() uses its own DSCP value */
                fl4->flowi4_dscp = inet_dsfield_to_dscp(tos);

                /* If this fails, retransmit mechanism of transport layer will
                 * keep trying until route appears or the connection times
                 * itself out.
                 */
                rt = ip_route_output_flow(net, fl4, sk);
                if (IS_ERR(rt))
                        goto no_route;
                sk_setup_caps(sk, &rt->dst);
        }
        skb_dst_set_noref(skb, &rt->dst);

packet_routed:
        if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)
                goto no_route;

        /* OK, we know where to send it, allocate and build IP header. */
        skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : 0));
        skb_reset_network_header(skb);
        iph = ip_hdr(skb);
        *((__be16 *)iph) = htons((4 << 12) | (5 << 8) | (tos & 0xff));
        if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)
                iph->frag_off = htons(IP_DF);
        else
                iph->frag_off = 0;
        iph->ttl      = ip_select_ttl(inet, &rt->dst);
        iph->protocol = sk->sk_protocol;
        ip_copy_addrs(iph, fl4);

        /* Transport layer set skb->h.foo itself. */

        if (inet_opt && inet_opt->opt.optlen) {
                iph->ihl += inet_opt->opt.optlen >> 2;
                ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt);
        }

        ip_select_ident_segs(net, skb, sk,
                             skb_shinfo(skb)->gso_segs ?: 1);

        /* TODO : should we use skb->sk here instead of sk ? */
        skb->priority = READ_ONCE(sk->sk_priority);
        skb->mark = READ_ONCE(sk->sk_mark);

        res = ip_local_out(net, sk, skb);
        rcu_read_unlock();
        return res;

no_route:
        rcu_read_unlock();
        IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
        kfree_skb_reason(skb, SKB_DROP_REASON_IP_OUTNOROUTES);
        return -EHOSTUNREACH;
}
EXPORT_SYMBOL(__ip_queue_xmit);

int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
{
        return __ip_queue_xmit(sk, skb, fl, READ_ONCE(inet_sk(sk)->tos));
}
EXPORT_SYMBOL(ip_queue_xmit);

static void ip_copy_metadata(struct sk_buff *to, struct sk_buff *from)
{
        to->pkt_type = from->pkt_type;
        to->priority = from->priority;
        to->protocol = from->protocol;
        to->skb_iif = from->skb_iif;
        skb_dst_drop(to);
        skb_dst_copy(to, from);
        to->dev = from->dev;
        to->mark = from->mark;

        skb_copy_hash(to, from);

#ifdef CONFIG_NET_SCHED
        to->tc_index = from->tc_index;
#endif
        nf_copy(to, from);
        skb_ext_copy(to, from);
#if IS_ENABLED(CONFIG_IP_VS)
        to->ipvs_property = from->ipvs_property;
#endif
        skb_copy_secmark(to, from);
}

static int ip_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
                       unsigned int mtu,
                       int (*output)(struct net *, struct sock *, struct sk_buff *))
{
        struct iphdr *iph = ip_hdr(skb);

        if ((iph->frag_off & htons(IP_DF)) == 0)
                return ip_do_fragment(net, sk, skb, output);

        if (unlikely(!skb->ignore_df ||
                     (IPCB(skb)->frag_max_size &&
                      IPCB(skb)->frag_max_size > mtu))) {
                IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
                icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
                          htonl(mtu));
                kfree_skb(skb);
                return -EMSGSIZE;
        }

        return ip_do_fragment(net, sk, skb, output);
}

void ip_fraglist_init(struct sk_buff *skb, struct iphdr *iph,
                      unsigned int hlen, struct ip_fraglist_iter *iter)
{
        unsigned int first_len = skb_pagelen(skb);

        iter->frag = skb_shinfo(skb)->frag_list;
        skb_frag_list_init(skb);

        iter->offset = 0;
        iter->iph = iph;
        iter->hlen = hlen;

        skb->data_len = first_len - skb_headlen(skb);
        skb->len = first_len;
        iph->tot_len = htons(first_len);
        iph->frag_off = htons(IP_MF);
        ip_send_check(iph);
}
EXPORT_SYMBOL(ip_fraglist_init);

void ip_fraglist_prepare(struct sk_buff *skb, struct ip_fraglist_iter *iter)
{
        unsigned int hlen = iter->hlen;
        struct iphdr *iph = iter->iph;
        struct sk_buff *frag;

        frag = iter->frag;
        frag->ip_summed = CHECKSUM_NONE;
        skb_reset_transport_header(frag);
        __skb_push(frag, hlen);
        skb_reset_network_header(frag);
        memcpy(skb_network_header(frag), iph, hlen);
        iter->iph = ip_hdr(frag);
        iph = iter->iph;
        iph->tot_len = htons(frag->len);
        ip_copy_metadata(frag, skb);
        iter->offset += skb->len - hlen;
        iph->frag_off = htons(iter->offset >> 3);
        if (frag->next)
                iph->frag_off |= htons(IP_MF);
        /* Ready, complete checksum */
        ip_send_check(iph);
}
EXPORT_SYMBOL(ip_fraglist_prepare);

void ip_frag_init(struct sk_buff *skb, unsigned int hlen,
                  unsigned int ll_rs, unsigned int mtu, bool DF,
                  struct ip_frag_state *state)
{
        struct iphdr *iph = ip_hdr(skb);

        state->DF = DF;
        state->hlen = hlen;
        state->ll_rs = ll_rs;
        state->mtu = mtu;

        state->left = skb->len - hlen;        /* Space per frame */
        state->ptr = hlen;                /* Where to start from */

        state->offset = (ntohs(iph->frag_off) & IP_OFFSET) << 3;
        state->not_last_frag = iph->frag_off & htons(IP_MF);
}
EXPORT_SYMBOL(ip_frag_init);

static void ip_frag_ipcb(struct sk_buff *from, struct sk_buff *to,
                         bool first_frag)
{
        /* Copy the flags to each fragment. */
        IPCB(to)->flags = IPCB(from)->flags;

        /* ANK: dirty, but effective trick. Upgrade options only if
         * the segment to be fragmented was THE FIRST (otherwise,
         * options are already fixed) and make it ONCE
         * on the initial skb, so that all the following fragments
         * will inherit fixed options.
         */
        if (first_frag)
                ip_options_fragment(from);
}

struct sk_buff *ip_frag_next(struct sk_buff *skb, struct ip_frag_state *state)
{
        unsigned int len = state->left;
        struct sk_buff *skb2;
        struct iphdr *iph;

        /* IF: it doesn't fit, use 'mtu' - the data space left */
        if (len > state->mtu)
                len = state->mtu;
        /* IF: we are not sending up to and including the packet end
           then align the next start on an eight byte boundary */
        if (len < state->left)        {
                len &= ~7;
        }

        /* Allocate buffer */
        skb2 = alloc_skb(len + state->hlen + state->ll_rs, GFP_ATOMIC);
        if (!skb2)
                return ERR_PTR(-ENOMEM);

        /*
         *        Set up data on packet
         */

        ip_copy_metadata(skb2, skb);
        skb_reserve(skb2, state->ll_rs);
        skb_put(skb2, len + state->hlen);
        skb_reset_network_header(skb2);
        skb2->transport_header = skb2->network_header + state->hlen;

        /*
         *        Charge the memory for the fragment to any owner
         *        it might possess
         */

        if (skb->sk)
                skb_set_owner_w(skb2, skb->sk);

        /*
         *        Copy the packet header into the new buffer.
         */

        skb_copy_from_linear_data(skb, skb_network_header(skb2), state->hlen);

        /*
         *        Copy a block of the IP datagram.
         */
        if (skb_copy_bits(skb, state->ptr, skb_transport_header(skb2), len))
                BUG();
        state->left -= len;

        /*
         *        Fill in the new header fields.
         */
        iph = ip_hdr(skb2);
        iph->frag_off = htons((state->offset >> 3));
        if (state->DF)
                iph->frag_off |= htons(IP_DF);

        /*
         *        Added AC : If we are fragmenting a fragment that's not the
         *                   last fragment then keep MF on each bit
         */
        if (state->left > 0 || state->not_last_frag)
                iph->frag_off |= htons(IP_MF);
        state->ptr += len;
        state->offset += len;

        iph->tot_len = htons(len + state->hlen);

        ip_send_check(iph);

        return skb2;
}
EXPORT_SYMBOL(ip_frag_next);

/*
 *        This IP datagram is too large to be sent in one piece.  Break it up into
 *        smaller pieces (each of size equal to IP header plus
 *        a block of the data of the original IP data part) that will yet fit in a
 *        single device frame, and queue such a frame for sending.
 */

int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
                   int (*output)(struct net *, struct sock *, struct sk_buff *))
{
        struct iphdr *iph;
        struct sk_buff *skb2;
        u8 tstamp_type = skb->tstamp_type;
        struct rtable *rt = skb_rtable(skb);
        unsigned int mtu, hlen, ll_rs;
        struct ip_fraglist_iter iter;
        ktime_t tstamp = skb->tstamp;
        struct ip_frag_state state;
        int err = 0;

        /* for offloaded checksums cleanup checksum before fragmentation */
        if (skb->ip_summed == CHECKSUM_PARTIAL &&
            (err = skb_checksum_help(skb)))
                goto fail;

        /*
         *        Point into the IP datagram header.
         */

        iph = ip_hdr(skb);

        mtu = ip_skb_dst_mtu(sk, skb);
        if (IPCB(skb)->frag_max_size && IPCB(skb)->frag_max_size < mtu)
                mtu = IPCB(skb)->frag_max_size;

        /*
         *        Setup starting values.
         */

        hlen = iph->ihl * 4;
        mtu = mtu - hlen;        /* Size of data space */
        IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE;
        ll_rs = LL_RESERVED_SPACE(rt->dst.dev);

        /* When frag_list is given, use it. First, check its validity:
         * some transformers could create wrong frag_list or break existing
         * one, it is not prohibited. In this case fall back to copying.
         *
         * LATER: this step can be merged to real generation of fragments,
         * we can switch to copy when see the first bad fragment.
         */
        if (skb_has_frag_list(skb)) {
                struct sk_buff *frag, *frag2;
                unsigned int first_len = skb_pagelen(skb);

                if (first_len - hlen > mtu ||
                    ((first_len - hlen) & 7) ||
                    ip_is_fragment(iph) ||
                    skb_cloned(skb) ||
                    skb_headroom(skb) < ll_rs)
                        goto slow_path;

                skb_walk_frags(skb, frag) {
                        /* Correct geometry. */
                        if (frag->len > mtu ||
                            ((frag->len & 7) && frag->next) ||
                            skb_headroom(frag) < hlen + ll_rs)
                                goto slow_path_clean;

                        /* Partially cloned skb? */
                        if (skb_shared(frag))
                                goto slow_path_clean;

                        BUG_ON(frag->sk);
                        if (skb->sk) {
                                frag->sk = skb->sk;
                                frag->destructor = sock_wfree;
                        }
                        skb->truesize -= frag->truesize;
                }

                /* Everything is OK. Generate! */
                ip_fraglist_init(skb, iph, hlen, &iter);

                for (;;) {
                        /* Prepare header of the next frame,
                         * before previous one went down. */
                        if (iter.frag) {
                                bool first_frag = (iter.offset == 0);

                                IPCB(iter.frag)->flags = IPCB(skb)->flags;
                                ip_fraglist_prepare(skb, &iter);
                                if (first_frag && IPCB(skb)->opt.optlen) {
                                        /* ipcb->opt is not populated for frags
                                         * coming from __ip_make_skb(),
                                         * ip_options_fragment() needs optlen
                                         */
                                        IPCB(iter.frag)->opt.optlen =
                                                IPCB(skb)->opt.optlen;
                                        ip_options_fragment(iter.frag);
                                        ip_send_check(iter.iph);
                                }
                        }

                        skb_set_delivery_time(skb, tstamp, tstamp_type);
                        err = output(net, sk, skb);

                        if (!err)
                                IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
                        if (err || !iter.frag)
                                break;

                        skb = ip_fraglist_next(&iter);
                }

                if (err == 0) {
                        IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
                        return 0;
                }

                kfree_skb_list(iter.frag);

                IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
                return err;

slow_path_clean:
                skb_walk_frags(skb, frag2) {
                        if (frag2 == frag)
                                break;
                        frag2->sk = NULL;
                        frag2->destructor = NULL;
                        skb->truesize += frag2->truesize;
                }
        }

slow_path:
        /*
         *        Fragment the datagram.
         */

        ip_frag_init(skb, hlen, ll_rs, mtu, IPCB(skb)->flags & IPSKB_FRAG_PMTU,
                     &state);

        /*
         *        Keep copying data until we run out.
         */

        while (state.left > 0) {
                bool first_frag = (state.offset == 0);

                skb2 = ip_frag_next(skb, &state);
                if (IS_ERR(skb2)) {
                        err = PTR_ERR(skb2);
                        goto fail;
                }
                ip_frag_ipcb(skb, skb2, first_frag);

                /*
                 *        Put this fragment into the sending queue.
                 */
                skb_set_delivery_time(skb2, tstamp, tstamp_type);
                err = output(net, sk, skb2);
                if (err)
                        goto fail;

                IP_INC_STATS(net, IPSTATS_MIB_FRAGCREATES);
        }
        consume_skb(skb);
        IP_INC_STATS(net, IPSTATS_MIB_FRAGOKS);
        return err;

fail:
        kfree_skb(skb);
        IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
        return err;
}
EXPORT_SYMBOL(ip_do_fragment);

int
ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb)
{
        struct msghdr *msg = from;

        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                if (!copy_from_iter_full(to, len, &msg->msg_iter))
                        return -EFAULT;
        } else {
                __wsum csum = 0;
                if (!csum_and_copy_from_iter_full(to, len, &csum, &msg->msg_iter))
                        return -EFAULT;
                skb->csum = csum_block_add(skb->csum, csum, odd);
        }
        return 0;
}
EXPORT_SYMBOL(ip_generic_getfrag);

static int __ip_append_data(struct sock *sk,
                            struct flowi4 *fl4,
                            struct sk_buff_head *queue,
                            struct inet_cork *cork,
                            struct page_frag *pfrag,
                            int getfrag(void *from, char *to, int offset,
                                        int len, int odd, struct sk_buff *skb),
                            void *from, int length, int transhdrlen,
                            unsigned int flags)
{
        struct inet_sock *inet = inet_sk(sk);
        struct ubuf_info *uarg = NULL;
        struct sk_buff *skb;
        struct ip_options *opt = cork->opt;
        int hh_len;
        int exthdrlen;
        int mtu;
        int copy;
        int err;
        int offset = 0;
        bool zc = false;
        unsigned int maxfraglen, fragheaderlen, maxnonfragsize;
        int csummode = CHECKSUM_NONE;
        struct rtable *rt = dst_rtable(cork->dst);
        bool paged, hold_tskey = false, extra_uref = false;
        unsigned int wmem_alloc_delta = 0;
        u32 tskey = 0;

        skb = skb_peek_tail(queue);

        exthdrlen = !skb ? rt->dst.header_len : 0;
        mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize;
        paged = !!cork->gso_size;

        hh_len = LL_RESERVED_SPACE(rt->dst.dev);

        fragheaderlen = sizeof(struct iphdr) + (opt ? opt->optlen : 0);
        maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen;
        maxnonfragsize = ip_sk_ignore_df(sk) ? IP_MAX_MTU : mtu;

        if (cork->length + length > maxnonfragsize - fragheaderlen) {
                ip_local_error(sk, EMSGSIZE, fl4->daddr, inet->inet_dport,
                               mtu - (opt ? opt->optlen : 0));
                return -EMSGSIZE;
        }

        /*
         * transhdrlen > 0 means that this is the first fragment and we wish
         * it won't be fragmented in the future.
         */
        if (transhdrlen &&
            length + fragheaderlen <= mtu &&
            rt->dst.dev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM) &&
            (!(flags & MSG_MORE) || cork->gso_size) &&
            (!exthdrlen || (rt->dst.dev->features & NETIF_F_HW_ESP_TX_CSUM)))
                csummode = CHECKSUM_PARTIAL;

        if ((flags & MSG_ZEROCOPY) && length) {
                struct msghdr *msg = from;

                if (getfrag == ip_generic_getfrag && msg->msg_ubuf) {
                        if (skb_zcopy(skb) && msg->msg_ubuf != skb_zcopy(skb))
                                return -EINVAL;

                        /* Leave uarg NULL if can't zerocopy, callers should
                         * be able to handle it.
                         */
                        if ((rt->dst.dev->features & NETIF_F_SG) &&
                            csummode == CHECKSUM_PARTIAL) {
                                paged = true;
                                zc = true;
                                uarg = msg->msg_ubuf;
                        }
                } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
                        uarg = msg_zerocopy_realloc(sk, length, skb_zcopy(skb),
                                                    false);
                        if (!uarg)
                                return -ENOBUFS;
                        extra_uref = !skb_zcopy(skb);        /* only ref on new uarg */
                        if (rt->dst.dev->features & NETIF_F_SG &&
                            csummode == CHECKSUM_PARTIAL) {
                                paged = true;
                                zc = true;
                        } else {
                                uarg_to_msgzc(uarg)->zerocopy = 0;
                                skb_zcopy_set(skb, uarg, &extra_uref);
                        }
                }
        } else if ((flags & MSG_SPLICE_PAGES) && length) {
                if (inet_test_bit(HDRINCL, sk))
                        return -EPERM;
                if (rt->dst.dev->features & NETIF_F_SG &&
                    getfrag == ip_generic_getfrag)
                        /* We need an empty buffer to attach stuff to */
                        paged = true;
                else
                        flags &= ~MSG_SPLICE_PAGES;
        }

        cork->length += length;

        if (cork->tx_flags & SKBTX_ANY_TSTAMP &&
            READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_OPT_ID) {
                if (cork->flags & IPCORK_TS_OPT_ID) {
                        tskey = cork->ts_opt_id;
                } else {
                        tskey = atomic_inc_return(&sk->sk_tskey) - 1;
                        hold_tskey = true;
                }
        }

        /* So, what's going on in the loop below?
         *
         * We use calculated fragment length to generate chained skb,
         * each of segments is IP fragment ready for sending to network after
         * adding appropriate IP header.
         */

        if (!skb)
                goto alloc_new_skb;

        while (length > 0) {
                /* Check if the remaining data fits into current packet. */
                copy = mtu - skb->len;
                if (copy < length)
                        copy = maxfraglen - skb->len;
                if (copy <= 0) {
                        char *data;
                        unsigned int datalen;
                        unsigned int fraglen;
                        unsigned int fraggap;
                        unsigned int alloclen, alloc_extra;
                        unsigned int pagedlen;
                        struct sk_buff *skb_prev;
alloc_new_skb:
                        skb_prev = skb;
                        if (skb_prev)
                                fraggap = skb_prev->len - maxfraglen;
                        else
                                fraggap = 0;

                        /*
                         * If remaining data exceeds the mtu,
                         * we know we need more fragment(s).
                         */
                        datalen = length + fraggap;
                        if (datalen > mtu - fragheaderlen)
                                datalen = maxfraglen - fragheaderlen;
                        fraglen = datalen + fragheaderlen;
                        pagedlen = 0;

                        alloc_extra = hh_len + 15;
                        alloc_extra += exthdrlen;

                        /* The last fragment gets additional space at tail.
                         * Note, with MSG_MORE we overallocate on fragments,
                         * because we have no idea what fragment will be
                         * the last.
                         */
                        if (datalen == length + fraggap)
                                alloc_extra += rt->dst.trailer_len;

                        if ((flags & MSG_MORE) &&
                            !(rt->dst.dev->features&NETIF_F_SG))
                                alloclen = mtu;
                        else if (!paged &&
                                 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
                                  !(rt->dst.dev->features & NETIF_F_SG)))
                                alloclen = fraglen;
                        else {
                                alloclen = fragheaderlen + transhdrlen;
                                pagedlen = datalen - transhdrlen;
                        }

                        alloclen += alloc_extra;

                        if (transhdrlen) {
                                skb = sock_alloc_send_skb(sk, alloclen,
                                                (flags & MSG_DONTWAIT), &err);
                        } else {
                                skb = NULL;
                                if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
                                    2 * sk->sk_sndbuf)
                                        skb = alloc_skb(alloclen,
                                                        sk->sk_allocation);
                                if (unlikely(!skb))
                                        err = -ENOBUFS;
                        }
                        if (!skb)
                                goto error;

                        /*
                         *        Fill in the control structures
                         */
                        skb->ip_summed = csummode;
                        skb->csum = 0;
                        skb_reserve(skb, hh_len);

                        /*
                         *        Find where to start putting bytes.
                         */
                        data = skb_put(skb, fraglen + exthdrlen - pagedlen);
                        skb_set_network_header(skb, exthdrlen);
                        skb->transport_header = (skb->network_header +
                                                 fragheaderlen);
                        data += fragheaderlen + exthdrlen;

                        if (fraggap) {
                                skb->csum = skb_copy_and_csum_bits(
                                        skb_prev, maxfraglen,
                                        data + transhdrlen, fraggap);
                                skb_prev->csum = csum_sub(skb_prev->csum,
                                                          skb->csum);
                                data += fraggap;
                                pskb_trim_unique(skb_prev, maxfraglen);
                        }

                        copy = datalen - transhdrlen - fraggap - pagedlen;
                        /* [!] NOTE: copy will be negative if pagedlen>0
                         * because then the equation reduces to -fraggap.
                         */
                        if (copy > 0 &&
                            INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
                                            from, data + transhdrlen, offset,
                                            copy, fraggap, skb) < 0) {
                                err = -EFAULT;
                                kfree_skb(skb);
                                goto error;
                        } else if (flags & MSG_SPLICE_PAGES) {
                                copy = 0;
                        }

                        offset += copy;
                        length -= copy + transhdrlen;
                        transhdrlen = 0;
                        exthdrlen = 0;
                        csummode = CHECKSUM_NONE;

                        /* only the initial fragment is time stamped */
                        skb_shinfo(skb)->tx_flags = cork->tx_flags;
                        cork->tx_flags = 0;
                        skb_shinfo(skb)->tskey = tskey;
                        tskey = 0;
                        skb_zcopy_set(skb, uarg, &extra_uref);

                        if ((flags & MSG_CONFIRM) && !skb_prev)
                                skb_set_dst_pending_confirm(skb, 1);

                        /*
                         * Put the packet on the pending queue.
                         */
                        if (!skb->destructor) {
                                skb->destructor = sock_wfree;
                                skb->sk = sk;
                                wmem_alloc_delta += skb->truesize;
                        }
                        __skb_queue_tail(queue, skb);
                        continue;
                }

                if (copy > length)
                        copy = length;

                if (!(rt->dst.dev->features&NETIF_F_SG) &&
                    skb_tailroom(skb) >= copy) {
                        unsigned int off;

                        off = skb->len;
                        if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
                                            from, skb_put(skb, copy),
                                            offset, copy, off, skb) < 0) {
                                __skb_trim(skb, off);
                                err = -EFAULT;
                                goto error;
                        }
                } else if (flags & MSG_SPLICE_PAGES) {
                        struct msghdr *msg = from;

                        err = -EIO;
                        if (WARN_ON_ONCE(copy > msg->msg_iter.count))
                                goto error;

                        err = skb_splice_from_iter(skb, &msg->msg_iter, copy);
                        if (err < 0)
                                goto error;
                        copy = err;
                        wmem_alloc_delta += copy;
                } else if (!zc) {
                        int i = skb_shinfo(skb)->nr_frags;

                        err = -ENOMEM;
                        if (!sk_page_frag_refill(sk, pfrag))
                                goto error;

                        skb_zcopy_downgrade_managed(skb);
                        if (!skb_can_coalesce(skb, i, pfrag->page,
                                              pfrag->offset)) {
                                err = -EMSGSIZE;
                                if (i == MAX_SKB_FRAGS)
                                        goto error;

                                __skb_fill_page_desc(skb, i, pfrag->page,
                                                     pfrag->offset, 0);
                                skb_shinfo(skb)->nr_frags = ++i;
                                get_page(pfrag->page);
                        }
                        copy = min_t(int, copy, pfrag->size - pfrag->offset);
                        if (INDIRECT_CALL_1(getfrag, ip_generic_getfrag,
                                    from,
                                    page_address(pfrag->page) + pfrag->offset,
                                    offset, copy, skb->len, skb) < 0)
                                goto error_efault;

                        pfrag->offset += copy;
                        skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
                        skb_len_add(skb, copy);
                        wmem_alloc_delta += copy;
                } else {
                        err = skb_zerocopy_iter_dgram(skb, from, copy);
                        if (err < 0)
                                goto error;
                }
                offset += copy;
                length -= copy;
        }

        if (wmem_alloc_delta)
                refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
        return 0;

error_efault:
        err = -EFAULT;
error:
        net_zcopy_put_abort(uarg, extra_uref);
        cork->length -= length;
        IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTDISCARDS);
        refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
        if (hold_tskey)
                atomic_dec(&sk->sk_tskey);
        return err;
}

static int ip_setup_cork(struct sock *sk, struct inet_cork *cork,
                         struct ipcm_cookie *ipc, struct rtable **rtp)
{
        struct ip_options_rcu *opt;
        struct rtable *rt;

        rt = *rtp;
        if (unlikely(!rt))
                return -EFAULT;

        cork->fragsize = ip_sk_use_pmtu(sk) ?
                         dst_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu);

        if (!inetdev_valid_mtu(cork->fragsize))
                return -ENETUNREACH;

        /*
         * setup for corking.
         */
        opt = ipc->opt;
        if (opt) {
                if (!cork->opt) {
                        cork->opt = kmalloc(sizeof(struct ip_options) + 40,
                                            sk->sk_allocation);
                        if (unlikely(!cork->opt))
                                return -ENOBUFS;
                }
                memcpy(cork->opt, &opt->opt, sizeof(struct ip_options) + opt->opt.optlen);
                cork->flags |= IPCORK_OPT;
                cork->addr = ipc->addr;
        }

        cork->gso_size = ipc->gso_size;

        cork->dst = &rt->dst;
        /* We stole this route, caller should not release it. */
        *rtp = NULL;

        cork->length = 0;
        cork->ttl = ipc->ttl;
        cork->tos = ipc->tos;
        cork->mark = ipc->sockc.mark;
        cork->priority = ipc->sockc.priority;
        cork->transmit_time = ipc->sockc.transmit_time;
        cork->tx_flags = 0;
        sock_tx_timestamp(sk, &ipc->sockc, &cork->tx_flags);
        if (ipc->sockc.tsflags & SOCKCM_FLAG_TS_OPT_ID) {
                cork->flags |= IPCORK_TS_OPT_ID;
                cork->ts_opt_id = ipc->sockc.ts_opt_id;
        }

        return 0;
}

/*
 *        ip_append_data() can make one large IP datagram from many pieces of
 *        data.  Each piece will be held on the socket until
 *        ip_push_pending_frames() is called. Each piece can be a page or
 *        non-page data.
 *
 *        Not only UDP, other transport protocols - e.g. raw sockets - can use
 *        this interface potentially.
 *
 *        LATER: length must be adjusted by pad at tail, when it is required.
 */
int ip_append_data(struct sock *sk, struct flowi4 *fl4,
                   int getfrag(void *from, char *to, int offset, int len,
                               int odd, struct sk_buff *skb),
                   void *from, int length, int transhdrlen,
                   struct ipcm_cookie *ipc, struct rtable **rtp,
                   unsigned int flags)
{
        struct inet_sock *inet = inet_sk(sk);
        int err;

        if (flags&MSG_PROBE)
                return 0;

        if (skb_queue_empty(&sk->sk_write_queue)) {
                err = ip_setup_cork(sk, &inet->cork.base, ipc, rtp);
                if (err)
                        return err;
        } else {
                transhdrlen = 0;
        }

        return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base,
                                sk_page_frag(sk), getfrag,
                                from, length, transhdrlen, flags);
}

static void ip_cork_release(struct inet_cork *cork)
{
        cork->flags &= ~IPCORK_OPT;
        kfree(cork->opt);
        cork->opt = NULL;
        dst_release(cork->dst);
        cork->dst = NULL;
}

/*
 *        Combined all pending IP fragments on the socket as one IP datagram
 *        and push them out.
 */
struct sk_buff *__ip_make_skb(struct sock *sk,
                              struct flowi4 *fl4,
                              struct sk_buff_head *queue,
                              struct inet_cork *cork)
{
        struct sk_buff *skb, *tmp_skb;
        struct sk_buff **tail_skb;
        struct inet_sock *inet = inet_sk(sk);
        struct net *net = sock_net(sk);
        struct ip_options *opt = NULL;
        struct rtable *rt = dst_rtable(cork->dst);
        struct iphdr *iph;
        u8 pmtudisc, ttl;
        __be16 df = 0;

        skb = __skb_dequeue(queue);
        if (!skb)
                goto out;
        tail_skb = &(skb_shinfo(skb)->frag_list);

        /* move skb->data to ip header from ext header */
        if (skb->data < skb_network_header(skb))
                __skb_pull(skb, skb_network_offset(skb));
        while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
                __skb_pull(tmp_skb, skb_network_header_len(skb));
                *tail_skb = tmp_skb;
                tail_skb = &(tmp_skb->next);
                skb->len += tmp_skb->len;
                skb->data_len += tmp_skb->len;
                skb->truesize += tmp_skb->truesize;
                tmp_skb->destructor = NULL;
                tmp_skb->sk = NULL;
        }

        /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
         * to fragment the frame generated here. No matter, what transforms
         * how transforms change size of the packet, it will come out.
         */
        skb->ignore_df = ip_sk_ignore_df(sk);

        /* DF bit is set when we want to see DF on outgoing frames.
         * If ignore_df is set too, we still allow to fragment this frame
         * locally. */
        pmtudisc = READ_ONCE(inet->pmtudisc);
        if (pmtudisc == IP_PMTUDISC_DO ||
            pmtudisc == IP_PMTUDISC_PROBE ||
            (skb->len <= dst_mtu(&rt->dst) &&
             ip_dont_fragment(sk, &rt->dst)))
                df = htons(IP_DF);

        if (cork->flags & IPCORK_OPT)
                opt = cork->opt;

        if (cork->ttl != 0)
                ttl = cork->ttl;
        else if (rt->rt_type == RTN_MULTICAST)
                ttl = READ_ONCE(inet->mc_ttl);
        else
                ttl = ip_select_ttl(inet, &rt->dst);

        iph = ip_hdr(skb);
        iph->version = 4;
        iph->ihl = 5;
        iph->tos = (cork->tos != -1) ? cork->tos : READ_ONCE(inet->tos);
        iph->frag_off = df;
        iph->ttl = ttl;
        iph->protocol = sk->sk_protocol;
        ip_copy_addrs(iph, fl4);
        ip_select_ident(net, skb, sk);

        if (opt) {
                iph->ihl += opt->optlen >> 2;
                ip_options_build(skb, opt, cork->addr, rt);
        }

        skb->priority = cork->priority;
        skb->mark = cork->mark;
        if (sk_is_tcp(sk))
                skb_set_delivery_time(skb, cork->transmit_time, SKB_CLOCK_MONOTONIC);
        else
                skb_set_delivery_type_by_clockid(skb, cork->transmit_time, sk->sk_clockid);
        /*
         * Steal rt from cork.dst to avoid a pair of atomic_inc/atomic_dec
         * on dst refcount
         */
        cork->dst = NULL;
        skb_dst_set(skb, &rt->dst);

        if (iph->protocol == IPPROTO_ICMP) {
                u8 icmp_type;

                /* For such sockets, transhdrlen is zero when do ip_append_data(),
                 * so icmphdr does not in skb linear region and can not get icmp_type
                 * by icmp_hdr(skb)->type.
                 */
                if (sk->sk_type == SOCK_RAW &&
                    !(fl4->flowi4_flags & FLOWI_FLAG_KNOWN_NH))
                        icmp_type = fl4->fl4_icmp_type;
                else
                        icmp_type = icmp_hdr(skb)->type;
                icmp_out_count(net, icmp_type);
        }

        ip_cork_release(cork);
out:
        return skb;
}

int ip_send_skb(struct net *net, struct sk_buff *skb)
{
        int err;

        err = ip_local_out(net, skb->sk, skb);
        if (err) {
                if (err > 0)
                        err = net_xmit_errno(err);
                if (err)
                        IP_INC_STATS(net, IPSTATS_MIB_OUTDISCARDS);
        }

        return err;
}

int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4)
{
        struct sk_buff *skb;

        skb = ip_finish_skb(sk, fl4);
        if (!skb)
                return 0;

        /* Netfilter gets whole the not fragmented skb. */
        return ip_send_skb(sock_net(sk), skb);
}

/*
 *        Throw away all pending data on the socket.
 */
static void __ip_flush_pending_frames(struct sock *sk,
                                      struct sk_buff_head *queue,
                                      struct inet_cork *cork)
{
        struct sk_buff *skb;

        while ((skb = __skb_dequeue_tail(queue)) != NULL)
                kfree_skb(skb);

        ip_cork_release(cork);
}

void ip_flush_pending_frames(struct sock *sk)
{
        __ip_flush_pending_frames(sk, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
}

struct sk_buff *ip_make_skb(struct sock *sk,
                            struct flowi4 *fl4,
                            int getfrag(void *from, char *to, int offset,
                                        int len, int odd, struct sk_buff *skb),
                            void *from, int length, int transhdrlen,
                            struct ipcm_cookie *ipc, struct rtable **rtp,
                            struct inet_cork *cork, unsigned int flags)
{
        struct sk_buff_head queue;
        int err;

        if (flags & MSG_PROBE)
                return NULL;

        __skb_queue_head_init(&queue);

        cork->flags = 0;
        cork->addr = 0;
        cork->opt = NULL;
        err = ip_setup_cork(sk, cork, ipc, rtp);
        if (err)
                return ERR_PTR(err);

        err = __ip_append_data(sk, fl4, &queue, cork,
                               &current->task_frag, getfrag,
                               from, length, transhdrlen, flags);
        if (err) {
                __ip_flush_pending_frames(sk, &queue, cork);
                return ERR_PTR(err);
        }

        return __ip_make_skb(sk, fl4, &queue, cork);
}

/*
 *        Fetch data from kernel space and fill in checksum if needed.
 */
static int ip_reply_glue_bits(void *dptr, char *to, int offset,
                              int len, int odd, struct sk_buff *skb)
{
        __wsum csum;

        csum = csum_partial_copy_nocheck(dptr+offset, to, len);
        skb->csum = csum_block_add(skb->csum, csum, odd);
        return 0;
}

/*
 *        Generic function to send a packet as reply to another packet.
 *        Used to send some TCP resets/acks so far.
 */
void ip_send_unicast_reply(struct sock *sk, const struct sock *orig_sk,
                           struct sk_buff *skb,
                           const struct ip_options *sopt,
                           __be32 daddr, __be32 saddr,
                           const struct ip_reply_arg *arg,
                           unsigned int len, u64 transmit_time, u32 txhash)
{
        struct ip_options_data replyopts;
        struct ipcm_cookie ipc;
        struct flowi4 fl4;
        struct rtable *rt = skb_rtable(skb);
        struct net *net = sock_net(sk);
        struct sk_buff *nskb;
        int err;
        int oif;

        if (__ip_options_echo(net, &replyopts.opt.opt, skb, sopt))
                return;

        ipcm_init(&ipc);
        ipc.addr = daddr;
        ipc.sockc.transmit_time = transmit_time;

        if (replyopts.opt.opt.optlen) {
                ipc.opt = &replyopts.opt;

                if (replyopts.opt.opt.srr)
                        daddr = replyopts.opt.opt.faddr;
        }

        oif = arg->bound_dev_if;
        if (!oif && netif_index_is_l3_master(net, skb->skb_iif))
                oif = skb->skb_iif;

        flowi4_init_output(&fl4, oif,
                           IP4_REPLY_MARK(net, skb->mark) ?: sk->sk_mark,
                           arg->tos & INET_DSCP_MASK,
                           RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,
                           ip_reply_arg_flowi_flags(arg),
                           daddr, saddr,
                           tcp_hdr(skb)->source, tcp_hdr(skb)->dest,
                           arg->uid);
        security_skb_classify_flow(skb, flowi4_to_flowi_common(&fl4));
        rt = ip_route_output_flow(net, &fl4, sk);
        if (IS_ERR(rt))
                return;

        inet_sk(sk)->tos = arg->tos;

        sk->sk_protocol = ip_hdr(skb)->protocol;
        sk->sk_bound_dev_if = arg->bound_dev_if;
        sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default);
        ipc.sockc.mark = fl4.flowi4_mark;
        err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base,
                             len, 0, &ipc, &rt, MSG_DONTWAIT);
        if (unlikely(err)) {
                ip_flush_pending_frames(sk);
                goto out;
        }

        nskb = skb_peek(&sk->sk_write_queue);
        if (nskb) {
                if (arg->csumoffset >= 0)
                        *((__sum16 *)skb_transport_header(nskb) +
                          arg->csumoffset) = csum_fold(csum_add(nskb->csum,
                                                                arg->csum));
                nskb->ip_summed = CHECKSUM_NONE;
                if (orig_sk) {
                        skb_set_owner_edemux(nskb, (struct sock *)orig_sk);
                        psp_reply_set_decrypted(orig_sk, nskb);
                }
                if (transmit_time)
                        nskb->tstamp_type = SKB_CLOCK_MONOTONIC;
                if (txhash)
                        skb_set_hash(nskb, txhash, PKT_HASH_TYPE_L4);
                ip_push_pending_frames(sk, &fl4);
        }
out:
        ip_rt_put(rt);
}

void __init ip_init(void)
{
        ip_rt_init();
        inet_initpeers();

#if defined(CONFIG_IP_MULTICAST)
        igmp_mc_init();
#endif
}


































































































































































































   31 




















   30 



   31 

























































































   29 













   19 






   18 



    2 
    4 













    3 





    2 
    4 













   17 








   17 






   23 
    3 




























































































   25 



    1 



   26 

   25 





   21 






   22 
   25 



    1 

    1 

    6 




    4 




    5 


















   16 





    8 



    1 


    1 






   14 

    1 






    1 
    2 































    4 






    4 














































    5 



    5 

































    8 















































































































    8 





























































   17 





   17 










   16 
    8 

    6 












   16 






    1 


    1 





    4 








    4 

















    4 


    4 






    8 
    8 
    8 


















    8 

































   26 









   17 


    9 




   26 





    9 


    5 





    1 




    4 







    3 






    3 











    3 







    3 



    3 



    3 






    2 




   23 





   32 




    1 

   31 


    1 






    1 


    1 









    7 
   20 






   25 
    1 





   25 




























    3 





































    3 





    3 



    3 









    2 






    3 


















    3 


    3 




    3 


    3 






































































































    3 




    1 






    1 
    1 


    1 



























    6 


    1 

    5 


    1 


    3 

















































































    2 




    1 


    1 




















    5 







    3 




    2 








    1 



    1 


    1 




    1 



    1 




























































































































































































































































































































































































































































































































































































































































































































































































































































































   13 










   13 





    1 




   11 

    1 

   11 



    2 




    2 



    1 




    4 



    4 


















   38 






    1 






   31 







    6 



















    5 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  ebtables
 *
 *  Author:
 *  Bart De Schuymer                <bdschuym@pandora.be>
 *
 *  ebtables.c,v 2.0, July, 2002
 *
 *  This code is strongly inspired by the iptables code which is
 *  Copyright (C) 1999 Paul `Rusty' Russell & Michael J. Neuling
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/kmod.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter_bridge/ebtables.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
#include <linux/audit.h>
#include <net/sock.h>
#include <net/netns/generic.h>
/* needed for logical [in,out]-dev filtering */
#include "../br_private.h"

/* Each cpu has its own set of counters, so there is no need for write_lock in
 * the softirq
 * For reading or updating the counters, the user context needs to
 * get a write_lock
 */

/* The size of each set of counters is altered to get cache alignment */
#define SMP_ALIGN(x) (((x) + SMP_CACHE_BYTES-1) & ~(SMP_CACHE_BYTES-1))
#define COUNTER_OFFSET(n) (SMP_ALIGN(n * sizeof(struct ebt_counter)))
#define COUNTER_BASE(c, n, cpu) ((struct ebt_counter *)(((char *)c) + \
                                 COUNTER_OFFSET(n) * cpu))

struct ebt_pernet {
        struct list_head tables;
};

struct ebt_template {
        struct list_head list;
        char name[EBT_TABLE_MAXNAMELEN];
        struct module *owner;
        /* called when table is needed in the given netns */
        int (*table_init)(struct net *net);
};

static unsigned int ebt_pernet_id __read_mostly;
static LIST_HEAD(template_tables);
static DEFINE_MUTEX(ebt_mutex);

#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
static void ebt_standard_compat_from_user(void *dst, const void *src)
{
        int v = *(compat_int_t *)src;

        if (v >= 0)
                v += xt_compat_calc_jump(NFPROTO_BRIDGE, v);
        memcpy(dst, &v, sizeof(v));
}

static int ebt_standard_compat_to_user(void __user *dst, const void *src)
{
        compat_int_t cv = *(int *)src;

        if (cv >= 0)
                cv -= xt_compat_calc_jump(NFPROTO_BRIDGE, cv);
        return copy_to_user(dst, &cv, sizeof(cv)) ? -EFAULT : 0;
}
#endif


static struct xt_target ebt_standard_target = {
        .name       = "standard",
        .revision   = 0,
        .family     = NFPROTO_BRIDGE,
        .targetsize = sizeof(int),
#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
        .compatsize = sizeof(compat_int_t),
        .compat_from_user = ebt_standard_compat_from_user,
        .compat_to_user =  ebt_standard_compat_to_user,
#endif
};

static inline int
ebt_do_watcher(const struct ebt_entry_watcher *w, struct sk_buff *skb,
               struct xt_action_param *par)
{
        par->target   = w->u.watcher;
        par->targinfo = w->data;
        w->u.watcher->target(skb, par);
        /* watchers don't give a verdict */
        return 0;
}

static inline int
ebt_do_match(struct ebt_entry_match *m, const struct sk_buff *skb,
             struct xt_action_param *par)
{
        par->match     = m->u.match;
        par->matchinfo = m->data;
        return !m->u.match->match(skb, par);
}

static inline int
ebt_dev_check(const char *entry, const struct net_device *device)
{
        int i = 0;
        const char *devname;

        if (*entry == '\0')
                return 0;
        if (!device)
                return 1;
        devname = device->name;
        /* 1 is the wildcard token */
        while (entry[i] != '\0' && entry[i] != 1 && entry[i] == devname[i])
                i++;
        return devname[i] != entry[i] && entry[i] != 1;
}

/* process standard matches */
static inline int
ebt_basic_match(const struct ebt_entry *e, const struct sk_buff *skb,
                const struct net_device *in, const struct net_device *out)
{
        const struct ethhdr *h = eth_hdr(skb);
        const struct net_bridge_port *p;
        __be16 ethproto;

        if (skb_vlan_tag_present(skb))
                ethproto = htons(ETH_P_8021Q);
        else
                ethproto = h->h_proto;

        if (e->bitmask & EBT_802_3) {
                if (NF_INVF(e, EBT_IPROTO, eth_proto_is_802_3(ethproto)))
                        return 1;
        } else if (!(e->bitmask & EBT_NOPROTO) &&
                   NF_INVF(e, EBT_IPROTO, e->ethproto != ethproto))
                return 1;

        if (NF_INVF(e, EBT_IIN, ebt_dev_check(e->in, in)))
                return 1;
        if (NF_INVF(e, EBT_IOUT, ebt_dev_check(e->out, out)))
                return 1;
        /* rcu_read_lock()ed by nf_hook_thresh */
        if (in && (p = br_port_get_rcu(in)) != NULL &&
            NF_INVF(e, EBT_ILOGICALIN,
                    ebt_dev_check(e->logical_in, p->br->dev)))
                return 1;
        if (out && (p = br_port_get_rcu(out)) != NULL &&
            NF_INVF(e, EBT_ILOGICALOUT,
                    ebt_dev_check(e->logical_out, p->br->dev)))
                return 1;

        if (e->bitmask & EBT_SOURCEMAC) {
                if (NF_INVF(e, EBT_ISOURCE,
                            !ether_addr_equal_masked(h->h_source, e->sourcemac,
                                                     e->sourcemsk)))
                        return 1;
        }
        if (e->bitmask & EBT_DESTMAC) {
                if (NF_INVF(e, EBT_IDEST,
                            !ether_addr_equal_masked(h->h_dest, e->destmac,
                                                     e->destmsk)))
                        return 1;
        }
        return 0;
}

static inline
struct ebt_entry *ebt_next_entry(const struct ebt_entry *entry)
{
        return (void *)entry + entry->next_offset;
}

static inline const struct ebt_entry_target *
ebt_get_target_c(const struct ebt_entry *e)
{
        return ebt_get_target((struct ebt_entry *)e);
}

/* Do some firewalling */
unsigned int ebt_do_table(void *priv, struct sk_buff *skb,
                          const struct nf_hook_state *state)
{
        struct ebt_table *table = priv;
        unsigned int hook = state->hook;
        int i, nentries;
        struct ebt_entry *point;
        struct ebt_counter *counter_base, *cb_base;
        const struct ebt_entry_target *t;
        int verdict, sp = 0;
        struct ebt_chainstack *cs;
        struct ebt_entries *chaininfo;
        const char *base;
        const struct ebt_table_info *private;
        struct xt_action_param acpar;

        acpar.state   = state;
        acpar.hotdrop = false;

        read_lock_bh(&table->lock);
        private = table->private;
        cb_base = COUNTER_BASE(private->counters, private->nentries,
           smp_processor_id());
        if (private->chainstack)
                cs = private->chainstack[smp_processor_id()];
        else
                cs = NULL;
        chaininfo = private->hook_entry[hook];
        nentries = private->hook_entry[hook]->nentries;
        point = (struct ebt_entry *)(private->hook_entry[hook]->data);
        counter_base = cb_base + private->hook_entry[hook]->counter_offset;
        /* base for chain jumps */
        base = private->entries;
        i = 0;
        while (i < nentries) {
                if (ebt_basic_match(point, skb, state->in, state->out))
                        goto letscontinue;

                if (EBT_MATCH_ITERATE(point, ebt_do_match, skb, &acpar) != 0)
                        goto letscontinue;
                if (acpar.hotdrop) {
                        read_unlock_bh(&table->lock);
                        return NF_DROP;
                }

                ADD_COUNTER(*(counter_base + i), skb->len, 1);

                /* these should only watch: not modify, nor tell us
                 * what to do with the packet
                 */
                EBT_WATCHER_ITERATE(point, ebt_do_watcher, skb, &acpar);

                t = ebt_get_target_c(point);
                /* standard target */
                if (!t->u.target->target)
                        verdict = ((struct ebt_standard_target *)t)->verdict;
                else {
                        acpar.target   = t->u.target;
                        acpar.targinfo = t->data;
                        verdict = t->u.target->target(skb, &acpar);
                }
                if (verdict == EBT_ACCEPT) {
                        read_unlock_bh(&table->lock);
                        return NF_ACCEPT;
                }
                if (verdict == EBT_DROP) {
                        read_unlock_bh(&table->lock);
                        return NF_DROP;
                }
                if (verdict == EBT_RETURN) {
letsreturn:
                        if (WARN(sp == 0, "RETURN on base chain")) {
                                /* act like this is EBT_CONTINUE */
                                goto letscontinue;
                        }

                        sp--;
                        /* put all the local variables right */
                        i = cs[sp].n;
                        chaininfo = cs[sp].chaininfo;
                        nentries = chaininfo->nentries;
                        point = cs[sp].e;
                        counter_base = cb_base +
                           chaininfo->counter_offset;
                        continue;
                }
                if (verdict == EBT_CONTINUE)
                        goto letscontinue;

                if (WARN(verdict < 0, "bogus standard verdict\n")) {
                        read_unlock_bh(&table->lock);
                        return NF_DROP;
                }

                /* jump to a udc */
                cs[sp].n = i + 1;
                cs[sp].chaininfo = chaininfo;
                cs[sp].e = ebt_next_entry(point);
                i = 0;
                chaininfo = (struct ebt_entries *) (base + verdict);

                if (WARN(chaininfo->distinguisher, "jump to non-chain\n")) {
                        read_unlock_bh(&table->lock);
                        return NF_DROP;
                }

                nentries = chaininfo->nentries;
                point = (struct ebt_entry *)chaininfo->data;
                counter_base = cb_base + chaininfo->counter_offset;
                sp++;
                continue;
letscontinue:
                point = ebt_next_entry(point);
                i++;
        }

        /* I actually like this :) */
        if (chaininfo->policy == EBT_RETURN)
                goto letsreturn;
        if (chaininfo->policy == EBT_ACCEPT) {
                read_unlock_bh(&table->lock);
                return NF_ACCEPT;
        }
        read_unlock_bh(&table->lock);
        return NF_DROP;
}

/* If it succeeds, returns element and locks mutex */
static inline void *
find_inlist_lock_noload(struct net *net, const char *name, int *error,
                        struct mutex *mutex)
{
        struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
        struct ebt_template *tmpl;
        struct ebt_table *table;

        mutex_lock(mutex);
        list_for_each_entry(table, &ebt_net->tables, list) {
                if (strcmp(table->name, name) == 0)
                        return table;
        }

        list_for_each_entry(tmpl, &template_tables, list) {
                if (strcmp(name, tmpl->name) == 0) {
                        struct module *owner = tmpl->owner;

                        if (!try_module_get(owner))
                                goto out;

                        mutex_unlock(mutex);

                        *error = tmpl->table_init(net);
                        if (*error) {
                                module_put(owner);
                                return NULL;
                        }

                        mutex_lock(mutex);
                        module_put(owner);
                        break;
                }
        }

        list_for_each_entry(table, &ebt_net->tables, list) {
                if (strcmp(table->name, name) == 0)
                        return table;
        }

out:
        *error = -ENOENT;
        mutex_unlock(mutex);
        return NULL;
}

static void *
find_inlist_lock(struct net *net, const char *name, const char *prefix,
                 int *error, struct mutex *mutex)
{
        return try_then_request_module(
                        find_inlist_lock_noload(net, name, error, mutex),
                        "%s%s", prefix, name);
}

static inline struct ebt_table *
find_table_lock(struct net *net, const char *name, int *error,
                struct mutex *mutex)
{
        return find_inlist_lock(net, name, "ebtable_", error, mutex);
}

static inline void ebt_free_table_info(struct ebt_table_info *info)
{
        int i;

        if (info->chainstack) {
                for_each_possible_cpu(i)
                        vfree(info->chainstack[i]);
                vfree(info->chainstack);
        }
}
static inline int
ebt_check_match(struct ebt_entry_match *m, struct xt_mtchk_param *par,
                unsigned int *cnt)
{
        const struct ebt_entry *e = par->entryinfo;
        struct xt_match *match;
        size_t left = ((char *)e + e->watchers_offset) - (char *)m;
        int ret;

        if (left < sizeof(struct ebt_entry_match) ||
            left - sizeof(struct ebt_entry_match) < m->match_size)
                return -EINVAL;

        match = xt_find_match(NFPROTO_BRIDGE, m->u.name, m->u.revision);
        if (IS_ERR(match) || match->family != NFPROTO_BRIDGE) {
                if (!IS_ERR(match))
                        module_put(match->me);
                request_module("ebt_%s", m->u.name);
                match = xt_find_match(NFPROTO_BRIDGE, m->u.name, m->u.revision);
        }
        if (IS_ERR(match))
                return PTR_ERR(match);
        m->u.match = match;

        par->match     = match;
        par->matchinfo = m->data;
        ret = xt_check_match(par, m->match_size,
              ntohs(e->ethproto), e->invflags & EBT_IPROTO);
        if (ret < 0) {
                module_put(match->me);
                return ret;
        }

        (*cnt)++;
        return 0;
}

static inline int
ebt_check_watcher(struct ebt_entry_watcher *w, struct xt_tgchk_param *par,
                  unsigned int *cnt)
{
        const struct ebt_entry *e = par->entryinfo;
        struct xt_target *watcher;
        size_t left = ((char *)e + e->target_offset) - (char *)w;
        int ret;

        if (left < sizeof(struct ebt_entry_watcher) ||
           left - sizeof(struct ebt_entry_watcher) < w->watcher_size)
                return -EINVAL;

        watcher = xt_request_find_target(NFPROTO_BRIDGE, w->u.name, 0);
        if (IS_ERR(watcher))
                return PTR_ERR(watcher);

        if (watcher->family != NFPROTO_BRIDGE) {
                module_put(watcher->me);
                return -ENOENT;
        }

        w->u.watcher = watcher;

        par->target   = watcher;
        par->targinfo = w->data;
        ret = xt_check_target(par, w->watcher_size,
              ntohs(e->ethproto), e->invflags & EBT_IPROTO);
        if (ret < 0) {
                module_put(watcher->me);
                return ret;
        }

        (*cnt)++;
        return 0;
}

static int ebt_verify_pointers(const struct ebt_replace *repl,
                               struct ebt_table_info *newinfo)
{
        unsigned int limit = repl->entries_size;
        unsigned int valid_hooks = repl->valid_hooks;
        unsigned int offset = 0;
        int i;

        for (i = 0; i < NF_BR_NUMHOOKS; i++)
                newinfo->hook_entry[i] = NULL;

        newinfo->entries_size = repl->entries_size;
        newinfo->nentries = repl->nentries;

        while (offset < limit) {
                size_t left = limit - offset;
                struct ebt_entry *e = (void *)newinfo->entries + offset;

                if (left < sizeof(unsigned int))
                        break;

                for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                        if ((valid_hooks & (1 << i)) == 0)
                                continue;
                        if ((char __user *)repl->hook_entry[i] ==
                             repl->entries + offset)
                                break;
                }

                if (i != NF_BR_NUMHOOKS || !(e->bitmask & EBT_ENTRY_OR_ENTRIES)) {
                        if (e->bitmask != 0) {
                                /* we make userspace set this right,
                                 * so there is no misunderstanding
                                 */
                                return -EINVAL;
                        }
                        if (i != NF_BR_NUMHOOKS)
                                newinfo->hook_entry[i] = (struct ebt_entries *)e;
                        if (left < sizeof(struct ebt_entries))
                                break;
                        offset += sizeof(struct ebt_entries);
                } else {
                        if (left < sizeof(struct ebt_entry))
                                break;
                        if (left < e->next_offset)
                                break;
                        if (e->next_offset < sizeof(struct ebt_entry))
                                return -EINVAL;
                        offset += e->next_offset;
                }
        }
        if (offset != limit)
                return -EINVAL;

        /* check if all valid hooks have a chain */
        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                if (!newinfo->hook_entry[i] &&
                   (valid_hooks & (1 << i)))
                        return -EINVAL;
        }
        return 0;
}

/* this one is very careful, as it is the first function
 * to parse the userspace data
 */
static inline int
ebt_check_entry_size_and_hooks(const struct ebt_entry *e,
                               const struct ebt_table_info *newinfo,
                               unsigned int *n, unsigned int *cnt,
                               unsigned int *totalcnt, unsigned int *udc_cnt)
{
        int i;

        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                if ((void *)e == (void *)newinfo->hook_entry[i])
                        break;
        }
        /* beginning of a new chain
         * if i == NF_BR_NUMHOOKS it must be a user defined chain
         */
        if (i != NF_BR_NUMHOOKS || !e->bitmask) {
                /* this checks if the previous chain has as many entries
                 * as it said it has
                 */
                if (*n != *cnt)
                        return -EINVAL;

                if (((struct ebt_entries *)e)->policy != EBT_DROP &&
                   ((struct ebt_entries *)e)->policy != EBT_ACCEPT) {
                        /* only RETURN from udc */
                        if (i != NF_BR_NUMHOOKS ||
                           ((struct ebt_entries *)e)->policy != EBT_RETURN)
                                return -EINVAL;
                }
                if (i == NF_BR_NUMHOOKS) /* it's a user defined chain */
                        (*udc_cnt)++;
                if (((struct ebt_entries *)e)->counter_offset != *totalcnt)
                        return -EINVAL;
                *n = ((struct ebt_entries *)e)->nentries;
                *cnt = 0;
                return 0;
        }
        /* a plain old entry, heh */
        if (sizeof(struct ebt_entry) > e->watchers_offset ||
           e->watchers_offset > e->target_offset ||
           e->target_offset >= e->next_offset)
                return -EINVAL;

        /* this is not checked anywhere else */
        if (e->next_offset - e->target_offset < sizeof(struct ebt_entry_target))
                return -EINVAL;

        (*cnt)++;
        (*totalcnt)++;
        return 0;
}

struct ebt_cl_stack {
        struct ebt_chainstack cs;
        int from;
        unsigned int hookmask;
};

/* We need these positions to check that the jumps to a different part of the
 * entries is a jump to the beginning of a new chain.
 */
static inline int
ebt_get_udc_positions(struct ebt_entry *e, struct ebt_table_info *newinfo,
                      unsigned int *n, struct ebt_cl_stack *udc)
{
        int i;

        /* we're only interested in chain starts */
        if (e->bitmask)
                return 0;
        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                if (newinfo->hook_entry[i] == (struct ebt_entries *)e)
                        break;
        }
        /* only care about udc */
        if (i != NF_BR_NUMHOOKS)
                return 0;

        udc[*n].cs.chaininfo = (struct ebt_entries *)e;
        /* these initialisations are depended on later in check_chainloops() */
        udc[*n].cs.n = 0;
        udc[*n].hookmask = 0;

        (*n)++;
        return 0;
}

static inline int
ebt_cleanup_match(struct ebt_entry_match *m, struct net *net, unsigned int *i)
{
        struct xt_mtdtor_param par;

        if (i && (*i)-- == 0)
                return 1;

        par.net       = net;
        par.match     = m->u.match;
        par.matchinfo = m->data;
        par.family    = NFPROTO_BRIDGE;
        if (par.match->destroy != NULL)
                par.match->destroy(&par);
        module_put(par.match->me);
        return 0;
}

static inline int
ebt_cleanup_watcher(struct ebt_entry_watcher *w, struct net *net, unsigned int *i)
{
        struct xt_tgdtor_param par;

        if (i && (*i)-- == 0)
                return 1;

        par.net      = net;
        par.target   = w->u.watcher;
        par.targinfo = w->data;
        par.family   = NFPROTO_BRIDGE;
        if (par.target->destroy != NULL)
                par.target->destroy(&par);
        module_put(par.target->me);
        return 0;
}

static inline int
ebt_cleanup_entry(struct ebt_entry *e, struct net *net, unsigned int *cnt)
{
        struct xt_tgdtor_param par;
        struct ebt_entry_target *t;

        if (e->bitmask == 0)
                return 0;
        /* we're done */
        if (cnt && (*cnt)-- == 0)
                return 1;
        EBT_WATCHER_ITERATE(e, ebt_cleanup_watcher, net, NULL);
        EBT_MATCH_ITERATE(e, ebt_cleanup_match, net, NULL);
        t = ebt_get_target(e);

        par.net      = net;
        par.target   = t->u.target;
        par.targinfo = t->data;
        par.family   = NFPROTO_BRIDGE;
        if (par.target->destroy != NULL)
                par.target->destroy(&par);
        module_put(par.target->me);
        return 0;
}

static inline int
ebt_check_entry(struct ebt_entry *e, struct net *net,
                const struct ebt_table_info *newinfo,
                const char *name, unsigned int *cnt,
                struct ebt_cl_stack *cl_s, unsigned int udc_cnt)
{
        struct ebt_entry_target *t;
        struct xt_target *target;
        unsigned int i, j, hook = 0, hookmask = 0;
        size_t gap;
        int ret;
        struct xt_mtchk_param mtpar;
        struct xt_tgchk_param tgpar;

        /* don't mess with the struct ebt_entries */
        if (e->bitmask == 0)
                return 0;

        if (e->bitmask & ~EBT_F_MASK)
                return -EINVAL;

        if (e->invflags & ~EBT_INV_MASK)
                return -EINVAL;

        if ((e->bitmask & EBT_NOPROTO) && (e->bitmask & EBT_802_3))
                return -EINVAL;

        /* what hook do we belong to? */
        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                if (!newinfo->hook_entry[i])
                        continue;
                if ((char *)newinfo->hook_entry[i] < (char *)e)
                        hook = i;
                else
                        break;
        }
        /* (1 << NF_BR_NUMHOOKS) tells the check functions the rule is on
         * a base chain
         */
        if (i < NF_BR_NUMHOOKS)
                hookmask = (1 << hook) | (1 << NF_BR_NUMHOOKS);
        else {
                for (i = 0; i < udc_cnt; i++)
                        if ((char *)(cl_s[i].cs.chaininfo) > (char *)e)
                                break;
                if (i == 0)
                        hookmask = (1 << hook) | (1 << NF_BR_NUMHOOKS);
                else
                        hookmask = cl_s[i - 1].hookmask;
        }
        i = 0;

        memset(&mtpar, 0, sizeof(mtpar));
        memset(&tgpar, 0, sizeof(tgpar));
        mtpar.net        = tgpar.net       = net;
        mtpar.table     = tgpar.table     = name;
        mtpar.entryinfo = tgpar.entryinfo = e;
        mtpar.hook_mask = tgpar.hook_mask = hookmask;
        mtpar.family    = tgpar.family    = NFPROTO_BRIDGE;
        ret = EBT_MATCH_ITERATE(e, ebt_check_match, &mtpar, &i);
        if (ret != 0)
                goto cleanup_matches;
        j = 0;
        ret = EBT_WATCHER_ITERATE(e, ebt_check_watcher, &tgpar, &j);
        if (ret != 0)
                goto cleanup_watchers;
        t = ebt_get_target(e);
        gap = e->next_offset - e->target_offset;

        target = xt_request_find_target(NFPROTO_BRIDGE, t->u.name, 0);
        if (IS_ERR(target)) {
                ret = PTR_ERR(target);
                goto cleanup_watchers;
        }

        /* Reject UNSPEC, xtables verdicts/return values are incompatible */
        if (target->family != NFPROTO_BRIDGE) {
                module_put(target->me);
                ret = -ENOENT;
                goto cleanup_watchers;
        }

        t->u.target = target;
        if (t->u.target == &ebt_standard_target) {
                if (gap < sizeof(struct ebt_standard_target)) {
                        ret = -EFAULT;
                        goto cleanup_watchers;
                }
                if (((struct ebt_standard_target *)t)->verdict <
                   -NUM_STANDARD_TARGETS) {
                        ret = -EFAULT;
                        goto cleanup_watchers;
                }
        } else if (t->target_size > gap - sizeof(struct ebt_entry_target)) {
                module_put(t->u.target->me);
                ret = -EFAULT;
                goto cleanup_watchers;
        }

        tgpar.target   = target;
        tgpar.targinfo = t->data;
        ret = xt_check_target(&tgpar, t->target_size,
              ntohs(e->ethproto), e->invflags & EBT_IPROTO);
        if (ret < 0) {
                module_put(target->me);
                goto cleanup_watchers;
        }
        (*cnt)++;
        return 0;
cleanup_watchers:
        EBT_WATCHER_ITERATE(e, ebt_cleanup_watcher, net, &j);
cleanup_matches:
        EBT_MATCH_ITERATE(e, ebt_cleanup_match, net, &i);
        return ret;
}

/* checks for loops and sets the hook mask for udc
 * the hook mask for udc tells us from which base chains the udc can be
 * accessed. This mask is a parameter to the check() functions of the extensions
 */
static int check_chainloops(const struct ebt_entries *chain, struct ebt_cl_stack *cl_s,
                            unsigned int udc_cnt, unsigned int hooknr, char *base)
{
        int i, chain_nr = -1, pos = 0, nentries = chain->nentries, verdict;
        const struct ebt_entry *e = (struct ebt_entry *)chain->data;
        const struct ebt_entry_target *t;

        while (pos < nentries || chain_nr != -1) {
                /* end of udc, go back one 'recursion' step */
                if (pos == nentries) {
                        /* put back values of the time when this chain was called */
                        e = cl_s[chain_nr].cs.e;
                        if (cl_s[chain_nr].from != -1)
                                nentries =
                                cl_s[cl_s[chain_nr].from].cs.chaininfo->nentries;
                        else
                                nentries = chain->nentries;
                        pos = cl_s[chain_nr].cs.n;
                        /* make sure we won't see a loop that isn't one */
                        cl_s[chain_nr].cs.n = 0;
                        chain_nr = cl_s[chain_nr].from;
                        if (pos == nentries)
                                continue;
                }
                t = ebt_get_target_c(e);
                if (strcmp(t->u.name, EBT_STANDARD_TARGET))
                        goto letscontinue;
                if (e->target_offset + sizeof(struct ebt_standard_target) >
                   e->next_offset)
                        return -1;

                verdict = ((struct ebt_standard_target *)t)->verdict;
                if (verdict >= 0) { /* jump to another chain */
                        struct ebt_entries *hlp2 =
                           (struct ebt_entries *)(base + verdict);
                        for (i = 0; i < udc_cnt; i++)
                                if (hlp2 == cl_s[i].cs.chaininfo)
                                        break;
                        /* bad destination or loop */
                        if (i == udc_cnt)
                                return -1;

                        if (cl_s[i].cs.n)
                                return -1;

                        if (cl_s[i].hookmask & (1 << hooknr))
                                goto letscontinue;
                        /* this can't be 0, so the loop test is correct */
                        cl_s[i].cs.n = pos + 1;
                        pos = 0;
                        cl_s[i].cs.e = ebt_next_entry(e);
                        e = (struct ebt_entry *)(hlp2->data);
                        nentries = hlp2->nentries;
                        cl_s[i].from = chain_nr;
                        chain_nr = i;
                        /* this udc is accessible from the base chain for hooknr */
                        cl_s[i].hookmask |= (1 << hooknr);
                        continue;
                }
letscontinue:
                e = ebt_next_entry(e);
                pos++;
        }
        return 0;
}

/* do the parsing of the table/chains/entries/matches/watchers/targets, heh */
static int translate_table(struct net *net, const char *name,
                           struct ebt_table_info *newinfo)
{
        unsigned int i, j, k, udc_cnt;
        int ret;
        struct ebt_cl_stack *cl_s = NULL; /* used in the checking for chain loops */

        i = 0;
        while (i < NF_BR_NUMHOOKS && !newinfo->hook_entry[i])
                i++;
        if (i == NF_BR_NUMHOOKS)
                return -EINVAL;

        if (newinfo->hook_entry[i] != (struct ebt_entries *)newinfo->entries)
                return -EINVAL;

        /* make sure chains are ordered after each other in same order
         * as their corresponding hooks
         */
        for (j = i + 1; j < NF_BR_NUMHOOKS; j++) {
                if (!newinfo->hook_entry[j])
                        continue;
                if (newinfo->hook_entry[j] <= newinfo->hook_entry[i])
                        return -EINVAL;

                i = j;
        }

        /* do some early checkings and initialize some things */
        i = 0; /* holds the expected nr. of entries for the chain */
        j = 0; /* holds the up to now counted entries for the chain */
        k = 0; /* holds the total nr. of entries, should equal
                * newinfo->nentries afterwards
                */
        udc_cnt = 0; /* will hold the nr. of user defined chains (udc) */
        ret = EBT_ENTRY_ITERATE(newinfo->entries, newinfo->entries_size,
           ebt_check_entry_size_and_hooks, newinfo,
           &i, &j, &k, &udc_cnt);

        if (ret != 0)
                return ret;

        if (i != j)
                return -EINVAL;

        if (k != newinfo->nentries)
                return -EINVAL;

        /* get the location of the udc, put them in an array
         * while we're at it, allocate the chainstack
         */
        if (udc_cnt) {
                /* this will get free'd in do_replace()/ebt_register_table()
                 * if an error occurs
                 */
                newinfo->chainstack =
                        vmalloc_array(nr_cpu_ids,
                                      sizeof(*(newinfo->chainstack)));
                if (!newinfo->chainstack)
                        return -ENOMEM;
                for_each_possible_cpu(i) {
                        newinfo->chainstack[i] =
                          vmalloc_node(array_size(udc_cnt,
                                          sizeof(*(newinfo->chainstack[0]))),
                                       cpu_to_node(i));
                        if (!newinfo->chainstack[i]) {
                                while (i)
                                        vfree(newinfo->chainstack[--i]);
                                vfree(newinfo->chainstack);
                                newinfo->chainstack = NULL;
                                return -ENOMEM;
                        }
                }

                cl_s = vmalloc_array(udc_cnt, sizeof(*cl_s));
                if (!cl_s)
                        return -ENOMEM;
                i = 0; /* the i'th udc */
                EBT_ENTRY_ITERATE(newinfo->entries, newinfo->entries_size,
                   ebt_get_udc_positions, newinfo, &i, cl_s);
                /* sanity check */
                if (i != udc_cnt) {
                        vfree(cl_s);
                        return -EFAULT;
                }
        }

        /* Check for loops */
        for (i = 0; i < NF_BR_NUMHOOKS; i++)
                if (newinfo->hook_entry[i])
                        if (check_chainloops(newinfo->hook_entry[i],
                           cl_s, udc_cnt, i, newinfo->entries)) {
                                vfree(cl_s);
                                return -EINVAL;
                        }

        /* we now know the following (along with E=mc²):
         *  - the nr of entries in each chain is right
         *  - the size of the allocated space is right
         *  - all valid hooks have a corresponding chain
         *  - there are no loops
         *  - wrong data can still be on the level of a single entry
         *  - could be there are jumps to places that are not the
         *    beginning of a chain. This can only occur in chains that
         *    are not accessible from any base chains, so we don't care.
         */

        /* used to know what we need to clean up if something goes wrong */
        i = 0;
        ret = EBT_ENTRY_ITERATE(newinfo->entries, newinfo->entries_size,
           ebt_check_entry, net, newinfo, name, &i, cl_s, udc_cnt);
        if (ret != 0) {
                EBT_ENTRY_ITERATE(newinfo->entries, newinfo->entries_size,
                                  ebt_cleanup_entry, net, &i);
        }
        vfree(cl_s);
        return ret;
}

/* called under write_lock */
static void get_counters(const struct ebt_counter *oldcounters,
                         struct ebt_counter *counters, unsigned int nentries)
{
        int i, cpu;
        struct ebt_counter *counter_base;

        /* counters of cpu 0 */
        memcpy(counters, oldcounters,
               sizeof(struct ebt_counter) * nentries);

        /* add other counters to those of cpu 0 */
        for_each_possible_cpu(cpu) {
                if (cpu == 0)
                        continue;
                counter_base = COUNTER_BASE(oldcounters, nentries, cpu);
                for (i = 0; i < nentries; i++)
                        ADD_COUNTER(counters[i], counter_base[i].bcnt,
                                    counter_base[i].pcnt);
        }
}

static int do_replace_finish(struct net *net, struct ebt_replace *repl,
                              struct ebt_table_info *newinfo)
{
        int ret;
        struct ebt_counter *counterstmp = NULL;
        /* used to be able to unlock earlier */
        struct ebt_table_info *table;
        struct ebt_table *t;

        /* the user wants counters back
         * the check on the size is done later, when we have the lock
         */
        if (repl->num_counters) {
                counterstmp = vmalloc_array(repl->num_counters,
                                            sizeof(*counterstmp));
                if (!counterstmp)
                        return -ENOMEM;
        }

        newinfo->chainstack = NULL;
        ret = ebt_verify_pointers(repl, newinfo);
        if (ret != 0)
                goto free_counterstmp;

        ret = translate_table(net, repl->name, newinfo);

        if (ret != 0)
                goto free_counterstmp;

        t = find_table_lock(net, repl->name, &ret, &ebt_mutex);
        if (!t) {
                ret = -ENOENT;
                goto free_iterate;
        }

        if (repl->valid_hooks != t->valid_hooks) {
                ret = -EINVAL;
                goto free_unlock;
        }

        if (repl->num_counters && repl->num_counters != t->private->nentries) {
                ret = -EINVAL;
                goto free_unlock;
        }

        /* we have the mutex lock, so no danger in reading this pointer */
        table = t->private;
        /* make sure the table can only be rmmod'ed if it contains no rules */
        if (!table->nentries && newinfo->nentries && !try_module_get(t->me)) {
                ret = -ENOENT;
                goto free_unlock;
        } else if (table->nentries && !newinfo->nentries)
                module_put(t->me);
        /* we need an atomic snapshot of the counters */
        write_lock_bh(&t->lock);
        if (repl->num_counters)
                get_counters(t->private->counters, counterstmp,
                   t->private->nentries);

        t->private = newinfo;
        write_unlock_bh(&t->lock);
        mutex_unlock(&ebt_mutex);
        /* so, a user can change the chains while having messed up her counter
         * allocation. Only reason why this is done is because this way the lock
         * is held only once, while this doesn't bring the kernel into a
         * dangerous state.
         */
        if (repl->num_counters &&
           copy_to_user(repl->counters, counterstmp,
           array_size(repl->num_counters, sizeof(struct ebt_counter)))) {
                /* Silent error, can't fail, new table is already in place */
                net_warn_ratelimited("ebtables: counters copy to user failed while replacing table\n");
        }

        /* decrease module count and free resources */
        EBT_ENTRY_ITERATE(table->entries, table->entries_size,
                          ebt_cleanup_entry, net, NULL);

        vfree(table->entries);
        ebt_free_table_info(table);
        vfree(table);
        vfree(counterstmp);

        audit_log_nfcfg(repl->name, AF_BRIDGE, repl->nentries,
                        AUDIT_XT_OP_REPLACE, GFP_KERNEL);
        return 0;

free_unlock:
        mutex_unlock(&ebt_mutex);
free_iterate:
        EBT_ENTRY_ITERATE(newinfo->entries, newinfo->entries_size,
                          ebt_cleanup_entry, net, NULL);
free_counterstmp:
        vfree(counterstmp);
        /* can be initialized in translate_table() */
        ebt_free_table_info(newinfo);
        return ret;
}

/* replace the table */
static int do_replace(struct net *net, sockptr_t arg, unsigned int len)
{
        int ret, countersize;
        struct ebt_table_info *newinfo;
        struct ebt_replace tmp;

        if (len < sizeof(tmp))
                return -EINVAL;
        if (copy_from_sockptr(&tmp, arg, sizeof(tmp)) != 0)
                return -EFAULT;

        if (len != sizeof(tmp) + tmp.entries_size)
                return -EINVAL;

        if (tmp.entries_size == 0)
                return -EINVAL;

        /* overflow check */
        if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) /
                        NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter))
                return -ENOMEM;
        if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter))
                return -ENOMEM;

        tmp.name[sizeof(tmp.name) - 1] = 0;

        countersize = COUNTER_OFFSET(tmp.nentries) * nr_cpu_ids;
        newinfo = __vmalloc(sizeof(*newinfo) + countersize, GFP_KERNEL_ACCOUNT);
        if (!newinfo)
                return -ENOMEM;

        if (countersize)
                memset(newinfo->counters, 0, countersize);

        newinfo->entries = __vmalloc(tmp.entries_size, GFP_KERNEL_ACCOUNT);
        if (!newinfo->entries) {
                ret = -ENOMEM;
                goto free_newinfo;
        }
        if (copy_from_user(
           newinfo->entries, tmp.entries, tmp.entries_size) != 0) {
                ret = -EFAULT;
                goto free_entries;
        }

        ret = do_replace_finish(net, &tmp, newinfo);
        if (ret == 0)
                return ret;
free_entries:
        vfree(newinfo->entries);
free_newinfo:
        vfree(newinfo);
        return ret;
}

static void __ebt_unregister_table(struct net *net, struct ebt_table *table)
{
        mutex_lock(&ebt_mutex);
        list_del(&table->list);
        mutex_unlock(&ebt_mutex);
        audit_log_nfcfg(table->name, AF_BRIDGE, table->private->nentries,
                        AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
        EBT_ENTRY_ITERATE(table->private->entries, table->private->entries_size,
                          ebt_cleanup_entry, net, NULL);
        if (table->private->nentries)
                module_put(table->me);
        vfree(table->private->entries);
        ebt_free_table_info(table->private);
        vfree(table->private);
        kfree(table->ops);
        kfree(table);
}

int ebt_register_table(struct net *net, const struct ebt_table *input_table,
                       const struct nf_hook_ops *template_ops)
{
        struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
        struct ebt_table_info *newinfo;
        struct ebt_table *t, *table;
        struct nf_hook_ops *ops;
        unsigned int num_ops;
        struct ebt_replace_kernel *repl;
        int ret, i, countersize;
        void *p;

        if (input_table == NULL || (repl = input_table->table) == NULL ||
            repl->entries == NULL || repl->entries_size == 0 ||
            repl->counters != NULL || input_table->private != NULL)
                return -EINVAL;

        /* Don't add one table to multiple lists. */
        table = kmemdup(input_table, sizeof(struct ebt_table), GFP_KERNEL);
        if (!table) {
                ret = -ENOMEM;
                goto out;
        }

        countersize = COUNTER_OFFSET(repl->nentries) * nr_cpu_ids;
        newinfo = vmalloc(sizeof(*newinfo) + countersize);
        ret = -ENOMEM;
        if (!newinfo)
                goto free_table;

        p = vmalloc(repl->entries_size);
        if (!p)
                goto free_newinfo;

        memcpy(p, repl->entries, repl->entries_size);
        newinfo->entries = p;

        newinfo->entries_size = repl->entries_size;
        newinfo->nentries = repl->nentries;

        if (countersize)
                memset(newinfo->counters, 0, countersize);

        /* fill in newinfo and parse the entries */
        newinfo->chainstack = NULL;
        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                if ((repl->valid_hooks & (1 << i)) == 0)
                        newinfo->hook_entry[i] = NULL;
                else
                        newinfo->hook_entry[i] = p +
                                ((char *)repl->hook_entry[i] - repl->entries);
        }
        ret = translate_table(net, repl->name, newinfo);
        if (ret != 0)
                goto free_chainstack;

        table->private = newinfo;
        rwlock_init(&table->lock);
        mutex_lock(&ebt_mutex);
        list_for_each_entry(t, &ebt_net->tables, list) {
                if (strcmp(t->name, table->name) == 0) {
                        ret = -EEXIST;
                        goto free_unlock;
                }
        }

        /* Hold a reference count if the chains aren't empty */
        if (newinfo->nentries && !try_module_get(table->me)) {
                ret = -ENOENT;
                goto free_unlock;
        }

        num_ops = hweight32(table->valid_hooks);
        if (num_ops == 0) {
                ret = -EINVAL;
                goto free_unlock;
        }

        ops = kmemdup_array(template_ops, num_ops, sizeof(*ops), GFP_KERNEL);
        if (!ops) {
                ret = -ENOMEM;
                if (newinfo->nentries)
                        module_put(table->me);
                goto free_unlock;
        }

        for (i = 0; i < num_ops; i++)
                ops[i].priv = table;

        list_add(&table->list, &ebt_net->tables);
        mutex_unlock(&ebt_mutex);

        table->ops = ops;
        ret = nf_register_net_hooks(net, ops, num_ops);
        if (ret)
                __ebt_unregister_table(net, table);

        audit_log_nfcfg(repl->name, AF_BRIDGE, repl->nentries,
                        AUDIT_XT_OP_REGISTER, GFP_KERNEL);
        return ret;
free_unlock:
        mutex_unlock(&ebt_mutex);
free_chainstack:
        ebt_free_table_info(newinfo);
        vfree(newinfo->entries);
free_newinfo:
        vfree(newinfo);
free_table:
        kfree(table);
out:
        return ret;
}

int ebt_register_template(const struct ebt_table *t, int (*table_init)(struct net *net))
{
        struct ebt_template *tmpl;

        mutex_lock(&ebt_mutex);
        list_for_each_entry(tmpl, &template_tables, list) {
                if (WARN_ON_ONCE(strcmp(t->name, tmpl->name) == 0)) {
                        mutex_unlock(&ebt_mutex);
                        return -EEXIST;
                }
        }

        tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL);
        if (!tmpl) {
                mutex_unlock(&ebt_mutex);
                return -ENOMEM;
        }

        tmpl->table_init = table_init;
        strscpy(tmpl->name, t->name, sizeof(tmpl->name));
        tmpl->owner = t->me;
        list_add(&tmpl->list, &template_tables);

        mutex_unlock(&ebt_mutex);
        return 0;
}
EXPORT_SYMBOL(ebt_register_template);

void ebt_unregister_template(const struct ebt_table *t)
{
        struct ebt_template *tmpl;

        mutex_lock(&ebt_mutex);
        list_for_each_entry(tmpl, &template_tables, list) {
                if (strcmp(t->name, tmpl->name))
                        continue;

                list_del(&tmpl->list);
                mutex_unlock(&ebt_mutex);
                kfree(tmpl);
                return;
        }

        mutex_unlock(&ebt_mutex);
        WARN_ON_ONCE(1);
}
EXPORT_SYMBOL(ebt_unregister_template);

static struct ebt_table *__ebt_find_table(struct net *net, const char *name)
{
        struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);
        struct ebt_table *t;

        mutex_lock(&ebt_mutex);

        list_for_each_entry(t, &ebt_net->tables, list) {
                if (strcmp(t->name, name) == 0) {
                        mutex_unlock(&ebt_mutex);
                        return t;
                }
        }

        mutex_unlock(&ebt_mutex);
        return NULL;
}

void ebt_unregister_table_pre_exit(struct net *net, const char *name)
{
        struct ebt_table *table = __ebt_find_table(net, name);

        if (table)
                nf_unregister_net_hooks(net, table->ops, hweight32(table->valid_hooks));
}
EXPORT_SYMBOL(ebt_unregister_table_pre_exit);

void ebt_unregister_table(struct net *net, const char *name)
{
        struct ebt_table *table = __ebt_find_table(net, name);

        if (table)
                __ebt_unregister_table(net, table);
}

/* userspace just supplied us with counters */
static int do_update_counters(struct net *net, const char *name,
                              struct ebt_counter __user *counters,
                              unsigned int num_counters, unsigned int len)
{
        int i, ret;
        struct ebt_counter *tmp;
        struct ebt_table *t;

        if (num_counters == 0)
                return -EINVAL;

        tmp = vmalloc_array(num_counters, sizeof(*tmp));
        if (!tmp)
                return -ENOMEM;

        t = find_table_lock(net, name, &ret, &ebt_mutex);
        if (!t)
                goto free_tmp;

        if (num_counters != t->private->nentries) {
                ret = -EINVAL;
                goto unlock_mutex;
        }

        if (copy_from_user(tmp, counters,
                           array_size(num_counters, sizeof(*counters)))) {
                ret = -EFAULT;
                goto unlock_mutex;
        }

        /* we want an atomic add of the counters */
        write_lock_bh(&t->lock);

        /* we add to the counters of the first cpu */
        for (i = 0; i < num_counters; i++)
                ADD_COUNTER(t->private->counters[i], tmp[i].bcnt, tmp[i].pcnt);

        write_unlock_bh(&t->lock);
        ret = 0;
unlock_mutex:
        mutex_unlock(&ebt_mutex);
free_tmp:
        vfree(tmp);
        return ret;
}

static int update_counters(struct net *net, sockptr_t arg, unsigned int len)
{
        struct ebt_replace hlp;

        if (len < sizeof(hlp))
                return -EINVAL;
        if (copy_from_sockptr(&hlp, arg, sizeof(hlp)))
                return -EFAULT;

        if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter))
                return -EINVAL;

        return do_update_counters(net, hlp.name, hlp.counters,
                                  hlp.num_counters, len);
}

static inline int ebt_obj_to_user(char __user *um, const char *_name,
                                  const char *data, int entrysize,
                                  int usersize, int datasize, u8 revision)
{
        char name[EBT_EXTENSION_MAXNAMELEN] = {0};

        /* ebtables expects 31 bytes long names but xt_match names are 29 bytes
         * long. Copy 29 bytes and fill remaining bytes with zeroes.
         */
        strscpy(name, _name, sizeof(name));
        if (copy_to_user(um, name, EBT_EXTENSION_MAXNAMELEN) ||
            put_user(revision, (u8 __user *)(um + EBT_EXTENSION_MAXNAMELEN)) ||
            put_user(datasize, (int __user *)(um + EBT_EXTENSION_MAXNAMELEN + 1)) ||
            xt_data_to_user(um + entrysize, data, usersize, datasize,
                            XT_ALIGN(datasize)))
                return -EFAULT;

        return 0;
}

static inline int ebt_match_to_user(const struct ebt_entry_match *m,
                                    const char *base, char __user *ubase)
{
        return ebt_obj_to_user(ubase + ((char *)m - base),
                               m->u.match->name, m->data, sizeof(*m),
                               m->u.match->usersize, m->match_size,
                               m->u.match->revision);
}

static inline int ebt_watcher_to_user(const struct ebt_entry_watcher *w,
                                      const char *base, char __user *ubase)
{
        return ebt_obj_to_user(ubase + ((char *)w - base),
                               w->u.watcher->name, w->data, sizeof(*w),
                               w->u.watcher->usersize, w->watcher_size,
                               w->u.watcher->revision);
}

static inline int ebt_entry_to_user(struct ebt_entry *e, const char *base,
                                    char __user *ubase)
{
        int ret;
        char __user *hlp;
        const struct ebt_entry_target *t;

        if (e->bitmask == 0) {
                /* special case !EBT_ENTRY_OR_ENTRIES */
                if (copy_to_user(ubase + ((char *)e - base), e,
                                 sizeof(struct ebt_entries)))
                        return -EFAULT;
                return 0;
        }

        if (copy_to_user(ubase + ((char *)e - base), e, sizeof(*e)))
                return -EFAULT;

        hlp = ubase + (((char *)e + e->target_offset) - base);
        t = ebt_get_target_c(e);

        ret = EBT_MATCH_ITERATE(e, ebt_match_to_user, base, ubase);
        if (ret != 0)
                return ret;
        ret = EBT_WATCHER_ITERATE(e, ebt_watcher_to_user, base, ubase);
        if (ret != 0)
                return ret;
        ret = ebt_obj_to_user(hlp, t->u.target->name, t->data, sizeof(*t),
                              t->u.target->usersize, t->target_size,
                              t->u.target->revision);
        if (ret != 0)
                return ret;

        return 0;
}

static int copy_counters_to_user(struct ebt_table *t,
                                 const struct ebt_counter *oldcounters,
                                 void __user *user, unsigned int num_counters,
                                 unsigned int nentries)
{
        struct ebt_counter *counterstmp;
        int ret = 0;

        /* userspace might not need the counters */
        if (num_counters == 0)
                return 0;

        if (num_counters != nentries)
                return -EINVAL;

        counterstmp = vmalloc_array(nentries, sizeof(*counterstmp));
        if (!counterstmp)
                return -ENOMEM;

        write_lock_bh(&t->lock);
        get_counters(oldcounters, counterstmp, nentries);
        write_unlock_bh(&t->lock);

        if (copy_to_user(user, counterstmp,
            array_size(nentries, sizeof(struct ebt_counter))))
                ret = -EFAULT;
        vfree(counterstmp);
        return ret;
}

/* called with ebt_mutex locked */
static int copy_everything_to_user(struct ebt_table *t, void __user *user,
                                   const int *len, int cmd)
{
        struct ebt_replace tmp;
        const struct ebt_counter *oldcounters;
        unsigned int entries_size, nentries;
        int ret;
        char *entries;

        if (cmd == EBT_SO_GET_ENTRIES) {
                entries_size = t->private->entries_size;
                nentries = t->private->nentries;
                entries = t->private->entries;
                oldcounters = t->private->counters;
        } else {
                entries_size = t->table->entries_size;
                nentries = t->table->nentries;
                entries = t->table->entries;
                oldcounters = t->table->counters;
        }

        if (copy_from_user(&tmp, user, sizeof(tmp)))
                return -EFAULT;

        if (*len != sizeof(struct ebt_replace) + entries_size +
           (tmp.num_counters ? nentries * sizeof(struct ebt_counter) : 0))
                return -EINVAL;

        if (tmp.nentries != nentries)
                return -EINVAL;

        if (tmp.entries_size != entries_size)
                return -EINVAL;

        ret = copy_counters_to_user(t, oldcounters, tmp.counters,
                                        tmp.num_counters, nentries);
        if (ret)
                return ret;

        /* set the match/watcher/target names right */
        return EBT_ENTRY_ITERATE(entries, entries_size,
           ebt_entry_to_user, entries, tmp.entries);
}

#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
/* 32 bit-userspace compatibility definitions. */
struct compat_ebt_replace {
        char name[EBT_TABLE_MAXNAMELEN];
        compat_uint_t valid_hooks;
        compat_uint_t nentries;
        compat_uint_t entries_size;
        /* start of the chains */
        compat_uptr_t hook_entry[NF_BR_NUMHOOKS];
        /* nr of counters userspace expects back */
        compat_uint_t num_counters;
        /* where the kernel will put the old counters. */
        compat_uptr_t counters;
        compat_uptr_t entries;
};

/* struct ebt_entry_match, _target and _watcher have same layout */
struct compat_ebt_entry_mwt {
        union {
                struct {
                        char name[EBT_EXTENSION_MAXNAMELEN];
                        u8 revision;
                };
                compat_uptr_t ptr;
        } u;
        compat_uint_t match_size;
        compat_uint_t data[] __aligned(__alignof__(struct compat_ebt_replace));
};

/* account for possible padding between match_size and ->data */
static int ebt_compat_entry_padsize(void)
{
        BUILD_BUG_ON(sizeof(struct ebt_entry_match) <
                        sizeof(struct compat_ebt_entry_mwt));
        return (int) sizeof(struct ebt_entry_match) -
                        sizeof(struct compat_ebt_entry_mwt);
}

static int ebt_compat_match_offset(const struct xt_match *match,
                                   unsigned int userlen)
{
        /* ebt_among needs special handling. The kernel .matchsize is
         * set to -1 at registration time; at runtime an EBT_ALIGN()ed
         * value is expected.
         * Example: userspace sends 4500, ebt_among.c wants 4504.
         */
        if (unlikely(match->matchsize == -1))
                return XT_ALIGN(userlen) - COMPAT_XT_ALIGN(userlen);
        return xt_compat_match_offset(match);
}

static int compat_match_to_user(struct ebt_entry_match *m, void __user **dstptr,
                                unsigned int *size)
{
        const struct xt_match *match = m->u.match;
        struct compat_ebt_entry_mwt __user *cm = *dstptr;
        int off = ebt_compat_match_offset(match, m->match_size);
        compat_uint_t msize = m->match_size - off;

        if (WARN_ON(off >= m->match_size))
                return -EINVAL;

        if (copy_to_user(cm->u.name, match->name, strlen(match->name) + 1) ||
            put_user(match->revision, &cm->u.revision) ||
            put_user(msize, &cm->match_size))
                return -EFAULT;

        if (match->compat_to_user) {
                if (match->compat_to_user(cm->data, m->data))
                        return -EFAULT;
        } else {
                if (xt_data_to_user(cm->data, m->data, match->usersize, msize,
                                    COMPAT_XT_ALIGN(msize)))
                        return -EFAULT;
        }

        *size -= ebt_compat_entry_padsize() + off;
        *dstptr = cm->data;
        *dstptr += msize;
        return 0;
}

static int compat_target_to_user(struct ebt_entry_target *t,
                                 void __user **dstptr,
                                 unsigned int *size)
{
        const struct xt_target *target = t->u.target;
        struct compat_ebt_entry_mwt __user *cm = *dstptr;
        int off = xt_compat_target_offset(target);
        compat_uint_t tsize = t->target_size - off;

        if (WARN_ON(off >= t->target_size))
                return -EINVAL;

        if (copy_to_user(cm->u.name, target->name, strlen(target->name) + 1) ||
            put_user(target->revision, &cm->u.revision) ||
            put_user(tsize, &cm->match_size))
                return -EFAULT;

        if (target->compat_to_user) {
                if (target->compat_to_user(cm->data, t->data))
                        return -EFAULT;
        } else {
                if (xt_data_to_user(cm->data, t->data, target->usersize, tsize,
                                    COMPAT_XT_ALIGN(tsize)))
                        return -EFAULT;
        }

        *size -= ebt_compat_entry_padsize() + off;
        *dstptr = cm->data;
        *dstptr += tsize;
        return 0;
}

static int compat_watcher_to_user(struct ebt_entry_watcher *w,
                                  void __user **dstptr,
                                  unsigned int *size)
{
        return compat_target_to_user((struct ebt_entry_target *)w,
                                                        dstptr, size);
}

static int compat_copy_entry_to_user(struct ebt_entry *e, void __user **dstptr,
                                unsigned int *size)
{
        struct ebt_entry_target *t;
        struct ebt_entry __user *ce;
        u32 watchers_offset, target_offset, next_offset;
        compat_uint_t origsize;
        int ret;

        if (e->bitmask == 0) {
                if (*size < sizeof(struct ebt_entries))
                        return -EINVAL;
                if (copy_to_user(*dstptr, e, sizeof(struct ebt_entries)))
                        return -EFAULT;

                *dstptr += sizeof(struct ebt_entries);
                *size -= sizeof(struct ebt_entries);
                return 0;
        }

        if (*size < sizeof(*ce))
                return -EINVAL;

        ce = *dstptr;
        if (copy_to_user(ce, e, sizeof(*ce)))
                return -EFAULT;

        origsize = *size;
        *dstptr += sizeof(*ce);

        ret = EBT_MATCH_ITERATE(e, compat_match_to_user, dstptr, size);
        if (ret)
                return ret;
        watchers_offset = e->watchers_offset - (origsize - *size);

        ret = EBT_WATCHER_ITERATE(e, compat_watcher_to_user, dstptr, size);
        if (ret)
                return ret;
        target_offset = e->target_offset - (origsize - *size);

        t = ebt_get_target(e);

        ret = compat_target_to_user(t, dstptr, size);
        if (ret)
                return ret;
        next_offset = e->next_offset - (origsize - *size);

        if (put_user(watchers_offset, &ce->watchers_offset) ||
            put_user(target_offset, &ce->target_offset) ||
            put_user(next_offset, &ce->next_offset))
                return -EFAULT;

        *size -= sizeof(*ce);
        return 0;
}

static int compat_calc_match(struct ebt_entry_match *m, int *off)
{
        *off += ebt_compat_match_offset(m->u.match, m->match_size);
        *off += ebt_compat_entry_padsize();
        return 0;
}

static int compat_calc_watcher(struct ebt_entry_watcher *w, int *off)
{
        *off += xt_compat_target_offset(w->u.watcher);
        *off += ebt_compat_entry_padsize();
        return 0;
}

static int compat_calc_entry(const struct ebt_entry *e,
                             const struct ebt_table_info *info,
                             const void *base,
                             struct compat_ebt_replace *newinfo)
{
        const struct ebt_entry_target *t;
        unsigned int entry_offset;
        int off, ret, i;

        if (e->bitmask == 0)
                return 0;

        off = 0;
        entry_offset = (void *)e - base;

        EBT_MATCH_ITERATE(e, compat_calc_match, &off);
        EBT_WATCHER_ITERATE(e, compat_calc_watcher, &off);

        t = ebt_get_target_c(e);

        off += xt_compat_target_offset(t->u.target);
        off += ebt_compat_entry_padsize();

        newinfo->entries_size -= off;

        ret = xt_compat_add_offset(NFPROTO_BRIDGE, entry_offset, off);
        if (ret)
                return ret;

        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                const void *hookptr = info->hook_entry[i];
                if (info->hook_entry[i] &&
                    (e < (struct ebt_entry *)(base - hookptr))) {
                        newinfo->hook_entry[i] -= off;
                        pr_debug("0x%08X -> 0x%08X\n",
                                        newinfo->hook_entry[i] + off,
                                        newinfo->hook_entry[i]);
                }
        }

        return 0;
}

static int ebt_compat_init_offsets(unsigned int number)
{
        if (number > INT_MAX)
                return -EINVAL;

        /* also count the base chain policies */
        number += NF_BR_NUMHOOKS;

        return xt_compat_init_offsets(NFPROTO_BRIDGE, number);
}

static int compat_table_info(const struct ebt_table_info *info,
                             struct compat_ebt_replace *newinfo)
{
        unsigned int size = info->entries_size;
        const void *entries = info->entries;
        int ret;

        newinfo->entries_size = size;
        ret = ebt_compat_init_offsets(info->nentries);
        if (ret)
                return ret;

        return EBT_ENTRY_ITERATE(entries, size, compat_calc_entry, info,
                                                        entries, newinfo);
}

static int compat_copy_everything_to_user(struct ebt_table *t,
                                          void __user *user, int *len, int cmd)
{
        struct compat_ebt_replace repl, tmp;
        struct ebt_counter *oldcounters;
        struct ebt_table_info tinfo;
        int ret;
        void __user *pos;

        memset(&tinfo, 0, sizeof(tinfo));

        if (cmd == EBT_SO_GET_ENTRIES) {
                tinfo.entries_size = t->private->entries_size;
                tinfo.nentries = t->private->nentries;
                tinfo.entries = t->private->entries;
                oldcounters = t->private->counters;
        } else {
                tinfo.entries_size = t->table->entries_size;
                tinfo.nentries = t->table->nentries;
                tinfo.entries = t->table->entries;
                oldcounters = t->table->counters;
        }

        if (copy_from_user(&tmp, user, sizeof(tmp)))
                return -EFAULT;

        if (tmp.nentries != tinfo.nentries ||
           (tmp.num_counters && tmp.num_counters != tinfo.nentries))
                return -EINVAL;

        memcpy(&repl, &tmp, sizeof(repl));
        if (cmd == EBT_SO_GET_ENTRIES)
                ret = compat_table_info(t->private, &repl);
        else
                ret = compat_table_info(&tinfo, &repl);
        if (ret)
                return ret;

        if (*len != sizeof(tmp) + repl.entries_size +
           (tmp.num_counters? tinfo.nentries * sizeof(struct ebt_counter): 0)) {
                pr_err("wrong size: *len %d, entries_size %u, replsz %d\n",
                                *len, tinfo.entries_size, repl.entries_size);
                return -EINVAL;
        }

        /* userspace might not need the counters */
        ret = copy_counters_to_user(t, oldcounters, compat_ptr(tmp.counters),
                                        tmp.num_counters, tinfo.nentries);
        if (ret)
                return ret;

        pos = compat_ptr(tmp.entries);
        return EBT_ENTRY_ITERATE(tinfo.entries, tinfo.entries_size,
                        compat_copy_entry_to_user, &pos, &tmp.entries_size);
}

struct ebt_entries_buf_state {
        char *buf_kern_start;        /* kernel buffer to copy (translated) data to */
        u32 buf_kern_len;        /* total size of kernel buffer */
        u32 buf_kern_offset;        /* amount of data copied so far */
        u32 buf_user_offset;        /* read position in userspace buffer */
};

static int ebt_buf_count(struct ebt_entries_buf_state *state, unsigned int sz)
{
        state->buf_kern_offset += sz;
        return state->buf_kern_offset >= sz ? 0 : -EINVAL;
}

static int ebt_buf_add(struct ebt_entries_buf_state *state,
                       const void *data, unsigned int sz)
{
        if (state->buf_kern_start == NULL)
                goto count_only;

        if (WARN_ON(state->buf_kern_offset + sz > state->buf_kern_len))
                return -EINVAL;

        memcpy(state->buf_kern_start + state->buf_kern_offset, data, sz);

 count_only:
        state->buf_user_offset += sz;
        return ebt_buf_count(state, sz);
}

static int ebt_buf_add_pad(struct ebt_entries_buf_state *state, unsigned int sz)
{
        char *b = state->buf_kern_start;

        if (WARN_ON(b && state->buf_kern_offset > state->buf_kern_len))
                return -EINVAL;

        if (b != NULL && sz > 0)
                memset(b + state->buf_kern_offset, 0, sz);
        /* do not adjust ->buf_user_offset here, we added kernel-side padding */
        return ebt_buf_count(state, sz);
}

enum compat_mwt {
        EBT_COMPAT_MATCH,
        EBT_COMPAT_WATCHER,
        EBT_COMPAT_TARGET,
};

static int compat_mtw_from_user(const struct compat_ebt_entry_mwt *mwt,
                                enum compat_mwt compat_mwt,
                                struct ebt_entries_buf_state *state,
                                const unsigned char *base)
{
        char name[EBT_EXTENSION_MAXNAMELEN];
        struct xt_match *match;
        struct xt_target *wt;
        void *dst = NULL;
        int off, pad = 0;
        unsigned int size_kern, match_size = mwt->match_size;

        if (strscpy(name, mwt->u.name, sizeof(name)) < 0)
                return -EINVAL;

        if (state->buf_kern_start)
                dst = state->buf_kern_start + state->buf_kern_offset;

        switch (compat_mwt) {
        case EBT_COMPAT_MATCH:
                match = xt_request_find_match(NFPROTO_BRIDGE, name,
                                              mwt->u.revision);
                if (IS_ERR(match))
                        return PTR_ERR(match);

                off = ebt_compat_match_offset(match, match_size);
                if (dst) {
                        if (match->compat_from_user)
                                match->compat_from_user(dst, mwt->data);
                        else
                                memcpy(dst, mwt->data, match_size);
                }

                size_kern = match->matchsize;
                if (unlikely(size_kern == -1))
                        size_kern = match_size;
                module_put(match->me);
                break;
        case EBT_COMPAT_WATCHER:
        case EBT_COMPAT_TARGET:
                wt = xt_request_find_target(NFPROTO_BRIDGE, name,
                                            mwt->u.revision);
                if (IS_ERR(wt))
                        return PTR_ERR(wt);
                off = xt_compat_target_offset(wt);

                if (dst) {
                        if (wt->compat_from_user)
                                wt->compat_from_user(dst, mwt->data);
                        else
                                memcpy(dst, mwt->data, match_size);
                }

                size_kern = wt->targetsize;
                module_put(wt->me);
                break;

        default:
                return -EINVAL;
        }

        state->buf_kern_offset += match_size + off;
        state->buf_user_offset += match_size;
        pad = XT_ALIGN(size_kern) - size_kern;

        if (pad > 0 && dst) {
                if (WARN_ON(state->buf_kern_len <= pad))
                        return -EINVAL;
                if (WARN_ON(state->buf_kern_offset - (match_size + off) + size_kern > state->buf_kern_len - pad))
                        return -EINVAL;
                memset(dst + size_kern, 0, pad);
        }
        return off + match_size;
}

/* return size of all matches, watchers or target, including necessary
 * alignment and padding.
 */
static int ebt_size_mwt(const struct compat_ebt_entry_mwt *match32,
                        unsigned int size_left, enum compat_mwt type,
                        struct ebt_entries_buf_state *state, const void *base)
{
        const char *buf = (const char *)match32;
        int growth = 0;

        if (size_left == 0)
                return 0;

        do {
                struct ebt_entry_match *match_kern;
                int ret;

                if (size_left < sizeof(*match32))
                        return -EINVAL;

                match_kern = (struct ebt_entry_match *) state->buf_kern_start;
                if (match_kern) {
                        char *tmp;
                        tmp = state->buf_kern_start + state->buf_kern_offset;
                        match_kern = (struct ebt_entry_match *) tmp;
                }
                ret = ebt_buf_add(state, buf, sizeof(*match32));
                if (ret < 0)
                        return ret;
                size_left -= sizeof(*match32);

                /* add padding before match->data (if any) */
                ret = ebt_buf_add_pad(state, ebt_compat_entry_padsize());
                if (ret < 0)
                        return ret;

                if (match32->match_size > size_left)
                        return -EINVAL;

                size_left -= match32->match_size;

                ret = compat_mtw_from_user(match32, type, state, base);
                if (ret < 0)
                        return ret;

                if (WARN_ON(ret < match32->match_size))
                        return -EINVAL;
                growth += ret - match32->match_size;
                growth += ebt_compat_entry_padsize();

                buf += sizeof(*match32);
                buf += match32->match_size;

                if (match_kern)
                        match_kern->match_size = ret;

                match32 = (struct compat_ebt_entry_mwt *) buf;
        } while (size_left);

        return growth;
}

/* called for all ebt_entry structures. */
static int size_entry_mwt(const struct ebt_entry *entry, const unsigned char *base,
                          unsigned int *total,
                          struct ebt_entries_buf_state *state)
{
        unsigned int i, j, startoff, next_expected_off, new_offset = 0;
        /* stores match/watchers/targets & offset of next struct ebt_entry: */
        unsigned int offsets[4];
        unsigned int *offsets_update = NULL;
        int ret;
        char *buf_start;

        if (*total < sizeof(struct ebt_entries))
                return -EINVAL;

        if (!entry->bitmask) {
                *total -= sizeof(struct ebt_entries);
                return ebt_buf_add(state, entry, sizeof(struct ebt_entries));
        }
        if (*total < sizeof(*entry) || entry->next_offset < sizeof(*entry))
                return -EINVAL;

        startoff = state->buf_user_offset;
        /* pull in most part of ebt_entry, it does not need to be changed. */
        ret = ebt_buf_add(state, entry,
                        offsetof(struct ebt_entry, watchers_offset));
        if (ret < 0)
                return ret;

        offsets[0] = sizeof(struct ebt_entry); /* matches come first */
        memcpy(&offsets[1], &entry->offsets, sizeof(entry->offsets));

        if (state->buf_kern_start) {
                buf_start = state->buf_kern_start + state->buf_kern_offset;
                offsets_update = (unsigned int *) buf_start;
        }
        ret = ebt_buf_add(state, &offsets[1],
                        sizeof(offsets) - sizeof(offsets[0]));
        if (ret < 0)
                return ret;
        buf_start = (char *) entry;
        /* 0: matches offset, always follows ebt_entry.
         * 1: watchers offset, from ebt_entry structure
         * 2: target offset, from ebt_entry structure
         * 3: next ebt_entry offset, from ebt_entry structure
         *
         * offsets are relative to beginning of struct ebt_entry (i.e., 0).
         */
        for (i = 0; i < 4 ; ++i) {
                if (offsets[i] > *total)
                        return -EINVAL;

                if (i < 3 && offsets[i] == *total)
                        return -EINVAL;

                if (i == 0)
                        continue;
                if (offsets[i-1] > offsets[i])
                        return -EINVAL;
        }

        for (i = 0, j = 1 ; j < 4 ; j++, i++) {
                struct compat_ebt_entry_mwt *match32;
                unsigned int size;
                char *buf = buf_start + offsets[i];

                if (offsets[i] > offsets[j])
                        return -EINVAL;

                match32 = (struct compat_ebt_entry_mwt *) buf;
                size = offsets[j] - offsets[i];
                ret = ebt_size_mwt(match32, size, i, state, base);
                if (ret < 0)
                        return ret;
                new_offset += ret;
                if (offsets_update && new_offset) {
                        pr_debug("change offset %d to %d\n",
                                offsets_update[i], offsets[j] + new_offset);
                        offsets_update[i] = offsets[j] + new_offset;
                }
        }

        if (state->buf_kern_start == NULL) {
                unsigned int offset = buf_start - (char *) base;

                ret = xt_compat_add_offset(NFPROTO_BRIDGE, offset, new_offset);
                if (ret < 0)
                        return ret;
        }

        next_expected_off = state->buf_user_offset - startoff;
        if (next_expected_off != entry->next_offset)
                return -EINVAL;

        if (*total < entry->next_offset)
                return -EINVAL;
        *total -= entry->next_offset;
        return 0;
}

/* repl->entries_size is the size of the ebt_entry blob in userspace.
 * It might need more memory when copied to a 64 bit kernel in case
 * userspace is 32-bit. So, first task: find out how much memory is needed.
 *
 * Called before validation is performed.
 */
static int compat_copy_entries(unsigned char *data, unsigned int size_user,
                                struct ebt_entries_buf_state *state)
{
        unsigned int size_remaining = size_user;
        int ret;

        ret = EBT_ENTRY_ITERATE(data, size_user, size_entry_mwt, data,
                                        &size_remaining, state);
        if (ret < 0)
                return ret;

        if (size_remaining)
                return -EINVAL;

        return state->buf_kern_offset;
}


static int compat_copy_ebt_replace_from_user(struct ebt_replace *repl,
                                             sockptr_t arg, unsigned int len)
{
        struct compat_ebt_replace tmp;
        int i;

        if (len < sizeof(tmp))
                return -EINVAL;

        if (copy_from_sockptr(&tmp, arg, sizeof(tmp)))
                return -EFAULT;

        if (len != sizeof(tmp) + tmp.entries_size)
                return -EINVAL;

        if (tmp.entries_size == 0)
                return -EINVAL;

        if (tmp.nentries >= ((INT_MAX - sizeof(struct ebt_table_info)) /
                        NR_CPUS - SMP_CACHE_BYTES) / sizeof(struct ebt_counter))
                return -ENOMEM;
        if (tmp.num_counters >= INT_MAX / sizeof(struct ebt_counter))
                return -ENOMEM;

        memcpy(repl, &tmp, offsetof(struct ebt_replace, hook_entry));

        /* starting with hook_entry, 32 vs. 64 bit structures are different */
        for (i = 0; i < NF_BR_NUMHOOKS; i++)
                repl->hook_entry[i] = compat_ptr(tmp.hook_entry[i]);

        repl->num_counters = tmp.num_counters;
        repl->counters = compat_ptr(tmp.counters);
        repl->entries = compat_ptr(tmp.entries);
        return 0;
}

static int compat_do_replace(struct net *net, sockptr_t arg, unsigned int len)
{
        int ret, i, countersize, size64;
        struct ebt_table_info *newinfo;
        struct ebt_replace tmp;
        struct ebt_entries_buf_state state;
        void *entries_tmp;

        ret = compat_copy_ebt_replace_from_user(&tmp, arg, len);
        if (ret) {
                /* try real handler in case userland supplied needed padding */
                if (ret == -EINVAL && do_replace(net, arg, len) == 0)
                        ret = 0;
                return ret;
        }

        countersize = COUNTER_OFFSET(tmp.nentries) * nr_cpu_ids;
        newinfo = vmalloc(sizeof(*newinfo) + countersize);
        if (!newinfo)
                return -ENOMEM;

        if (countersize)
                memset(newinfo->counters, 0, countersize);

        memset(&state, 0, sizeof(state));

        newinfo->entries = vmalloc(tmp.entries_size);
        if (!newinfo->entries) {
                ret = -ENOMEM;
                goto free_newinfo;
        }
        if (copy_from_user(
           newinfo->entries, tmp.entries, tmp.entries_size) != 0) {
                ret = -EFAULT;
                goto free_entries;
        }

        entries_tmp = newinfo->entries;

        xt_compat_lock(NFPROTO_BRIDGE);

        ret = ebt_compat_init_offsets(tmp.nentries);
        if (ret < 0)
                goto out_unlock;

        ret = compat_copy_entries(entries_tmp, tmp.entries_size, &state);
        if (ret < 0)
                goto out_unlock;

        pr_debug("tmp.entries_size %d, kern off %d, user off %d delta %d\n",
                tmp.entries_size, state.buf_kern_offset, state.buf_user_offset,
                xt_compat_calc_jump(NFPROTO_BRIDGE, tmp.entries_size));

        size64 = ret;
        newinfo->entries = vmalloc(size64);
        if (!newinfo->entries) {
                vfree(entries_tmp);
                ret = -ENOMEM;
                goto out_unlock;
        }

        memset(&state, 0, sizeof(state));
        state.buf_kern_start = newinfo->entries;
        state.buf_kern_len = size64;

        ret = compat_copy_entries(entries_tmp, tmp.entries_size, &state);
        if (WARN_ON(ret < 0)) {
                vfree(entries_tmp);
                goto out_unlock;
        }

        vfree(entries_tmp);
        tmp.entries_size = size64;

        for (i = 0; i < NF_BR_NUMHOOKS; i++) {
                char __user *usrptr;
                if (tmp.hook_entry[i]) {
                        unsigned int delta;
                        usrptr = (char __user *) tmp.hook_entry[i];
                        delta = usrptr - tmp.entries;
                        usrptr += xt_compat_calc_jump(NFPROTO_BRIDGE, delta);
                        tmp.hook_entry[i] = (struct ebt_entries __user *)usrptr;
                }
        }

        xt_compat_flush_offsets(NFPROTO_BRIDGE);
        xt_compat_unlock(NFPROTO_BRIDGE);

        ret = do_replace_finish(net, &tmp, newinfo);
        if (ret == 0)
                return ret;
free_entries:
        vfree(newinfo->entries);
free_newinfo:
        vfree(newinfo);
        return ret;
out_unlock:
        xt_compat_flush_offsets(NFPROTO_BRIDGE);
        xt_compat_unlock(NFPROTO_BRIDGE);
        goto free_entries;
}

static int compat_update_counters(struct net *net, sockptr_t arg,
                                  unsigned int len)
{
        struct compat_ebt_replace hlp;

        if (len < sizeof(hlp))
                return -EINVAL;
        if (copy_from_sockptr(&hlp, arg, sizeof(hlp)))
                return -EFAULT;

        /* try real handler in case userland supplied needed padding */
        if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter))
                return update_counters(net, arg, len);

        return do_update_counters(net, hlp.name, compat_ptr(hlp.counters),
                                  hlp.num_counters, len);
}

static int compat_do_ebt_get_ctl(struct sock *sk, int cmd,
                void __user *user, int *len)
{
        int ret;
        struct compat_ebt_replace tmp;
        struct ebt_table *t;
        struct net *net = sock_net(sk);

        if ((cmd == EBT_SO_GET_INFO || cmd == EBT_SO_GET_INIT_INFO) &&
            *len != sizeof(struct compat_ebt_replace))
                return -EINVAL;

        if (copy_from_user(&tmp, user, sizeof(tmp)))
                return -EFAULT;

        tmp.name[sizeof(tmp.name) - 1] = '\0';

        t = find_table_lock(net, tmp.name, &ret, &ebt_mutex);
        if (!t)
                return ret;

        xt_compat_lock(NFPROTO_BRIDGE);
        switch (cmd) {
        case EBT_SO_GET_INFO:
                tmp.nentries = t->private->nentries;
                ret = compat_table_info(t->private, &tmp);
                if (ret)
                        goto out;
                tmp.valid_hooks = t->valid_hooks;

                if (copy_to_user(user, &tmp, *len) != 0) {
                        ret = -EFAULT;
                        break;
                }
                ret = 0;
                break;
        case EBT_SO_GET_INIT_INFO:
                tmp.nentries = t->table->nentries;
                tmp.entries_size = t->table->entries_size;
                tmp.valid_hooks = t->table->valid_hooks;

                if (copy_to_user(user, &tmp, *len) != 0) {
                        ret = -EFAULT;
                        break;
                }
                ret = 0;
                break;
        case EBT_SO_GET_ENTRIES:
        case EBT_SO_GET_INIT_ENTRIES:
                /* try real handler first in case of userland-side padding.
                 * in case we are dealing with an 'ordinary' 32 bit binary
                 * without 64bit compatibility padding, this will fail right
                 * after copy_from_user when the *len argument is validated.
                 *
                 * the compat_ variant needs to do one pass over the kernel
                 * data set to adjust for size differences before it the check.
                 */
                if (copy_everything_to_user(t, user, len, cmd) == 0)
                        ret = 0;
                else
                        ret = compat_copy_everything_to_user(t, user, len, cmd);
                break;
        default:
                ret = -EINVAL;
        }
 out:
        xt_compat_flush_offsets(NFPROTO_BRIDGE);
        xt_compat_unlock(NFPROTO_BRIDGE);
        mutex_unlock(&ebt_mutex);
        return ret;
}
#endif

static int do_ebt_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
{
        struct net *net = sock_net(sk);
        struct ebt_replace tmp;
        struct ebt_table *t;
        int ret;

        if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
        /* try real handler in case userland supplied needed padding */
        if (in_compat_syscall() &&
            ((cmd != EBT_SO_GET_INFO && cmd != EBT_SO_GET_INIT_INFO) ||
             *len != sizeof(tmp)))
                return compat_do_ebt_get_ctl(sk, cmd, user, len);
#endif

        if (copy_from_user(&tmp, user, sizeof(tmp)))
                return -EFAULT;

        tmp.name[sizeof(tmp.name) - 1] = '\0';

        t = find_table_lock(net, tmp.name, &ret, &ebt_mutex);
        if (!t)
                return ret;

        switch (cmd) {
        case EBT_SO_GET_INFO:
        case EBT_SO_GET_INIT_INFO:
                if (*len != sizeof(struct ebt_replace)) {
                        ret = -EINVAL;
                        mutex_unlock(&ebt_mutex);
                        break;
                }
                if (cmd == EBT_SO_GET_INFO) {
                        tmp.nentries = t->private->nentries;
                        tmp.entries_size = t->private->entries_size;
                        tmp.valid_hooks = t->valid_hooks;
                } else {
                        tmp.nentries = t->table->nentries;
                        tmp.entries_size = t->table->entries_size;
                        tmp.valid_hooks = t->table->valid_hooks;
                }
                mutex_unlock(&ebt_mutex);
                if (copy_to_user(user, &tmp, *len) != 0) {
                        ret = -EFAULT;
                        break;
                }
                ret = 0;
                break;

        case EBT_SO_GET_ENTRIES:
        case EBT_SO_GET_INIT_ENTRIES:
                ret = copy_everything_to_user(t, user, len, cmd);
                mutex_unlock(&ebt_mutex);
                break;

        default:
                mutex_unlock(&ebt_mutex);
                ret = -EINVAL;
        }

        return ret;
}

static int do_ebt_set_ctl(struct sock *sk, int cmd, sockptr_t arg,
                unsigned int len)
{
        struct net *net = sock_net(sk);
        int ret;

        if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        switch (cmd) {
        case EBT_SO_SET_ENTRIES:
#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
                if (in_compat_syscall())
                        ret = compat_do_replace(net, arg, len);
                else
#endif
                        ret = do_replace(net, arg, len);
                break;
        case EBT_SO_SET_COUNTERS:
#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
                if (in_compat_syscall())
                        ret = compat_update_counters(net, arg, len);
                else
#endif
                        ret = update_counters(net, arg, len);
                break;
        default:
                ret = -EINVAL;
        }
        return ret;
}

static struct nf_sockopt_ops ebt_sockopts = {
        .pf                = PF_INET,
        .set_optmin        = EBT_BASE_CTL,
        .set_optmax        = EBT_SO_SET_MAX + 1,
        .set                = do_ebt_set_ctl,
        .get_optmin        = EBT_BASE_CTL,
        .get_optmax        = EBT_SO_GET_MAX + 1,
        .get                = do_ebt_get_ctl,
        .owner                = THIS_MODULE,
};

static int __net_init ebt_pernet_init(struct net *net)
{
        struct ebt_pernet *ebt_net = net_generic(net, ebt_pernet_id);

        INIT_LIST_HEAD(&ebt_net->tables);
        return 0;
}

static struct pernet_operations ebt_net_ops = {
        .init = ebt_pernet_init,
        .id   = &ebt_pernet_id,
        .size = sizeof(struct ebt_pernet),
};

static int __init ebtables_init(void)
{
        int ret;

        ret = xt_register_target(&ebt_standard_target);
        if (ret < 0)
                return ret;
        ret = nf_register_sockopt(&ebt_sockopts);
        if (ret < 0) {
                xt_unregister_target(&ebt_standard_target);
                return ret;
        }

        ret = register_pernet_subsys(&ebt_net_ops);
        if (ret < 0) {
                nf_unregister_sockopt(&ebt_sockopts);
                xt_unregister_target(&ebt_standard_target);
                return ret;
        }

        return 0;
}

static void ebtables_fini(void)
{
        nf_unregister_sockopt(&ebt_sockopts);
        xt_unregister_target(&ebt_standard_target);
        unregister_pernet_subsys(&ebt_net_ops);
}

EXPORT_SYMBOL(ebt_register_table);
EXPORT_SYMBOL(ebt_unregister_table);
EXPORT_SYMBOL(ebt_do_table);
module_init(ebtables_init);
module_exit(ebtables_fini);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("ebtables legacy core");




















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    6 































































































    5 





    5 





































    6 


    5 









    6 












































    6 


    6 






    4 


















    4 

























    6 


















    5 









    5 
    1 








    6 

















    6 




    6 



















    6 







    6 








    6 
    4 
    4 








    6 
    6 
    6 





















    5 
    5 






    6 







































    6 



























































    6 







    6 



    5 


    5 















    5 










    6 




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/ext4/namei.c
 *
 * Copyright (C) 1992, 1993, 1994, 1995
 * Remy Card (card@masi.ibp.fr)
 * Laboratoire MASI - Institut Blaise Pascal
 * Universite Pierre et Marie Curie (Paris VI)
 *
 *  from
 *
 *  linux/fs/minix/namei.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  Big-endian to little-endian byte-swapping/bitmaps by
 *        David S. Miller (davem@caip.rutgers.edu), 1995
 *  Directory entry file type support and forward compatibility hooks
 *        for B-tree directories by Theodore Ts'o (tytso@mit.edu), 1998
 *  Hash Tree Directory indexing (c)
 *        Daniel Phillips, 2001
 *  Hash Tree Directory indexing porting
 *        Christopher Li, 2002
 *  Hash Tree Directory indexing cleanup
 *        Theodore Ts'o, 2002
 */

#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/time.h>
#include <linux/fcntl.h>
#include <linux/stat.h>
#include <linux/string.h>
#include <linux/quotaops.h>
#include <linux/buffer_head.h>
#include <linux/bio.h>
#include <linux/iversion.h>
#include <linux/unicode.h>
#include "ext4.h"
#include "ext4_jbd2.h"

#include "xattr.h"
#include "acl.h"

#include <trace/events/ext4.h>
/*
 * define how far ahead to read directories while searching them.
 */
#define NAMEI_RA_CHUNKS  2
#define NAMEI_RA_BLOCKS  4
#define NAMEI_RA_SIZE             (NAMEI_RA_CHUNKS * NAMEI_RA_BLOCKS)

static struct buffer_head *ext4_append(handle_t *handle,
                                        struct inode *inode,
                                        ext4_lblk_t *block)
{
        struct ext4_map_blocks map;
        struct buffer_head *bh;
        int err;

        if (unlikely(EXT4_SB(inode->i_sb)->s_max_dir_size_kb &&
                     ((inode->i_size >> 10) >=
                      EXT4_SB(inode->i_sb)->s_max_dir_size_kb)))
                return ERR_PTR(-ENOSPC);

        *block = inode->i_size >> inode->i_sb->s_blocksize_bits;
        map.m_lblk = *block;
        map.m_len = 1;

        /*
         * We're appending new directory block. Make sure the block is not
         * allocated yet, otherwise we will end up corrupting the
         * directory.
         */
        err = ext4_map_blocks(NULL, inode, &map, 0);
        if (err < 0)
                return ERR_PTR(err);
        if (err) {
                EXT4_ERROR_INODE(inode, "Logical block already allocated");
                return ERR_PTR(-EFSCORRUPTED);
        }

        bh = ext4_bread(handle, inode, *block, EXT4_GET_BLOCKS_CREATE);
        if (IS_ERR(bh))
                return bh;
        inode->i_size += inode->i_sb->s_blocksize;
        EXT4_I(inode)->i_disksize = inode->i_size;
        err = ext4_mark_inode_dirty(handle, inode);
        if (err)
                goto out;
        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto out;
        return bh;

out:
        brelse(bh);
        ext4_std_error(inode->i_sb, err);
        return ERR_PTR(err);
}

static int ext4_dx_csum_verify(struct inode *inode,
                               struct ext4_dir_entry *dirent);

/*
 * Hints to ext4_read_dirblock regarding whether we expect a directory
 * block being read to be an index block, or a block containing
 * directory entries (and if the latter, whether it was found via a
 * logical block in an htree index block).  This is used to control
 * what sort of sanity checkinig ext4_read_dirblock() will do on the
 * directory block read from the storage device.  EITHER will means
 * the caller doesn't know what kind of directory block will be read,
 * so no specific verification will be done.
 */
typedef enum {
        EITHER, INDEX, DIRENT, DIRENT_HTREE
} dirblock_type_t;

#define ext4_read_dirblock(inode, block, type) \
        __ext4_read_dirblock((inode), (block), (type), __func__, __LINE__)

static struct buffer_head *__ext4_read_dirblock(struct inode *inode,
                                                ext4_lblk_t block,
                                                dirblock_type_t type,
                                                const char *func,
                                                unsigned int line)
{
        struct buffer_head *bh;
        struct ext4_dir_entry *dirent;
        int is_dx_block = 0;

        if (block >= inode->i_size >> inode->i_blkbits) {
                ext4_error_inode(inode, func, line, block,
                       "Attempting to read directory block (%u) that is past i_size (%llu)",
                       block, inode->i_size);
                return ERR_PTR(-EFSCORRUPTED);
        }

        if (ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_EIO))
                bh = ERR_PTR(-EIO);
        else
                bh = ext4_bread(NULL, inode, block, 0);
        if (IS_ERR(bh)) {
                __ext4_warning(inode->i_sb, func, line,
                               "inode #%lu: lblock %lu: comm %s: "
                               "error %ld reading directory block",
                               inode->i_ino, (unsigned long)block,
                               current->comm, PTR_ERR(bh));

                return bh;
        }
        /* The first directory block must not be a hole. */
        if (!bh && (type == INDEX || type == DIRENT_HTREE || block == 0)) {
                ext4_error_inode(inode, func, line, block,
                                 "Directory hole found for htree %s block %u",
                                 (type == INDEX) ? "index" : "leaf", block);
                return ERR_PTR(-EFSCORRUPTED);
        }
        if (!bh)
                return NULL;
        dirent = (struct ext4_dir_entry *) bh->b_data;
        /* Determine whether or not we have an index block */
        if (is_dx(inode)) {
                if (block == 0)
                        is_dx_block = 1;
                else if (ext4_rec_len_from_disk(dirent->rec_len,
                                                inode->i_sb->s_blocksize) ==
                         inode->i_sb->s_blocksize)
                        is_dx_block = 1;
        }
        if (!is_dx_block && type == INDEX) {
                ext4_error_inode(inode, func, line, block,
                       "directory leaf block found instead of index block");
                brelse(bh);
                return ERR_PTR(-EFSCORRUPTED);
        }
        if (!ext4_has_feature_metadata_csum(inode->i_sb) ||
            buffer_verified(bh))
                return bh;

        /*
         * An empty leaf block can get mistaken for a index block; for
         * this reason, we can only check the index checksum when the
         * caller is sure it should be an index block.
         */
        if (is_dx_block && type == INDEX) {
                if (ext4_dx_csum_verify(inode, dirent) &&
                    !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC))
                        set_buffer_verified(bh);
                else {
                        ext4_error_inode_err(inode, func, line, block,
                                             EFSBADCRC,
                                             "Directory index failed checksum");
                        brelse(bh);
                        return ERR_PTR(-EFSBADCRC);
                }
        }
        if (!is_dx_block) {
                if (ext4_dirblock_csum_verify(inode, bh) &&
                    !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC))
                        set_buffer_verified(bh);
                else {
                        ext4_error_inode_err(inode, func, line, block,
                                             EFSBADCRC,
                                             "Directory block failed checksum");
                        brelse(bh);
                        return ERR_PTR(-EFSBADCRC);
                }
        }
        return bh;
}

#ifdef DX_DEBUG
#define dxtrace(command) command
#else
#define dxtrace(command)
#endif

struct fake_dirent
{
        __le32 inode;
        __le16 rec_len;
        u8 name_len;
        u8 file_type;
};

struct dx_countlimit
{
        __le16 limit;
        __le16 count;
};

struct dx_entry
{
        __le32 hash;
        __le32 block;
};

/*
 * dx_root_info is laid out so that if it should somehow get overlaid by a
 * dirent the two low bits of the hash version will be zero.  Therefore, the
 * hash version mod 4 should never be 0.  Sincerely, the paranoia department.
 */

struct dx_root
{
        struct fake_dirent dot;
        char dot_name[4];
        struct fake_dirent dotdot;
        char dotdot_name[4];
        struct dx_root_info
        {
                __le32 reserved_zero;
                u8 hash_version;
                u8 info_length; /* 8 */
                u8 indirect_levels;
                u8 unused_flags;
        }
        info;
        struct dx_entry        entries[];
};

struct dx_node
{
        struct fake_dirent fake;
        struct dx_entry        entries[];
};


struct dx_frame
{
        struct buffer_head *bh;
        struct dx_entry *entries;
        struct dx_entry *at;
};

struct dx_map_entry
{
        u32 hash;
        u16 offs;
        u16 size;
};

/*
 * This goes at the end of each htree block.
 */
struct dx_tail {
        u32 dt_reserved;
        __le32 dt_checksum;        /* crc32c(uuid+inum+dirblock) */
};

static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
                struct ext4_filename *fname,
                struct ext4_dir_entry_2 **res_dir);
static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir, struct inode *inode);

/* checksumming functions */
void ext4_initialize_dirent_tail(struct buffer_head *bh,
                                 unsigned int blocksize)
{
        struct ext4_dir_entry_tail *t = EXT4_DIRENT_TAIL(bh->b_data, blocksize);

        memset(t, 0, sizeof(struct ext4_dir_entry_tail));
        t->det_rec_len = ext4_rec_len_to_disk(
                        sizeof(struct ext4_dir_entry_tail), blocksize);
        t->det_reserved_ft = EXT4_FT_DIR_CSUM;
}

/* Walk through a dirent block to find a checksum "dirent" at the tail */
static struct ext4_dir_entry_tail *get_dirent_tail(struct inode *inode,
                                                   struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;
        int blocksize = EXT4_BLOCK_SIZE(inode->i_sb);

#ifdef PARANOID
        struct ext4_dir_entry *d, *top;

        d = (struct ext4_dir_entry *)bh->b_data;
        top = (struct ext4_dir_entry *)(bh->b_data +
                (blocksize - sizeof(struct ext4_dir_entry_tail)));
        while (d < top && ext4_rec_len_from_disk(d->rec_len, blocksize))
                d = (struct ext4_dir_entry *)(((void *)d) +
                    ext4_rec_len_from_disk(d->rec_len, blocksize));

        if (d != top)
                return NULL;

        t = (struct ext4_dir_entry_tail *)d;
#else
        t = EXT4_DIRENT_TAIL(bh->b_data, EXT4_BLOCK_SIZE(inode->i_sb));
#endif

        if (t->det_reserved_zero1 ||
            (ext4_rec_len_from_disk(t->det_rec_len, blocksize) !=
             sizeof(struct ext4_dir_entry_tail)) ||
            t->det_reserved_zero2 ||
            t->det_reserved_ft != EXT4_FT_DIR_CSUM)
                return NULL;

        return t;
}

static __le32 ext4_dirblock_csum(struct inode *inode, void *dirent, int size)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        __u32 csum;

        csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size);
        return cpu_to_le32(csum);
}

#define warn_no_space_for_csum(inode)                                        \
        __warn_no_space_for_csum((inode), __func__, __LINE__)

static void __warn_no_space_for_csum(struct inode *inode, const char *func,
                                     unsigned int line)
{
        __ext4_warning_inode(inode, func, line,
                "No space for directory leaf checksum. Please run e2fsck -D.");
}

int ext4_dirblock_csum_verify(struct inode *inode, struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return 1;

        t = get_dirent_tail(inode, bh);
        if (!t) {
                warn_no_space_for_csum(inode);
                return 0;
        }

        if (t->det_checksum != ext4_dirblock_csum(inode, bh->b_data,
                                                  (char *)t - bh->b_data))
                return 0;

        return 1;
}

static void ext4_dirblock_csum_set(struct inode *inode,
                                 struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return;

        t = get_dirent_tail(inode, bh);
        if (!t) {
                warn_no_space_for_csum(inode);
                return;
        }

        t->det_checksum = ext4_dirblock_csum(inode, bh->b_data,
                                             (char *)t - bh->b_data);
}

int ext4_handle_dirty_dirblock(handle_t *handle,
                               struct inode *inode,
                               struct buffer_head *bh)
{
        ext4_dirblock_csum_set(inode, bh);
        return ext4_handle_dirty_metadata(handle, inode, bh);
}

static struct dx_countlimit *get_dx_countlimit(struct inode *inode,
                                               struct ext4_dir_entry *dirent,
                                               int *offset)
{
        struct ext4_dir_entry *dp;
        struct dx_root_info *root;
        int count_offset;
        int blocksize = EXT4_BLOCK_SIZE(inode->i_sb);
        unsigned int rlen = ext4_rec_len_from_disk(dirent->rec_len, blocksize);

        if (rlen == blocksize)
                count_offset = 8;
        else if (rlen == 12) {
                dp = (struct ext4_dir_entry *)(((void *)dirent) + 12);
                if (ext4_rec_len_from_disk(dp->rec_len, blocksize) != blocksize - 12)
                        return NULL;
                root = (struct dx_root_info *)(((void *)dp + 12));
                if (root->reserved_zero ||
                    root->info_length != sizeof(struct dx_root_info))
                        return NULL;
                count_offset = 32;
        } else
                return NULL;

        if (offset)
                *offset = count_offset;
        return (struct dx_countlimit *)(((void *)dirent) + count_offset);
}

static __le32 ext4_dx_csum(struct inode *inode, struct ext4_dir_entry *dirent,
                           int count_offset, int count, struct dx_tail *t)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        __u32 csum;
        int size;
        __u32 dummy_csum = 0;
        int offset = offsetof(struct dx_tail, dt_checksum);

        size = count_offset + (count * sizeof(struct dx_entry));
        csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size);
        csum = ext4_chksum(csum, (__u8 *)t, offset);
        csum = ext4_chksum(csum, (__u8 *)&dummy_csum, sizeof(dummy_csum));

        return cpu_to_le32(csum);
}

static int ext4_dx_csum_verify(struct inode *inode,
                               struct ext4_dir_entry *dirent)
{
        struct dx_countlimit *c;
        struct dx_tail *t;
        int count_offset, limit, count;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return 1;

        c = get_dx_countlimit(inode, dirent, &count_offset);
        if (!c) {
                EXT4_ERROR_INODE(inode, "dir seems corrupt?  Run e2fsck -D.");
                return 0;
        }
        limit = le16_to_cpu(c->limit);
        count = le16_to_cpu(c->count);
        if (count_offset + (limit * sizeof(struct dx_entry)) >
            EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) {
                warn_no_space_for_csum(inode);
                return 0;
        }
        t = (struct dx_tail *)(((struct dx_entry *)c) + limit);

        if (t->dt_checksum != ext4_dx_csum(inode, dirent, count_offset,
                                            count, t))
                return 0;
        return 1;
}

static void ext4_dx_csum_set(struct inode *inode, struct ext4_dir_entry *dirent)
{
        struct dx_countlimit *c;
        struct dx_tail *t;
        int count_offset, limit, count;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return;

        c = get_dx_countlimit(inode, dirent, &count_offset);
        if (!c) {
                EXT4_ERROR_INODE(inode, "dir seems corrupt?  Run e2fsck -D.");
                return;
        }
        limit = le16_to_cpu(c->limit);
        count = le16_to_cpu(c->count);
        if (count_offset + (limit * sizeof(struct dx_entry)) >
            EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) {
                warn_no_space_for_csum(inode);
                return;
        }
        t = (struct dx_tail *)(((struct dx_entry *)c) + limit);

        t->dt_checksum = ext4_dx_csum(inode, dirent, count_offset, count, t);
}

static inline int ext4_handle_dirty_dx_node(handle_t *handle,
                                            struct inode *inode,
                                            struct buffer_head *bh)
{
        ext4_dx_csum_set(inode, (struct ext4_dir_entry *)bh->b_data);
        return ext4_handle_dirty_metadata(handle, inode, bh);
}

/*
 * p is at least 6 bytes before the end of page
 */
static inline struct ext4_dir_entry_2 *
ext4_next_entry(struct ext4_dir_entry_2 *p, unsigned long blocksize)
{
        return (struct ext4_dir_entry_2 *)((char *)p +
                ext4_rec_len_from_disk(p->rec_len, blocksize));
}

/*
 * Future: use high four bits of block for coalesce-on-delete flags
 * Mask them off for now.
 */

static inline ext4_lblk_t dx_get_block(struct dx_entry *entry)
{
        return le32_to_cpu(entry->block) & 0x0fffffff;
}

static inline void dx_set_block(struct dx_entry *entry, ext4_lblk_t value)
{
        entry->block = cpu_to_le32(value);
}

static inline unsigned dx_get_hash(struct dx_entry *entry)
{
        return le32_to_cpu(entry->hash);
}

static inline void dx_set_hash(struct dx_entry *entry, unsigned value)
{
        entry->hash = cpu_to_le32(value);
}

static inline unsigned dx_get_count(struct dx_entry *entries)
{
        return le16_to_cpu(((struct dx_countlimit *) entries)->count);
}

static inline unsigned dx_get_limit(struct dx_entry *entries)
{
        return le16_to_cpu(((struct dx_countlimit *) entries)->limit);
}

static inline void dx_set_count(struct dx_entry *entries, unsigned value)
{
        ((struct dx_countlimit *) entries)->count = cpu_to_le16(value);
}

static inline void dx_set_limit(struct dx_entry *entries, unsigned value)
{
        ((struct dx_countlimit *) entries)->limit = cpu_to_le16(value);
}

static inline unsigned dx_root_limit(struct inode *dir, unsigned infosize)
{
        unsigned int entry_space = dir->i_sb->s_blocksize -
                        ext4_dir_rec_len(1, NULL) -
                        ext4_dir_rec_len(2, NULL) - infosize;

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                entry_space -= sizeof(struct dx_tail);
        return entry_space / sizeof(struct dx_entry);
}

static inline unsigned dx_node_limit(struct inode *dir)
{
        unsigned int entry_space = dir->i_sb->s_blocksize -
                        ext4_dir_rec_len(0, dir);

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                entry_space -= sizeof(struct dx_tail);
        return entry_space / sizeof(struct dx_entry);
}

/*
 * Debug
 */
#ifdef DX_DEBUG
static void dx_show_index(char * label, struct dx_entry *entries)
{
        int i, n = dx_get_count (entries);
        printk(KERN_DEBUG "%s index", label);
        for (i = 0; i < n; i++) {
                printk(KERN_CONT " %x->%lu",
                       i ? dx_get_hash(entries + i) : 0,
                       (unsigned long)dx_get_block(entries + i));
        }
        printk(KERN_CONT "\n");
}

struct stats
{
        unsigned names;
        unsigned space;
        unsigned bcount;
};

static struct stats dx_show_leaf(struct inode *dir,
                                struct dx_hash_info *hinfo,
                                struct ext4_dir_entry_2 *de,
                                int size, int show_names)
{
        unsigned names = 0, space = 0;
        char *base = (char *) de;
        struct dx_hash_info h = *hinfo;

        printk("names: ");
        while ((char *) de < base + size)
        {
                if (de->inode)
                {
                        if (show_names)
                        {
#ifdef CONFIG_FS_ENCRYPTION
                                int len;
                                char *name;
                                struct fscrypt_str fname_crypto_str =
                                        FSTR_INIT(NULL, 0);
                                int res = 0;

                                name  = de->name;
                                len = de->name_len;
                                if (!IS_ENCRYPTED(dir)) {
                                        /* Directory is not encrypted */
                                        (void) ext4fs_dirhash(dir, de->name,
                                                de->name_len, &h);
                                        printk("%*.s:(U)%x.%u ", len,
                                               name, h.hash,
                                               (unsigned) ((char *) de
                                                           - base));
                                } else {
                                        struct fscrypt_str de_name =
                                                FSTR_INIT(name, len);

                                        /* Directory is encrypted */
                                        res = fscrypt_fname_alloc_buffer(
                                                len, &fname_crypto_str);
                                        if (res)
                                                printk(KERN_WARNING "Error "
                                                        "allocating crypto "
                                                        "buffer--skipping "
                                                        "crypto\n");
                                        res = fscrypt_fname_disk_to_usr(dir,
                                                0, 0, &de_name,
                                                &fname_crypto_str);
                                        if (res) {
                                                printk(KERN_WARNING "Error "
                                                        "converting filename "
                                                        "from disk to usr"
                                                        "\n");
                                                name = "??";
                                                len = 2;
                                        } else {
                                                name = fname_crypto_str.name;
                                                len = fname_crypto_str.len;
                                        }
                                        if (IS_CASEFOLDED(dir))
                                                h.hash = EXT4_DIRENT_HASH(de);
                                        else
                                                (void) ext4fs_dirhash(dir,
                                                        de->name,
                                                        de->name_len, &h);
                                        printk("%*.s:(E)%x.%u ", len, name,
                                               h.hash, (unsigned) ((char *) de
                                                                   - base));
                                        fscrypt_fname_free_buffer(
                                                        &fname_crypto_str);
                                }
#else
                                int len = de->name_len;
                                char *name = de->name;
                                (void) ext4fs_dirhash(dir, de->name,
                                                      de->name_len, &h);
                                printk("%*.s:%x.%u ", len, name, h.hash,
                                       (unsigned) ((char *) de - base));
#endif
                        }
                        space += ext4_dir_rec_len(de->name_len, dir);
                        names++;
                }
                de = ext4_next_entry(de, size);
        }
        printk(KERN_CONT "(%i)\n", names);
        return (struct stats) { names, space, 1 };
}

struct stats dx_show_entries(struct dx_hash_info *hinfo, struct inode *dir,
                             struct dx_entry *entries, int levels)
{
        unsigned blocksize = dir->i_sb->s_blocksize;
        unsigned count = dx_get_count(entries), names = 0, space = 0, i;
        unsigned bcount = 0;
        struct buffer_head *bh;
        printk("%i indexed blocks...\n", count);
        for (i = 0; i < count; i++, entries++)
        {
                ext4_lblk_t block = dx_get_block(entries);
                ext4_lblk_t hash  = i ? dx_get_hash(entries): 0;
                u32 range = i < count - 1? (dx_get_hash(entries + 1) - hash): ~hash;
                struct stats stats;
                printk("%s%3u:%03u hash %8x/%8x ",levels?"":"   ", i, block, hash, range);
                bh = ext4_bread(NULL,dir, block, 0);
                if (!bh || IS_ERR(bh))
                        continue;
                stats = levels?
                   dx_show_entries(hinfo, dir, ((struct dx_node *) bh->b_data)->entries, levels - 1):
                   dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *)
                        bh->b_data, blocksize, 0);
                names += stats.names;
                space += stats.space;
                bcount += stats.bcount;
                brelse(bh);
        }
        if (bcount)
                printk(KERN_DEBUG "%snames %u, fullness %u (%u%%)\n",
                       levels ? "" : "   ", names, space/bcount,
                       (space/bcount)*100/blocksize);
        return (struct stats) { names, space, bcount};
}

/*
 * Linear search cross check
 */
static inline void htree_rep_invariant_check(struct dx_entry *at,
                                             struct dx_entry *target,
                                             u32 hash, unsigned int n)
{
        while (n--) {
                dxtrace(printk(KERN_CONT ","));
                if (dx_get_hash(++at) > hash) {
                        at--;
                        break;
                }
        }
        ASSERT(at == target - 1);
}
#else /* DX_DEBUG */
static inline void htree_rep_invariant_check(struct dx_entry *at,
                                             struct dx_entry *target,
                                             u32 hash, unsigned int n)
{
}
#endif /* DX_DEBUG */

/*
 * Probe for a directory leaf block to search.
 *
 * dx_probe can return ERR_BAD_DX_DIR, which means there was a format
 * error in the directory index, and the caller should fall back to
 * searching the directory normally.  The callers of dx_probe **MUST**
 * check for this error code, and make sure it never gets reflected
 * back to userspace.
 */
static struct dx_frame *
dx_probe(struct ext4_filename *fname, struct inode *dir,
         struct dx_hash_info *hinfo, struct dx_frame *frame_in)
{
        unsigned count, indirect, level, i;
        struct dx_entry *at, *entries, *p, *q, *m;
        struct dx_root *root;
        struct dx_frame *frame = frame_in;
        struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR);
        u32 hash;
        ext4_lblk_t block;
        ext4_lblk_t blocks[EXT4_HTREE_LEVEL];

        memset(frame_in, 0, EXT4_HTREE_LEVEL * sizeof(frame_in[0]));
        frame->bh = ext4_read_dirblock(dir, 0, INDEX);
        if (IS_ERR(frame->bh))
                return (struct dx_frame *) frame->bh;

        root = (struct dx_root *) frame->bh->b_data;
        if (root->info.hash_version != DX_HASH_TEA &&
            root->info.hash_version != DX_HASH_HALF_MD4 &&
            root->info.hash_version != DX_HASH_LEGACY &&
            root->info.hash_version != DX_HASH_SIPHASH) {
                ext4_warning_inode(dir, "Unrecognised inode hash code %u",
                                   root->info.hash_version);
                goto fail;
        }
        if (ext4_hash_in_dirent(dir)) {
                if (root->info.hash_version != DX_HASH_SIPHASH) {
                        ext4_warning_inode(dir,
                                "Hash in dirent, but hash is not SIPHASH");
                        goto fail;
                }
        } else {
                if (root->info.hash_version == DX_HASH_SIPHASH) {
                        ext4_warning_inode(dir,
                                "Hash code is SIPHASH, but hash not in dirent");
                        goto fail;
                }
        }
        if (fname)
                hinfo = &fname->hinfo;
        hinfo->hash_version = root->info.hash_version;
        if (hinfo->hash_version <= DX_HASH_TEA)
                hinfo->hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned;
        hinfo->seed = EXT4_SB(dir->i_sb)->s_hash_seed;
        /* hash is already computed for encrypted casefolded directory */
        if (fname && fname_name(fname) &&
            !(IS_ENCRYPTED(dir) && IS_CASEFOLDED(dir))) {
                int ret = ext4fs_dirhash(dir, fname_name(fname),
                                         fname_len(fname), hinfo);
                if (ret < 0) {
                        ret_err = ERR_PTR(ret);
                        goto fail;
                }
        }
        hash = hinfo->hash;

        if (root->info.unused_flags & 1) {
                ext4_warning_inode(dir, "Unimplemented hash flags: %#06x",
                                   root->info.unused_flags);
                goto fail;
        }

        indirect = root->info.indirect_levels;
        if (indirect >= ext4_dir_htree_level(dir->i_sb)) {
                ext4_warning(dir->i_sb,
                             "Directory (ino: %lu) htree depth %#06x exceed"
                             "supported value", dir->i_ino,
                             ext4_dir_htree_level(dir->i_sb));
                if (ext4_dir_htree_level(dir->i_sb) < EXT4_HTREE_LEVEL) {
                        ext4_warning(dir->i_sb, "Enable large directory "
                                                "feature to access it");
                }
                goto fail;
        }

        entries = (struct dx_entry *)(((char *)&root->info) +
                                      root->info.info_length);

        if (dx_get_limit(entries) != dx_root_limit(dir,
                                                   root->info.info_length)) {
                ext4_warning_inode(dir, "dx entry: limit %u != root limit %u",
                                   dx_get_limit(entries),
                                   dx_root_limit(dir, root->info.info_length));
                goto fail;
        }

        dxtrace(printk("Look up %x", hash));
        level = 0;
        blocks[0] = 0;
        while (1) {
                count = dx_get_count(entries);
                if (!count || count > dx_get_limit(entries)) {
                        ext4_warning_inode(dir,
                                           "dx entry: count %u beyond limit %u",
                                           count, dx_get_limit(entries));
                        goto fail;
                }

                p = entries + 1;
                q = entries + count - 1;
                while (p <= q) {
                        m = p + (q - p) / 2;
                        dxtrace(printk(KERN_CONT "."));
                        if (dx_get_hash(m) > hash)
                                q = m - 1;
                        else
                                p = m + 1;
                }

                htree_rep_invariant_check(entries, p, hash, count - 1);

                at = p - 1;
                dxtrace(printk(KERN_CONT " %x->%u\n",
                               at == entries ? 0 : dx_get_hash(at),
                               dx_get_block(at)));
                frame->entries = entries;
                frame->at = at;

                block = dx_get_block(at);
                for (i = 0; i <= level; i++) {
                        if (blocks[i] == block) {
                                ext4_warning_inode(dir,
                                        "dx entry: tree cycle block %u points back to block %u",
                                        blocks[level], block);
                                goto fail;
                        }
                }
                if (++level > indirect)
                        return frame;
                blocks[level] = block;
                frame++;
                frame->bh = ext4_read_dirblock(dir, block, INDEX);
                if (IS_ERR(frame->bh)) {
                        ret_err = (struct dx_frame *) frame->bh;
                        frame->bh = NULL;
                        goto fail;
                }

                entries = ((struct dx_node *) frame->bh->b_data)->entries;

                if (dx_get_limit(entries) != dx_node_limit(dir)) {
                        ext4_warning_inode(dir,
                                "dx entry: limit %u != node limit %u",
                                dx_get_limit(entries), dx_node_limit(dir));
                        goto fail;
                }
        }
fail:
        while (frame >= frame_in) {
                brelse(frame->bh);
                frame--;
        }

        if (ret_err == ERR_PTR(ERR_BAD_DX_DIR))
                ext4_warning_inode(dir,
                        "Corrupt directory, running e2fsck is recommended");
        return ret_err;
}

static void dx_release(struct dx_frame *frames)
{
        struct dx_root_info *info;
        int i;
        unsigned int indirect_levels;

        if (frames[0].bh == NULL)
                return;

        info = &((struct dx_root *)frames[0].bh->b_data)->info;
        /* save local copy, "info" may be freed after brelse() */
        indirect_levels = info->indirect_levels;
        for (i = 0; i <= indirect_levels; i++) {
                if (frames[i].bh == NULL)
                        break;
                brelse(frames[i].bh);
                frames[i].bh = NULL;
        }
}

/*
 * This function increments the frame pointer to search the next leaf
 * block, and reads in the necessary intervening nodes if the search
 * should be necessary.  Whether or not the search is necessary is
 * controlled by the hash parameter.  If the hash value is even, then
 * the search is only continued if the next block starts with that
 * hash value.  This is used if we are searching for a specific file.
 *
 * If the hash value is HASH_NB_ALWAYS, then always go to the next block.
 *
 * This function returns 1 if the caller should continue to search,
 * or 0 if it should not.  If there is an error reading one of the
 * index blocks, it will a negative error code.
 *
 * If start_hash is non-null, it will be filled in with the starting
 * hash of the next page.
 */
static int ext4_htree_next_block(struct inode *dir, __u32 hash,
                                 struct dx_frame *frame,
                                 struct dx_frame *frames,
                                 __u32 *start_hash)
{
        struct dx_frame *p;
        struct buffer_head *bh;
        int num_frames = 0;
        __u32 bhash;

        p = frame;
        /*
         * Find the next leaf page by incrementing the frame pointer.
         * If we run out of entries in the interior node, loop around and
         * increment pointer in the parent node.  When we break out of
         * this loop, num_frames indicates the number of interior
         * nodes need to be read.
         */
        while (1) {
                if (++(p->at) < p->entries + dx_get_count(p->entries))
                        break;
                if (p == frames)
                        return 0;
                num_frames++;
                p--;
        }

        /*
         * If the hash is 1, then continue only if the next page has a
         * continuation hash of any value.  This is used for readdir
         * handling.  Otherwise, check to see if the hash matches the
         * desired continuation hash.  If it doesn't, return since
         * there's no point to read in the successive index pages.
         */
        bhash = dx_get_hash(p->at);
        if (start_hash)
                *start_hash = bhash;
        if ((hash & 1) == 0) {
                if ((bhash & ~1) != hash)
                        return 0;
        }
        /*
         * If the hash is HASH_NB_ALWAYS, we always go to the next
         * block so no check is necessary
         */
        while (num_frames--) {
                bh = ext4_read_dirblock(dir, dx_get_block(p->at), INDEX);
                if (IS_ERR(bh))
                        return PTR_ERR(bh);
                p++;
                brelse(p->bh);
                p->bh = bh;
                p->at = p->entries = ((struct dx_node *) bh->b_data)->entries;
        }
        return 1;
}


/*
 * This function fills a red-black tree with information from a
 * directory block.  It returns the number directory entries loaded
 * into the tree.  If there is an error it is returned in err.
 */
static int htree_dirblock_to_tree(struct file *dir_file,
                                  struct inode *dir, ext4_lblk_t block,
                                  struct dx_hash_info *hinfo,
                                  __u32 start_hash, __u32 start_minor_hash)
{
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de, *top;
        int err = 0, count = 0;
        struct fscrypt_str fname_crypto_str = FSTR_INIT(NULL, 0), tmp_str;
        int csum = ext4_has_feature_metadata_csum(dir->i_sb);

        dxtrace(printk(KERN_INFO "In htree dirblock_to_tree: block %lu\n",
                                                        (unsigned long)block));
        bh = ext4_read_dirblock(dir, block, DIRENT_HTREE);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        de = (struct ext4_dir_entry_2 *) bh->b_data;
        /* csum entries are not larger in the casefolded encrypted case */
        top = (struct ext4_dir_entry_2 *) ((char *) de +
                                           dir->i_sb->s_blocksize -
                                           ext4_dir_rec_len(0,
                                                           csum ? NULL : dir));
        /* Check if the directory is encrypted */
        if (IS_ENCRYPTED(dir)) {
                err = fscrypt_prepare_readdir(dir);
                if (err < 0) {
                        brelse(bh);
                        return err;
                }
                err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN,
                                                 &fname_crypto_str);
                if (err < 0) {
                        brelse(bh);
                        return err;
                }
        }

        for (; de < top; de = ext4_next_entry(de, dir->i_sb->s_blocksize)) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                bh->b_data, bh->b_size,
                                (block<<EXT4_BLOCK_SIZE_BITS(dir->i_sb))
                                         + ((char *)de - bh->b_data))) {
                        /* silently ignore the rest of the block */
                        break;
                }
                if (ext4_hash_in_dirent(dir)) {
                        if (de->name_len && de->inode) {
                                hinfo->hash = EXT4_DIRENT_HASH(de);
                                hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de);
                        } else {
                                hinfo->hash = 0;
                                hinfo->minor_hash = 0;
                        }
                } else {
                        err = ext4fs_dirhash(dir, de->name,
                                             de->name_len, hinfo);
                        if (err < 0) {
                                count = err;
                                goto errout;
                        }
                }
                if ((hinfo->hash < start_hash) ||
                    ((hinfo->hash == start_hash) &&
                     (hinfo->minor_hash < start_minor_hash)))
                        continue;
                if (de->inode == 0)
                        continue;
                if (!IS_ENCRYPTED(dir)) {
                        tmp_str.name = de->name;
                        tmp_str.len = de->name_len;
                        err = ext4_htree_store_dirent(dir_file,
                                   hinfo->hash, hinfo->minor_hash, de,
                                   &tmp_str);
                } else {
                        int save_len = fname_crypto_str.len;
                        struct fscrypt_str de_name = FSTR_INIT(de->name,
                                                                de->name_len);

                        /* Directory is encrypted */
                        err = fscrypt_fname_disk_to_usr(dir, hinfo->hash,
                                        hinfo->minor_hash, &de_name,
                                        &fname_crypto_str);
                        if (err) {
                                count = err;
                                goto errout;
                        }
                        err = ext4_htree_store_dirent(dir_file,
                                   hinfo->hash, hinfo->minor_hash, de,
                                        &fname_crypto_str);
                        fname_crypto_str.len = save_len;
                }
                if (err != 0) {
                        count = err;
                        goto errout;
                }
                count++;
        }
errout:
        brelse(bh);
        fscrypt_fname_free_buffer(&fname_crypto_str);
        return count;
}


/*
 * This function fills a red-black tree with information from a
 * directory.  We start scanning the directory in hash order, starting
 * at start_hash and start_minor_hash.
 *
 * This function returns the number of entries inserted into the tree,
 * or a negative error code.
 */
int ext4_htree_fill_tree(struct file *dir_file, __u32 start_hash,
                         __u32 start_minor_hash, __u32 *next_hash)
{
        struct dx_hash_info hinfo;
        struct ext4_dir_entry_2 *de;
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct inode *dir;
        ext4_lblk_t block;
        int count = 0;
        int ret, err;
        __u32 hashval;
        struct fscrypt_str tmp_str;

        dxtrace(printk(KERN_DEBUG "In htree_fill_tree, start hash: %x:%x\n",
                       start_hash, start_minor_hash));
        dir = file_inode(dir_file);
        if (!(ext4_test_inode_flag(dir, EXT4_INODE_INDEX))) {
                if (ext4_hash_in_dirent(dir))
                        hinfo.hash_version = DX_HASH_SIPHASH;
                else
                        hinfo.hash_version =
                                        EXT4_SB(dir->i_sb)->s_def_hash_version;
                if (hinfo.hash_version <= DX_HASH_TEA)
                        hinfo.hash_version +=
                                EXT4_SB(dir->i_sb)->s_hash_unsigned;
                hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed;
                if (ext4_has_inline_data(dir)) {
                        int has_inline_data = 1;
                        count = ext4_inlinedir_to_tree(dir_file, dir, 0,
                                                       &hinfo, start_hash,
                                                       start_minor_hash,
                                                       &has_inline_data);
                        if (has_inline_data) {
                                *next_hash = ~0;
                                return count;
                        }
                }
                count = htree_dirblock_to_tree(dir_file, dir, 0, &hinfo,
                                               start_hash, start_minor_hash);
                *next_hash = ~0;
                return count;
        }
        hinfo.hash = start_hash;
        hinfo.minor_hash = 0;
        frame = dx_probe(NULL, dir, &hinfo, frames);
        if (IS_ERR(frame))
                return PTR_ERR(frame);

        /* Add '.' and '..' from the htree header */
        if (!start_hash && !start_minor_hash) {
                de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
                tmp_str.name = de->name;
                tmp_str.len = de->name_len;
                err = ext4_htree_store_dirent(dir_file, 0, 0,
                                              de, &tmp_str);
                if (err != 0)
                        goto errout;
                count++;
        }
        if (start_hash < 2 || (start_hash ==2 && start_minor_hash==0)) {
                de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
                de = ext4_next_entry(de, dir->i_sb->s_blocksize);
                tmp_str.name = de->name;
                tmp_str.len = de->name_len;
                err = ext4_htree_store_dirent(dir_file, 2, 0,
                                              de, &tmp_str);
                if (err != 0)
                        goto errout;
                count++;
        }

        while (1) {
                if (fatal_signal_pending(current)) {
                        err = -ERESTARTSYS;
                        goto errout;
                }
                cond_resched();
                block = dx_get_block(frame->at);
                ret = htree_dirblock_to_tree(dir_file, dir, block, &hinfo,
                                             start_hash, start_minor_hash);
                if (ret < 0) {
                        err = ret;
                        goto errout;
                }
                count += ret;
                hashval = ~0;
                ret = ext4_htree_next_block(dir, HASH_NB_ALWAYS,
                                            frame, frames, &hashval);
                *next_hash = hashval;
                if (ret < 0) {
                        err = ret;
                        goto errout;
                }
                /*
                 * Stop if:  (a) there are no more entries, or
                 * (b) we have inserted at least one entry and the
                 * next hash value is not a continuation
                 */
                if ((ret == 0) ||
                    (count && ((hashval & 1) == 0)))
                        break;
        }
        dx_release(frames);
        dxtrace(printk(KERN_DEBUG "Fill tree: returned %d entries, "
                       "next hash: %x\n", count, *next_hash));
        return count;
errout:
        dx_release(frames);
        return (err);
}

static inline int search_dirblock(struct buffer_head *bh,
                                  struct inode *dir,
                                  struct ext4_filename *fname,
                                  unsigned int offset,
                                  struct ext4_dir_entry_2 **res_dir)
{
        return ext4_search_dir(bh, bh->b_data, dir->i_sb->s_blocksize, dir,
                               fname, offset, res_dir);
}

/*
 * Directory block splitting, compacting
 */

/*
 * Create map of hash values, offsets, and sizes, stored at end of block.
 * Returns number of entries mapped.
 */
static int dx_make_map(struct inode *dir, struct buffer_head *bh,
                       struct dx_hash_info *hinfo,
                       struct dx_map_entry *map_tail)
{
        int count = 0;
        struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)bh->b_data;
        unsigned int buflen = bh->b_size;
        char *base = bh->b_data;
        struct dx_hash_info h = *hinfo;
        int blocksize = EXT4_BLOCK_SIZE(dir->i_sb);

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                buflen -= sizeof(struct ext4_dir_entry_tail);

        while ((char *) de < base + buflen) {
                if (ext4_check_dir_entry(dir, NULL, de, bh, base, buflen,
                                         ((char *)de) - base))
                        return -EFSCORRUPTED;
                if (de->name_len && de->inode) {
                        if (ext4_hash_in_dirent(dir))
                                h.hash = EXT4_DIRENT_HASH(de);
                        else {
                                int err = ext4fs_dirhash(dir, de->name,
                                                     de->name_len, &h);
                                if (err < 0)
                                        return err;
                        }
                        map_tail--;
                        map_tail->hash = h.hash;
                        map_tail->offs = ((char *) de - base)>>2;
                        map_tail->size = ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize);
                        count++;
                        cond_resched();
                }
                de = ext4_next_entry(de, blocksize);
        }
        return count;
}

/* Sort map by hash value */
static void dx_sort_map (struct dx_map_entry *map, unsigned count)
{
        struct dx_map_entry *p, *q, *top = map + count - 1;
        int more;
        /* Combsort until bubble sort doesn't suck */
        while (count > 2) {
                count = count*10/13;
                if (count - 9 < 2) /* 9, 10 -> 11 */
                        count = 11;
                for (p = top, q = p - count; q >= map; p--, q--)
                        if (p->hash < q->hash)
                                swap(*p, *q);
        }
        /* Garden variety bubble sort */
        do {
                more = 0;
                q = top;
                while (q-- > map) {
                        if (q[1].hash >= q[0].hash)
                                continue;
                        swap(*(q+1), *q);
                        more = 1;
                }
        } while(more);
}

static void dx_insert_block(struct dx_frame *frame, u32 hash, ext4_lblk_t block)
{
        struct dx_entry *entries = frame->entries;
        struct dx_entry *old = frame->at, *new = old + 1;
        int count = dx_get_count(entries);

        ASSERT(count < dx_get_limit(entries));
        ASSERT(old < entries + count);
        memmove(new + 1, new, (char *)(entries + count) - (char *)(new));
        dx_set_hash(new, hash);
        dx_set_block(new, block);
        dx_set_count(entries, count + 1);
}

#if IS_ENABLED(CONFIG_UNICODE)
int ext4_fname_setup_ci_filename(struct inode *dir, const struct qstr *iname,
                                  struct ext4_filename *name)
{
        struct qstr *cf_name = &name->cf_name;
        unsigned char *buf;
        struct dx_hash_info *hinfo = &name->hinfo;
        int len;

        if (!IS_CASEFOLDED(dir) ||
            (IS_ENCRYPTED(dir) && !fscrypt_has_encryption_key(dir))) {
                cf_name->name = NULL;
                return 0;
        }

        buf = kmalloc(EXT4_NAME_LEN, GFP_NOFS);
        if (!buf)
                return -ENOMEM;

        len = utf8_casefold(dir->i_sb->s_encoding, iname, buf, EXT4_NAME_LEN);
        if (len <= 0) {
                kfree(buf);
                buf = NULL;
        }
        cf_name->name = buf;
        cf_name->len = (unsigned) len;

        if (!IS_ENCRYPTED(dir))
                return 0;

        hinfo->hash_version = DX_HASH_SIPHASH;
        hinfo->seed = NULL;
        if (cf_name->name)
                return ext4fs_dirhash(dir, cf_name->name, cf_name->len, hinfo);
        else
                return ext4fs_dirhash(dir, iname->name, iname->len, hinfo);
}
#endif

/*
 * Test whether a directory entry matches the filename being searched for.
 *
 * Return: %true if the directory entry matches, otherwise %false.
 */
static bool ext4_match(struct inode *parent,
                              const struct ext4_filename *fname,
                              struct ext4_dir_entry_2 *de)
{
        struct fscrypt_name f;

        if (!de->inode)
                return false;

        f.usr_fname = fname->usr_fname;
        f.disk_name = fname->disk_name;
#ifdef CONFIG_FS_ENCRYPTION
        f.crypto_buf = fname->crypto_buf;
#endif

#if IS_ENABLED(CONFIG_UNICODE)
        if (IS_CASEFOLDED(parent) &&
            (!IS_ENCRYPTED(parent) || fscrypt_has_encryption_key(parent))) {
                /*
                 * Just checking IS_ENCRYPTED(parent) below is not
                 * sufficient to decide whether one can use the hash for
                 * skipping the string comparison, because the key might
                 * have been added right after
                 * ext4_fname_setup_ci_filename().  In this case, a hash
                 * mismatch will be a false negative.  Therefore, make
                 * sure cf_name was properly initialized before
                 * considering the calculated hash.
                 */
                if (sb_no_casefold_compat_fallback(parent->i_sb) &&
                    IS_ENCRYPTED(parent) && fname->cf_name.name &&
                    (fname->hinfo.hash != EXT4_DIRENT_HASH(de) ||
                     fname->hinfo.minor_hash != EXT4_DIRENT_MINOR_HASH(de)))
                        return false;
                /*
                 * Treat comparison errors as not a match.  The
                 * only case where it happens is on a disk
                 * corruption or ENOMEM.
                 */

                return generic_ci_match(parent, fname->usr_fname,
                                        &fname->cf_name, de->name,
                                        de->name_len) > 0;
        }
#endif

        return fscrypt_match_name(&f, de->name, de->name_len);
}

/*
 * Returns 0 if not found, -EFSCORRUPTED on failure, and 1 on success
 */
int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size,
                    struct inode *dir, struct ext4_filename *fname,
                    unsigned int offset, struct ext4_dir_entry_2 **res_dir)
{
        struct ext4_dir_entry_2 * de;
        char * dlimit;
        int de_len;

        de = (struct ext4_dir_entry_2 *)search_buf;
        dlimit = search_buf + buf_size;
        while ((char *) de < dlimit - EXT4_BASE_DIR_LEN) {
                /* this code is executed quadratically often */
                /* do minimal checking `by hand' */
                if (de->name + de->name_len <= dlimit &&
                    ext4_match(dir, fname, de)) {
                        /* found a match - just to be sure, do
                         * a full check */
                        if (ext4_check_dir_entry(dir, NULL, de, bh, search_buf,
                                                 buf_size, offset))
                                return -EFSCORRUPTED;
                        *res_dir = de;
                        return 1;
                }
                /* prevent looping on a bad block */
                de_len = ext4_rec_len_from_disk(de->rec_len,
                                                dir->i_sb->s_blocksize);
                if (de_len <= 0)
                        return -EFSCORRUPTED;
                offset += de_len;
                de = (struct ext4_dir_entry_2 *) ((char *) de + de_len);
        }
        return 0;
}

static int is_dx_internal_node(struct inode *dir, ext4_lblk_t block,
                               struct ext4_dir_entry *de)
{
        struct super_block *sb = dir->i_sb;

        if (!is_dx(dir))
                return 0;
        if (block == 0)
                return 1;
        if (de->inode == 0 &&
            ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize) ==
                        sb->s_blocksize)
                return 1;
        return 0;
}

/*
 *        __ext4_find_entry()
 *
 * finds an entry in the specified directory with the wanted name. It
 * returns the cache buffer in which the entry was found, and the entry
 * itself (as a parameter - res_dir). It does NOT read the inode of the
 * entry - you'll have to do that yourself if you want to.
 *
 * The returned buffer_head has ->b_count elevated.  The caller is expected
 * to brelse() it when appropriate.
 */
static struct buffer_head *__ext4_find_entry(struct inode *dir,
                                             struct ext4_filename *fname,
                                             struct ext4_dir_entry_2 **res_dir,
                                             int *inlined)
{
        struct super_block *sb;
        struct buffer_head *bh_use[NAMEI_RA_SIZE];
        struct buffer_head *bh, *ret = NULL;
        ext4_lblk_t start, block;
        const u8 *name = fname->usr_fname->name;
        size_t ra_max = 0;        /* Number of bh's in the readahead
                                   buffer, bh_use[] */
        size_t ra_ptr = 0;        /* Current index into readahead
                                   buffer */
        ext4_lblk_t  nblocks;
        int i, namelen, retval;

        *res_dir = NULL;
        sb = dir->i_sb;
        namelen = fname->usr_fname->len;
        if (namelen > EXT4_NAME_LEN)
                return NULL;

        if (ext4_has_inline_data(dir)) {
                int has_inline_data = 1;
                ret = ext4_find_inline_entry(dir, fname, res_dir,
                                             &has_inline_data);
                if (inlined)
                        *inlined = has_inline_data;
                if (has_inline_data || IS_ERR(ret))
                        goto cleanup_and_exit;
        }

        if ((namelen <= 2) && (name[0] == '.') &&
            (name[1] == '.' || name[1] == '\0')) {
                /*
                 * "." or ".." will only be in the first block
                 * NFS may look up ".."; "." should be handled by the VFS
                 */
                block = start = 0;
                nblocks = 1;
                goto restart;
        }
        if (is_dx(dir)) {
                ret = ext4_dx_find_entry(dir, fname, res_dir);
                /*
                 * On success, or if the error was file not found,
                 * return.  Otherwise, fall back to doing a search the
                 * old fashioned way.
                 */
                if (IS_ERR(ret) && PTR_ERR(ret) == ERR_BAD_DX_DIR)
                        dxtrace(printk(KERN_DEBUG "ext4_find_entry: dx failed, "
                                       "falling back\n"));
                else if (!sb_no_casefold_compat_fallback(dir->i_sb) &&
                         *res_dir == NULL && IS_CASEFOLDED(dir))
                        dxtrace(printk(KERN_DEBUG "ext4_find_entry: casefold "
                                       "failed, falling back\n"));
                else
                        goto cleanup_and_exit;
                ret = NULL;
        }
        nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
        if (!nblocks) {
                ret = NULL;
                goto cleanup_and_exit;
        }
        start = EXT4_I(dir)->i_dir_start_lookup;
        if (start >= nblocks)
                start = 0;
        block = start;
restart:
        do {
                /*
                 * We deal with the read-ahead logic here.
                 */
                cond_resched();
                if (ra_ptr >= ra_max) {
                        /* Refill the readahead buffer */
                        ra_ptr = 0;
                        if (block < start)
                                ra_max = start - block;
                        else
                                ra_max = nblocks - block;
                        ra_max = min(ra_max, ARRAY_SIZE(bh_use));
                        retval = ext4_bread_batch(dir, block, ra_max,
                                                  false /* wait */, bh_use);
                        if (retval) {
                                ret = ERR_PTR(retval);
                                ra_max = 0;
                                goto cleanup_and_exit;
                        }
                }
                if ((bh = bh_use[ra_ptr++]) == NULL)
                        goto next;
                wait_on_buffer(bh);
                if (!buffer_uptodate(bh)) {
                        EXT4_ERROR_INODE_ERR(dir, EIO,
                                             "reading directory lblock %lu",
                                             (unsigned long) block);
                        brelse(bh);
                        ret = ERR_PTR(-EIO);
                        goto cleanup_and_exit;
                }
                if (!buffer_verified(bh) &&
                    !is_dx_internal_node(dir, block,
                                         (struct ext4_dir_entry *)bh->b_data) &&
                    !ext4_dirblock_csum_verify(dir, bh)) {
                        EXT4_ERROR_INODE_ERR(dir, EFSBADCRC,
                                             "checksumming directory "
                                             "block %lu", (unsigned long)block);
                        brelse(bh);
                        ret = ERR_PTR(-EFSBADCRC);
                        goto cleanup_and_exit;
                }
                set_buffer_verified(bh);
                i = search_dirblock(bh, dir, fname,
                            block << EXT4_BLOCK_SIZE_BITS(sb), res_dir);
                if (i == 1) {
                        EXT4_I(dir)->i_dir_start_lookup = block;
                        ret = bh;
                        goto cleanup_and_exit;
                } else {
                        brelse(bh);
                        if (i < 0) {
                                ret = ERR_PTR(i);
                                goto cleanup_and_exit;
                        }
                }
        next:
                if (++block >= nblocks)
                        block = 0;
        } while (block != start);

        /*
         * If the directory has grown while we were searching, then
         * search the last part of the directory before giving up.
         */
        block = nblocks;
        nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
        if (block < nblocks) {
                start = 0;
                goto restart;
        }

cleanup_and_exit:
        /* Clean up the read-ahead blocks */
        for (; ra_ptr < ra_max; ra_ptr++)
                brelse(bh_use[ra_ptr]);
        return ret;
}

static struct buffer_head *ext4_find_entry(struct inode *dir,
                                           const struct qstr *d_name,
                                           struct ext4_dir_entry_2 **res_dir,
                                           int *inlined)
{
        int err;
        struct ext4_filename fname;
        struct buffer_head *bh;

        err = ext4_fname_setup_filename(dir, d_name, 1, &fname);
        if (err == -ENOENT)
                return NULL;
        if (err)
                return ERR_PTR(err);

        bh = __ext4_find_entry(dir, &fname, res_dir, inlined);

        ext4_fname_free_filename(&fname);
        return bh;
}

static struct buffer_head *ext4_lookup_entry(struct inode *dir,
                                             struct dentry *dentry,
                                             struct ext4_dir_entry_2 **res_dir)
{
        int err;
        struct ext4_filename fname;
        struct buffer_head *bh;

        err = ext4_fname_prepare_lookup(dir, dentry, &fname);
        if (err == -ENOENT)
                return NULL;
        if (err)
                return ERR_PTR(err);

        bh = __ext4_find_entry(dir, &fname, res_dir, NULL);

        ext4_fname_free_filename(&fname);
        return bh;
}

static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
                        struct ext4_filename *fname,
                        struct ext4_dir_entry_2 **res_dir)
{
        struct super_block * sb = dir->i_sb;
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct buffer_head *bh;
        ext4_lblk_t block;
        int retval;

#ifdef CONFIG_FS_ENCRYPTION
        *res_dir = NULL;
#endif
        frame = dx_probe(fname, dir, NULL, frames);
        if (IS_ERR(frame))
                return ERR_CAST(frame);
        do {
                block = dx_get_block(frame->at);
                bh = ext4_read_dirblock(dir, block, DIRENT_HTREE);
                if (IS_ERR(bh))
                        goto errout;

                retval = search_dirblock(bh, dir, fname,
                                         block << EXT4_BLOCK_SIZE_BITS(sb),
                                         res_dir);
                if (retval == 1)
                        goto success;
                brelse(bh);
                if (retval < 0) {
                        bh = ERR_PTR(ERR_BAD_DX_DIR);
                        goto errout;
                }

                /* Check to see if we should continue to search */
                retval = ext4_htree_next_block(dir, fname->hinfo.hash, frame,
                                               frames, NULL);
                if (retval < 0) {
                        ext4_warning_inode(dir,
                                "error %d reading directory index block",
                                retval);
                        bh = ERR_PTR(retval);
                        goto errout;
                }
        } while (retval == 1);

        bh = NULL;
errout:
        dxtrace(printk(KERN_DEBUG "%s not found\n", fname->usr_fname->name));
success:
        dx_release(frames);
        return bh;
}

static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
        struct inode *inode;
        struct ext4_dir_entry_2 *de;
        struct buffer_head *bh;

        if (dentry->d_name.len > EXT4_NAME_LEN)
                return ERR_PTR(-ENAMETOOLONG);

        bh = ext4_lookup_entry(dir, dentry, &de);
        if (IS_ERR(bh))
                return ERR_CAST(bh);
        inode = NULL;
        if (bh) {
                __u32 ino = le32_to_cpu(de->inode);
                brelse(bh);
                if (!ext4_valid_inum(dir->i_sb, ino)) {
                        EXT4_ERROR_INODE(dir, "bad inode number: %u", ino);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                if (unlikely(ino == dir->i_ino)) {
                        EXT4_ERROR_INODE(dir, "'%pd' linked to parent dir",
                                         dentry);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                inode = ext4_iget(dir->i_sb, ino, EXT4_IGET_NORMAL);
                if (inode == ERR_PTR(-ESTALE)) {
                        EXT4_ERROR_INODE(dir,
                                         "deleted inode referenced: %u",
                                         ino);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                if (!IS_ERR(inode) && IS_ENCRYPTED(dir) &&
                    (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
                    !fscrypt_has_permitted_context(dir, inode)) {
                        ext4_warning(inode->i_sb,
                                     "Inconsistent encryption contexts: %lu/%lu",
                                     dir->i_ino, inode->i_ino);
                        iput(inode);
                        return ERR_PTR(-EPERM);
                }
        }

        if (IS_ENABLED(CONFIG_UNICODE) && !inode && IS_CASEFOLDED(dir)) {
                /* Eventually we want to call d_add_ci(dentry, NULL)
                 * for negative dentries in the encoding case as
                 * well.  For now, prevent the negative dentry
                 * from being cached.
                 */
                return NULL;
        }

        return d_splice_alias(inode, dentry);
}


struct dentry *ext4_get_parent(struct dentry *child)
{
        __u32 ino;
        struct ext4_dir_entry_2 * de;
        struct buffer_head *bh;

        bh = ext4_find_entry(d_inode(child), &dotdot_name, &de, NULL);
        if (IS_ERR(bh))
                return ERR_CAST(bh);
        if (!bh)
                return ERR_PTR(-ENOENT);
        ino = le32_to_cpu(de->inode);
        brelse(bh);

        if (!ext4_valid_inum(child->d_sb, ino)) {
                EXT4_ERROR_INODE(d_inode(child),
                                 "bad parent inode number: %u", ino);
                return ERR_PTR(-EFSCORRUPTED);
        }

        return d_obtain_alias(ext4_iget(child->d_sb, ino, EXT4_IGET_NORMAL));
}

/*
 * Move count entries from end of map between two memory locations.
 * Returns pointer to last entry moved.
 */
static struct ext4_dir_entry_2 *
dx_move_dirents(struct inode *dir, char *from, char *to,
                struct dx_map_entry *map, int count,
                unsigned blocksize)
{
        unsigned rec_len = 0;

        while (count--) {
                struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)
                                                (from + (map->offs<<2));
                rec_len = ext4_dir_rec_len(de->name_len, dir);

                memcpy (to, de, rec_len);
                ((struct ext4_dir_entry_2 *) to)->rec_len =
                                ext4_rec_len_to_disk(rec_len, blocksize);

                /* wipe dir_entry excluding the rec_len field */
                de->inode = 0;
                memset(&de->name_len, 0, ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize) -
                                         offsetof(struct ext4_dir_entry_2,
                                                                name_len));

                map++;
                to += rec_len;
        }
        return (struct ext4_dir_entry_2 *) (to - rec_len);
}

/*
 * Compact each dir entry in the range to the minimal rec_len.
 * Returns pointer to last entry in range.
 */
static struct ext4_dir_entry_2 *dx_pack_dirents(struct inode *dir, char *base,
                                                        unsigned int blocksize)
{
        struct ext4_dir_entry_2 *next, *to, *prev, *de = (struct ext4_dir_entry_2 *) base;
        unsigned rec_len = 0;

        prev = to = de;
        while ((char*)de < base + blocksize) {
                next = ext4_next_entry(de, blocksize);
                if (de->inode && de->name_len) {
                        rec_len = ext4_dir_rec_len(de->name_len, dir);
                        if (de > to)
                                memmove(to, de, rec_len);
                        to->rec_len = ext4_rec_len_to_disk(rec_len, blocksize);
                        prev = to;
                        to = (struct ext4_dir_entry_2 *) (((char *) to) + rec_len);
                }
                de = next;
        }
        return prev;
}

/*
 * Split a full leaf block to make room for a new dir entry.
 * Allocate a new block, and move entries so that they are approx. equally full.
 * Returns pointer to de in block into which the new entry will be inserted.
 */
static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir,
                        struct buffer_head **bh,struct dx_frame *frame,
                        struct dx_hash_info *hinfo)
{
        unsigned blocksize = dir->i_sb->s_blocksize;
        unsigned continued;
        int count;
        struct buffer_head *bh2;
        ext4_lblk_t newblock;
        u32 hash2;
        struct dx_map_entry *map;
        char *data1 = (*bh)->b_data, *data2;
        unsigned split, move, size;
        struct ext4_dir_entry_2 *de = NULL, *de2;
        int        csum_size = 0;
        int        err = 0, i;

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        bh2 = ext4_append(handle, dir, &newblock);
        if (IS_ERR(bh2)) {
                brelse(*bh);
                *bh = NULL;
                return ERR_CAST(bh2);
        }

        BUFFER_TRACE(*bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, *bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        BUFFER_TRACE(frame->bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, frame->bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        data2 = bh2->b_data;

        /* create map in the end of data2 block */
        map = (struct dx_map_entry *) (data2 + blocksize);
        count = dx_make_map(dir, *bh, hinfo, map);
        if (count < 0) {
                err = count;
                goto journal_error;
        }
        map -= count;
        dx_sort_map(map, count);
        /* Ensure that neither split block is over half full */
        size = 0;
        move = 0;
        for (i = count-1; i >= 0; i--) {
                /* is more than half of this entry in 2nd half of the block? */
                if (size + map[i].size/2 > blocksize/2)
                        break;
                size += map[i].size;
                move++;
        }
        /*
         * map index at which we will split
         *
         * If the sum of active entries didn't exceed half the block size, just
         * split it in half by count; each resulting block will have at least
         * half the space free.
         */
        if (i >= 0)
                split = count - move;
        else
                split = count/2;

        if (WARN_ON_ONCE(split == 0)) {
                /* Should never happen, but avoid out-of-bounds access below */
                ext4_error_inode_block(dir, (*bh)->b_blocknr, 0,
                        "bad indexed directory? hash=%08x:%08x count=%d move=%u",
                        hinfo->hash, hinfo->minor_hash, count, move);
                err = -EFSCORRUPTED;
                goto out;
        }

        hash2 = map[split].hash;
        continued = hash2 == map[split - 1].hash;
        dxtrace(printk(KERN_INFO "Split block %lu at %x, %i/%i\n",
                        (unsigned long)dx_get_block(frame->at),
                                        hash2, split, count-split));

        /* Fancy dance to stay within two buffers */
        de2 = dx_move_dirents(dir, data1, data2, map + split, count - split,
                              blocksize);
        de = dx_pack_dirents(dir, data1, blocksize);
        de->rec_len = ext4_rec_len_to_disk(data1 + (blocksize - csum_size) -
                                           (char *) de,
                                           blocksize);
        de2->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) -
                                            (char *) de2,
                                            blocksize);
        if (csum_size) {
                ext4_initialize_dirent_tail(*bh, blocksize);
                ext4_initialize_dirent_tail(bh2, blocksize);
        }

        dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data1,
                        blocksize, 1));
        dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data2,
                        blocksize, 1));

        /* Which block gets the new entry? */
        if (hinfo->hash >= hash2) {
                swap(*bh, bh2);
                de = de2;
        }
        dx_insert_block(frame, hash2 + continued, newblock);
        err = ext4_handle_dirty_dirblock(handle, dir, bh2);
        if (err)
                goto journal_error;
        err = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
        if (err)
                goto journal_error;
        brelse(bh2);
        dxtrace(dx_show_index("frame", frame->entries));
        return de;

journal_error:
        ext4_std_error(dir->i_sb, err);
out:
        brelse(*bh);
        brelse(bh2);
        *bh = NULL;
        return ERR_PTR(err);
}

int ext4_find_dest_de(struct inode *dir, struct buffer_head *bh,
                      void *buf, int buf_size,
                      struct ext4_filename *fname,
                      struct ext4_dir_entry_2 **dest_de)
{
        struct ext4_dir_entry_2 *de;
        unsigned short reclen = ext4_dir_rec_len(fname_len(fname), dir);
        int nlen, rlen;
        unsigned int offset = 0;
        char *top;

        de = buf;
        top = buf + buf_size - reclen;
        while ((char *) de <= top) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                         buf, buf_size, offset))
                        return -EFSCORRUPTED;
                if (ext4_match(dir, fname, de))
                        return -EEXIST;
                nlen = ext4_dir_rec_len(de->name_len, dir);
                rlen = ext4_rec_len_from_disk(de->rec_len, buf_size);
                if ((de->inode ? rlen - nlen : rlen) >= reclen)
                        break;
                de = (struct ext4_dir_entry_2 *)((char *)de + rlen);
                offset += rlen;
        }
        if ((char *) de > top)
                return -ENOSPC;

        *dest_de = de;
        return 0;
}

void ext4_insert_dentry(struct inode *dir,
                        struct inode *inode,
                        struct ext4_dir_entry_2 *de,
                        int buf_size,
                        struct ext4_filename *fname)
{

        int nlen, rlen;

        nlen = ext4_dir_rec_len(de->name_len, dir);
        rlen = ext4_rec_len_from_disk(de->rec_len, buf_size);
        if (de->inode) {
                struct ext4_dir_entry_2 *de1 =
                        (struct ext4_dir_entry_2 *)((char *)de + nlen);
                de1->rec_len = ext4_rec_len_to_disk(rlen - nlen, buf_size);
                de->rec_len = ext4_rec_len_to_disk(nlen, buf_size);
                de = de1;
        }
        de->file_type = EXT4_FT_UNKNOWN;
        de->inode = cpu_to_le32(inode->i_ino);
        ext4_set_de_type(inode->i_sb, de, inode->i_mode);
        de->name_len = fname_len(fname);
        memcpy(de->name, fname_name(fname), fname_len(fname));
        if (ext4_hash_in_dirent(dir)) {
                struct dx_hash_info *hinfo = &fname->hinfo;

                EXT4_DIRENT_HASHES(de)->hash = cpu_to_le32(hinfo->hash);
                EXT4_DIRENT_HASHES(de)->minor_hash =
                                                cpu_to_le32(hinfo->minor_hash);
        }
}

/*
 * Add a new entry into a directory (leaf) block.  If de is non-NULL,
 * it points to a directory entry which is guaranteed to be large
 * enough for new directory entry.  If de is NULL, then
 * add_dirent_to_buf will attempt search the directory block for
 * space.  It will return -ENOSPC if no space is available, and -EIO
 * and -EEXIST if directory entry already exists.
 */
static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir,
                             struct inode *inode, struct ext4_dir_entry_2 *de,
                             struct buffer_head *bh)
{
        unsigned int        blocksize = dir->i_sb->s_blocksize;
        int                csum_size = 0;
        int                err, err2;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        if (!de) {
                err = ext4_find_dest_de(dir, bh, bh->b_data,
                                        blocksize - csum_size, fname, &de);
                if (err)
                        return err;
        }
        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (err) {
                ext4_std_error(dir->i_sb, err);
                return err;
        }

        /* By now the buffer is marked for journaling */
        ext4_insert_dentry(dir, inode, de, blocksize, fname);

        /*
         * XXX shouldn't update any times until successful
         * completion of syscall, but too many callers depend
         * on this.
         *
         * XXX similarly, too many callers depend on
         * ext4_new_inode() setting the times, but error
         * recovery deletes the inode, so the worst that can
         * happen is that the times are slightly out of date
         * and/or different from the directory change time.
         */
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        ext4_update_dx_flag(dir);
        inode_inc_iversion(dir);
        err2 = ext4_mark_inode_dirty(handle, dir);
        BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
        err = ext4_handle_dirty_dirblock(handle, dir, bh);
        if (err)
                ext4_std_error(dir->i_sb, err);
        return err ? err : err2;
}

static bool ext4_check_dx_root(struct inode *dir, struct dx_root *root)
{
        struct fake_dirent *fde;
        const char *error_msg;
        unsigned int rlen;
        unsigned int blocksize = dir->i_sb->s_blocksize;
        char *blockend = (char *)root + dir->i_sb->s_blocksize;

        fde = &root->dot;
        if (unlikely(fde->name_len != 1)) {
                error_msg = "invalid name_len for '.'";
                goto corrupted;
        }
        if (unlikely(strncmp(root->dot_name, ".", fde->name_len))) {
                error_msg = "invalid name for '.'";
                goto corrupted;
        }
        rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize);
        if (unlikely((char *)fde + rlen >= blockend)) {
                error_msg = "invalid rec_len for '.'";
                goto corrupted;
        }

        fde = &root->dotdot;
        if (unlikely(fde->name_len != 2)) {
                error_msg = "invalid name_len for '..'";
                goto corrupted;
        }
        if (unlikely(strncmp(root->dotdot_name, "..", fde->name_len))) {
                error_msg = "invalid name for '..'";
                goto corrupted;
        }
        rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize);
        if (unlikely((char *)fde + rlen >= blockend)) {
                error_msg = "invalid rec_len for '..'";
                goto corrupted;
        }

        return true;

corrupted:
        EXT4_ERROR_INODE(dir, "Corrupt dir, %s, running e2fsck is recommended",
                         error_msg);
        return false;
}

/*
 * This converts a one block unindexed directory to a 3 block indexed
 * directory, and adds the dentry to the indexed directory.
 */
static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname,
                            struct inode *dir,
                            struct inode *inode, struct buffer_head *bh)
{
        struct buffer_head *bh2;
        struct dx_root        *root;
        struct dx_frame        frames[EXT4_HTREE_LEVEL], *frame;
        struct dx_entry *entries;
        struct ext4_dir_entry_2        *de, *de2;
        char                *data2, *top;
        unsigned        len;
        int                retval;
        unsigned        blocksize;
        ext4_lblk_t  block;
        struct fake_dirent *fde;
        int csum_size = 0;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        blocksize =  dir->i_sb->s_blocksize;
        dxtrace(printk(KERN_DEBUG "Creating index: inode %lu\n", dir->i_ino));
        BUFFER_TRACE(bh, "get_write_access");
        retval = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                               EXT4_JTR_NONE);
        if (retval) {
                ext4_std_error(dir->i_sb, retval);
                brelse(bh);
                return retval;
        }

        root = (struct dx_root *) bh->b_data;
        if (!ext4_check_dx_root(dir, root)) {
                brelse(bh);
                return -EFSCORRUPTED;
        }

        /* The 0th block becomes the root, move the dirents out */
        fde = &root->dotdot;
        de = (struct ext4_dir_entry_2 *)((char *)fde +
                ext4_rec_len_from_disk(fde->rec_len, blocksize));
        len = ((char *) root) + (blocksize - csum_size) - (char *) de;

        /* Allocate new block for the 0th block's dirents */
        bh2 = ext4_append(handle, dir, &block);
        if (IS_ERR(bh2)) {
                brelse(bh);
                return PTR_ERR(bh2);
        }
        ext4_set_inode_flag(dir, EXT4_INODE_INDEX);
        data2 = bh2->b_data;

        memcpy(data2, de, len);
        memset(de, 0, len); /* wipe old data */
        de = (struct ext4_dir_entry_2 *) data2;
        top = data2 + len;
        while ((char *)(de2 = ext4_next_entry(de, blocksize)) < top) {
                if (ext4_check_dir_entry(dir, NULL, de, bh2, data2, len,
                                        (char *)de - data2)) {
                        brelse(bh2);
                        brelse(bh);
                        return -EFSCORRUPTED;
                }
                de = de2;
        }
        de->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) -
                                           (char *) de, blocksize);

        if (csum_size)
                ext4_initialize_dirent_tail(bh2, blocksize);

        /* Initialize the root; the dot dirents already exist */
        de = (struct ext4_dir_entry_2 *) (&root->dotdot);
        de->rec_len = ext4_rec_len_to_disk(
                        blocksize - ext4_dir_rec_len(2, NULL), blocksize);
        memset (&root->info, 0, sizeof(root->info));
        root->info.info_length = sizeof(root->info);
        if (ext4_hash_in_dirent(dir))
                root->info.hash_version = DX_HASH_SIPHASH;
        else
                root->info.hash_version =
                                EXT4_SB(dir->i_sb)->s_def_hash_version;

        entries = root->entries;
        dx_set_block(entries, 1);
        dx_set_count(entries, 1);
        dx_set_limit(entries, dx_root_limit(dir, sizeof(root->info)));

        /* Initialize as for dx_probe */
        fname->hinfo.hash_version = root->info.hash_version;
        if (fname->hinfo.hash_version <= DX_HASH_TEA)
                fname->hinfo.hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned;
        fname->hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed;

        /* casefolded encrypted hashes are computed on fname setup */
        if (!ext4_hash_in_dirent(dir)) {
                int err = ext4fs_dirhash(dir, fname_name(fname),
                                         fname_len(fname), &fname->hinfo);
                if (err < 0) {
                        brelse(bh2);
                        brelse(bh);
                        return err;
                }
        }
        memset(frames, 0, sizeof(frames));
        frame = frames;
        frame->entries = entries;
        frame->at = entries;
        frame->bh = bh;

        retval = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
        if (retval)
                goto out_frames;
        retval = ext4_handle_dirty_dirblock(handle, dir, bh2);
        if (retval)
                goto out_frames;

        de = do_split(handle,dir, &bh2, frame, &fname->hinfo);
        if (IS_ERR(de)) {
                retval = PTR_ERR(de);
                goto out_frames;
        }

        retval = add_dirent_to_buf(handle, fname, dir, inode, de, bh2);
out_frames:
        /*
         * Even if the block split failed, we have to properly write
         * out all the changes we did so far. Otherwise we can end up
         * with corrupted filesystem.
         */
        if (retval)
                ext4_mark_inode_dirty(handle, dir);
        dx_release(frames);
        brelse(bh2);
        return retval;
}

/*
 *        ext4_add_entry()
 *
 * adds a file entry to the specified directory, using the same
 * semantics as ext4_find_entry(). It returns NULL if it failed.
 *
 * NOTE!! The inode part of 'de' is left at 0 - which means you
 * may not sleep between calling this and putting something into
 * the entry, as someone else might have used it while you slept.
 */
static int ext4_add_entry(handle_t *handle, struct dentry *dentry,
                          struct inode *inode)
{
        struct inode *dir = d_inode(dentry->d_parent);
        struct buffer_head *bh = NULL;
        struct ext4_dir_entry_2 *de;
        struct super_block *sb;
        struct ext4_filename fname;
        int        retval;
        int        dx_fallback=0;
        unsigned blocksize;
        ext4_lblk_t block, blocks;
        int        csum_size = 0;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        sb = dir->i_sb;
        blocksize = sb->s_blocksize;

        if (fscrypt_is_nokey_name(dentry))
                return -ENOKEY;

        if (!generic_ci_validate_strict_name(dir, &dentry->d_name))
                return -EINVAL;

        retval = ext4_fname_setup_filename(dir, &dentry->d_name, 0, &fname);
        if (retval)
                return retval;

        if (ext4_has_inline_data(dir)) {
                retval = ext4_try_add_inline_entry(handle, &fname, dir, inode);
                if (retval < 0)
                        goto out;
                if (retval == 1) {
                        retval = 0;
                        goto out;
                }
        }

        if (is_dx(dir)) {
                retval = ext4_dx_add_entry(handle, &fname, dir, inode);
                if (!retval || (retval != ERR_BAD_DX_DIR))
                        goto out;
                /* Can we just ignore htree data? */
                if (ext4_has_feature_metadata_csum(sb)) {
                        EXT4_ERROR_INODE(dir,
                                "Directory has corrupted htree index.");
                        retval = -EFSCORRUPTED;
                        goto out;
                }
                ext4_clear_inode_flag(dir, EXT4_INODE_INDEX);
                dx_fallback++;
                retval = ext4_mark_inode_dirty(handle, dir);
                if (unlikely(retval))
                        goto out;
        }
        blocks = dir->i_size >> sb->s_blocksize_bits;
        for (block = 0; block < blocks; block++) {
                bh = ext4_read_dirblock(dir, block, DIRENT);
                if (bh == NULL) {
                        bh = ext4_bread(handle, dir, block,
                                        EXT4_GET_BLOCKS_CREATE);
                        goto add_to_new_block;
                }
                if (IS_ERR(bh)) {
                        retval = PTR_ERR(bh);
                        bh = NULL;
                        goto out;
                }
                retval = add_dirent_to_buf(handle, &fname, dir, inode,
                                           NULL, bh);
                if (retval != -ENOSPC)
                        goto out;

                if (blocks == 1 && !dx_fallback &&
                    ext4_has_feature_dir_index(sb)) {
                        retval = make_indexed_dir(handle, &fname, dir,
                                                  inode, bh);
                        bh = NULL; /* make_indexed_dir releases bh */
                        goto out;
                }
                brelse(bh);
        }
        bh = ext4_append(handle, dir, &block);
add_to_new_block:
        if (IS_ERR(bh)) {
                retval = PTR_ERR(bh);
                bh = NULL;
                goto out;
        }
        de = (struct ext4_dir_entry_2 *) bh->b_data;
        de->inode = 0;
        de->rec_len = ext4_rec_len_to_disk(blocksize - csum_size, blocksize);

        if (csum_size)
                ext4_initialize_dirent_tail(bh, blocksize);

        retval = add_dirent_to_buf(handle, &fname, dir, inode, de, bh);
out:
        ext4_fname_free_filename(&fname);
        brelse(bh);
        if (retval == 0)
                ext4_set_inode_state(inode, EXT4_STATE_NEWENTRY);
        return retval;
}

/*
 * Returns 0 for success, or a negative error value
 */
static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir, struct inode *inode)
{
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct dx_entry *entries, *at;
        struct buffer_head *bh;
        struct super_block *sb = dir->i_sb;
        struct ext4_dir_entry_2 *de;
        int restart;
        int err;

again:
        restart = 0;
        frame = dx_probe(fname, dir, NULL, frames);
        if (IS_ERR(frame))
                return PTR_ERR(frame);
        entries = frame->entries;
        at = frame->at;
        bh = ext4_read_dirblock(dir, dx_get_block(frame->at), DIRENT_HTREE);
        if (IS_ERR(bh)) {
                err = PTR_ERR(bh);
                bh = NULL;
                goto cleanup;
        }

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        err = add_dirent_to_buf(handle, fname, dir, inode, NULL, bh);
        if (err != -ENOSPC)
                goto cleanup;

        err = 0;
        /* Block full, should compress but for now just split */
        dxtrace(printk(KERN_DEBUG "using %u of %u node entries\n",
                       dx_get_count(entries), dx_get_limit(entries)));
        /* Need to split index? */
        if (dx_get_count(entries) == dx_get_limit(entries)) {
                ext4_lblk_t newblock;
                int levels = frame - frames + 1;
                unsigned int icount;
                int add_level = 1;
                struct dx_entry *entries2;
                struct dx_node *node2;
                struct buffer_head *bh2;

                while (frame > frames) {
                        if (dx_get_count((frame - 1)->entries) <
                            dx_get_limit((frame - 1)->entries)) {
                                add_level = 0;
                                break;
                        }
                        frame--; /* split higher index block */
                        at = frame->at;
                        entries = frame->entries;
                        restart = 1;
                }
                if (add_level && levels == ext4_dir_htree_level(sb)) {
                        ext4_warning(sb, "Directory (ino: %lu) index full, "
                                         "reach max htree level :%d",
                                         dir->i_ino, levels);
                        if (ext4_dir_htree_level(sb) < EXT4_HTREE_LEVEL) {
                                ext4_warning(sb, "Large directory feature is "
                                                 "not enabled on this "
                                                 "filesystem");
                        }
                        err = -ENOSPC;
                        goto cleanup;
                }
                icount = dx_get_count(entries);
                bh2 = ext4_append(handle, dir, &newblock);
                if (IS_ERR(bh2)) {
                        err = PTR_ERR(bh2);
                        goto cleanup;
                }
                node2 = (struct dx_node *)(bh2->b_data);
                entries2 = node2->entries;
                memset(&node2->fake, 0, sizeof(struct fake_dirent));
                node2->fake.rec_len = ext4_rec_len_to_disk(sb->s_blocksize,
                                                           sb->s_blocksize);
                BUFFER_TRACE(frame->bh, "get_write_access");
                err = ext4_journal_get_write_access(handle, sb, frame->bh,
                                                    EXT4_JTR_NONE);
                if (err) {
                        brelse(bh2);
                        goto journal_error;
                }
                if (!add_level) {
                        unsigned icount1 = icount/2, icount2 = icount - icount1;
                        unsigned hash2 = dx_get_hash(entries + icount1);
                        dxtrace(printk(KERN_DEBUG "Split index %i/%i\n",
                                       icount1, icount2));

                        BUFFER_TRACE(frame->bh, "get_write_access"); /* index root */
                        err = ext4_journal_get_write_access(handle, sb,
                                                            (frame - 1)->bh,
                                                            EXT4_JTR_NONE);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }

                        memcpy((char *) entries2, (char *) (entries + icount1),
                               icount2 * sizeof(struct dx_entry));
                        dx_set_count(entries, icount1);
                        dx_set_count(entries2, icount2);
                        dx_set_limit(entries2, dx_node_limit(dir));

                        /* Which index block gets the new entry? */
                        if (at - entries >= icount1) {
                                frame->at = at - entries - icount1 + entries2;
                                frame->entries = entries = entries2;
                                swap(frame->bh, bh2);
                        }
                        dx_insert_block((frame - 1), hash2, newblock);
                        dxtrace(dx_show_index("node", frame->entries));
                        dxtrace(dx_show_index("node",
                               ((struct dx_node *) bh2->b_data)->entries));
                        err = ext4_handle_dirty_dx_node(handle, dir, bh2);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }
                        brelse (bh2);
                        err = ext4_handle_dirty_dx_node(handle, dir,
                                                   (frame - 1)->bh);
                        if (err)
                                goto journal_error;
                        err = ext4_handle_dirty_dx_node(handle, dir,
                                                        frame->bh);
                        if (restart || err)
                                goto journal_error;
                } else {
                        struct dx_root *dxroot;
                        memcpy((char *) entries2, (char *) entries,
                               icount * sizeof(struct dx_entry));
                        dx_set_limit(entries2, dx_node_limit(dir));

                        /* Set up root */
                        dx_set_count(entries, 1);
                        dx_set_block(entries + 0, newblock);
                        dxroot = (struct dx_root *)frames[0].bh->b_data;
                        dxroot->info.indirect_levels += 1;
                        dxtrace(printk(KERN_DEBUG
                                       "Creating %d level index...\n",
                                       dxroot->info.indirect_levels));
                        err = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }
                        err = ext4_handle_dirty_dx_node(handle, dir, bh2);
                        brelse(bh2);
                        restart = 1;
                        goto journal_error;
                }
        }
        de = do_split(handle, dir, &bh, frame, &fname->hinfo);
        if (IS_ERR(de)) {
                err = PTR_ERR(de);
                goto cleanup;
        }
        err = add_dirent_to_buf(handle, fname, dir, inode, de, bh);
        goto cleanup;

journal_error:
        ext4_std_error(dir->i_sb, err); /* this is a no-op if err == 0 */
cleanup:
        brelse(bh);
        dx_release(frames);
        /* @restart is true means htree-path has been changed, we need to
         * repeat dx_probe() to find out valid htree-path
         */
        if (restart && err == 0)
                goto again;
        return err;
}

/*
 * ext4_generic_delete_entry deletes a directory entry by merging it
 * with the previous entry
 */
int ext4_generic_delete_entry(struct inode *dir,
                              struct ext4_dir_entry_2 *de_del,
                              struct buffer_head *bh,
                              void *entry_buf,
                              int buf_size,
                              int csum_size)
{
        struct ext4_dir_entry_2 *de, *pde;
        unsigned int blocksize = dir->i_sb->s_blocksize;
        int i;

        i = 0;
        pde = NULL;
        de = entry_buf;
        while (i < buf_size - csum_size) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                         entry_buf, buf_size, i))
                        return -EFSCORRUPTED;
                if (de == de_del)  {
                        if (pde) {
                                pde->rec_len = ext4_rec_len_to_disk(
                                        ext4_rec_len_from_disk(pde->rec_len,
                                                               blocksize) +
                                        ext4_rec_len_from_disk(de->rec_len,
                                                               blocksize),
                                        blocksize);

                                /* wipe entire dir_entry */
                                memset(de, 0, ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize));
                        } else {
                                /* wipe dir_entry excluding the rec_len field */
                                de->inode = 0;
                                memset(&de->name_len, 0,
                                        ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize) -
                                        offsetof(struct ext4_dir_entry_2,
                                                                name_len));
                        }

                        inode_inc_iversion(dir);
                        return 0;
                }
                i += ext4_rec_len_from_disk(de->rec_len, blocksize);
                pde = de;
                de = ext4_next_entry(de, blocksize);
        }
        return -ENOENT;
}

static int ext4_delete_entry(handle_t *handle,
                             struct inode *dir,
                             struct ext4_dir_entry_2 *de_del,
                             struct buffer_head *bh)
{
        int err, csum_size = 0;

        if (ext4_has_inline_data(dir)) {
                int has_inline_data = 1;
                err = ext4_delete_inline_entry(handle, dir, de_del, bh,
                                               &has_inline_data);
                if (has_inline_data)
                        return err;
        }

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (unlikely(err))
                goto out;

        err = ext4_generic_delete_entry(dir, de_del, bh, bh->b_data,
                                        dir->i_sb->s_blocksize, csum_size);
        if (err)
                goto out;

        BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
        err = ext4_handle_dirty_dirblock(handle, dir, bh);
        if (unlikely(err))
                goto out;

        return 0;
out:
        if (err != -ENOENT)
                ext4_std_error(dir->i_sb, err);
        return err;
}

/*
 * Set directory link count to 1 if nlinks > EXT4_LINK_MAX, or if nlinks == 2
 * since this indicates that nlinks count was previously 1 to avoid overflowing
 * the 16-bit i_links_count field on disk.  Directories with i_nlink == 1 mean
 * that subdirectory link counts are not being maintained accurately.
 *
 * The caller has already checked for i_nlink overflow in case the DIR_LINK
 * feature is not enabled and returned -EMLINK.  The is_dx() check is a proxy
 * for checking S_ISDIR(inode) (since the INODE_INDEX feature will not be set
 * on regular files) and to avoid creating huge/slow non-HTREE directories.
 */
static void ext4_inc_count(struct inode *inode)
{
        inc_nlink(inode);
        if (is_dx(inode) &&
            (inode->i_nlink > EXT4_LINK_MAX || inode->i_nlink == 2))
                set_nlink(inode, 1);
}

/*
 * If a directory had nlink == 1, then we should let it be 1. This indicates
 * directory has >EXT4_LINK_MAX subdirs.
 */
static void ext4_dec_count(struct inode *inode)
{
        if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2)
                drop_nlink(inode);
}


/*
 * Add non-directory inode to a directory. On success, the inode reference is
 * consumed by dentry is instantiation. This is also indicated by clearing of
 * *inodep pointer. On failure, the caller is responsible for dropping the
 * inode reference in the safe context.
 */
static int ext4_add_nondir(handle_t *handle,
                struct dentry *dentry, struct inode **inodep)
{
        struct inode *dir = d_inode(dentry->d_parent);
        struct inode *inode = *inodep;
        int err = ext4_add_entry(handle, dentry, inode);
        if (!err) {
                err = ext4_mark_inode_dirty(handle, inode);
                if (IS_DIRSYNC(dir))
                        ext4_handle_sync(handle);
                d_instantiate_new(dentry, inode);
                *inodep = NULL;
                return err;
        }
        drop_nlink(inode);
        ext4_mark_inode_dirty(handle, inode);
        ext4_orphan_add(handle, inode);
        unlock_new_inode(inode);
        return err;
}

/*
 * By the time this is called, we already have created
 * the directory cache entry for the new file, but it
 * is so far negative - it has no inode.
 *
 * If the create succeeds, we fill in the inode information
 * with d_instantiate().
 */
static int ext4_create(struct mnt_idmap *idmap, struct inode *dir,
                       struct dentry *dentry, umode_t mode, bool excl)
{
        handle_t *handle;
        struct inode *inode;
        int err, credits, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                inode->i_op = &ext4_file_inode_operations;
                inode->i_fop = &ext4_file_operations;
                ext4_set_aops(inode);
                err = ext4_add_nondir(handle, dentry, &inode);
                if (!err)
                        ext4_fc_track_create(handle, dentry);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (!IS_ERR_OR_NULL(inode))
                iput(inode);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_mknod(struct mnt_idmap *idmap, struct inode *dir,
                      struct dentry *dentry, umode_t mode, dev_t rdev)
{
        handle_t *handle;
        struct inode *inode;
        int err, credits, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                init_special_inode(inode, inode->i_mode, rdev);
                inode->i_op = &ext4_special_inode_operations;
                err = ext4_add_nondir(handle, dentry, &inode);
                if (!err)
                        ext4_fc_track_create(handle, dentry);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (!IS_ERR_OR_NULL(inode))
                iput(inode);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
                        struct file *file, umode_t mode)
{
        handle_t *handle;
        struct inode *inode;
        int err, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode,
                                            NULL, 0, NULL,
                                            EXT4_HT_DIR,
                        EXT4_MAXQUOTAS_TRANS_BLOCKS(dir->i_sb) +
                          4 + EXT4_XATTR_TRANS_BLOCKS);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                inode->i_op = &ext4_file_inode_operations;
                inode->i_fop = &ext4_file_operations;
                ext4_set_aops(inode);
                d_tmpfile(file, inode);
                err = ext4_orphan_add(handle, inode);
                if (err)
                        goto err_unlock_inode;
                mark_inode_dirty(inode);
                unlock_new_inode(inode);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return finish_open_simple(file, err);
err_unlock_inode:
        ext4_journal_stop(handle);
        unlock_new_inode(inode);
        return err;
}

int ext4_init_dirblock(handle_t *handle, struct inode *inode,
                       struct buffer_head *bh, unsigned int parent_ino,
                       void *inline_buf, int inline_size)
{
        struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *) bh->b_data;
        size_t                        blocksize = bh->b_size;
        int                        csum_size = 0, header_size;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        de->inode = cpu_to_le32(inode->i_ino);
        de->name_len = 1;
        de->rec_len = ext4_rec_len_to_disk(ext4_dir_rec_len(de->name_len, NULL),
                                           blocksize);
        memcpy(de->name, ".", 2);
        ext4_set_de_type(inode->i_sb, de, S_IFDIR);

        de = ext4_next_entry(de, blocksize);
        de->inode = cpu_to_le32(parent_ino);
        de->name_len = 2;
        memcpy(de->name, "..", 3);
        ext4_set_de_type(inode->i_sb, de, S_IFDIR);
        if (inline_buf) {
                de->rec_len = ext4_rec_len_to_disk(
                                        ext4_dir_rec_len(de->name_len, NULL),
                                        blocksize);
                de = ext4_next_entry(de, blocksize);
                header_size = (char *)de - bh->b_data;
                memcpy((void *)de, inline_buf, inline_size);
                ext4_update_final_de(bh->b_data, inline_size + header_size,
                        blocksize - csum_size);
        } else {
                de->rec_len = ext4_rec_len_to_disk(blocksize -
                                        (csum_size + ext4_dir_rec_len(1, NULL)),
                                        blocksize);
        }

        if (csum_size)
                ext4_initialize_dirent_tail(bh, blocksize);
        BUFFER_TRACE(dir_block, "call ext4_handle_dirty_metadata");
        set_buffer_uptodate(bh);
        set_buffer_verified(bh);
        return ext4_handle_dirty_dirblock(handle, inode, bh);
}

int ext4_init_new_dir(handle_t *handle, struct inode *dir,
                             struct inode *inode)
{
        struct buffer_head *dir_block = NULL;
        ext4_lblk_t block = 0;
        int err;

        if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
                err = ext4_try_create_inline_dir(handle, dir, inode);
                if (err < 0 && err != -ENOSPC)
                        goto out;
                if (!err)
                        goto out;
        }

        set_nlink(inode, 2);
        inode->i_size = 0;
        dir_block = ext4_append(handle, inode, &block);
        if (IS_ERR(dir_block))
                return PTR_ERR(dir_block);
        err = ext4_init_dirblock(handle, inode, dir_block, dir->i_ino, NULL, 0);
out:
        brelse(dir_block);
        return err;
}

static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                                 struct dentry *dentry, umode_t mode)
{
        handle_t *handle;
        struct inode *inode;
        int err, err2 = 0, credits, retries = 0;

        if (EXT4_DIR_LINK_MAX(dir))
                return ERR_PTR(-EMLINK);

        err = dquot_initialize(dir);
        if (err)
                return ERR_PTR(err);

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, S_IFDIR | mode,
                                            &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (IS_ERR(inode))
                goto out_stop;

        inode->i_op = &ext4_dir_inode_operations;
        inode->i_fop = &ext4_dir_operations;
        err = ext4_init_new_dir(handle, dir, inode);
        if (err)
                goto out_clear_inode;
        err = ext4_mark_inode_dirty(handle, inode);
        if (!err)
                err = ext4_add_entry(handle, dentry, inode);
        if (err) {
out_clear_inode:
                clear_nlink(inode);
                ext4_orphan_add(handle, inode);
                unlock_new_inode(inode);
                err2 = ext4_mark_inode_dirty(handle, inode);
                if (unlikely(err2))
                        err = err2;
                ext4_journal_stop(handle);
                iput(inode);
                goto out_retry;
        }
        ext4_inc_count(dir);

        ext4_update_dx_flag(dir);
        err = ext4_mark_inode_dirty(handle, dir);
        if (err)
                goto out_clear_inode;
        d_instantiate_new(dentry, inode);
        ext4_fc_track_create(handle, dentry);
        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

out_stop:
        if (handle)
                ext4_journal_stop(handle);
out_retry:
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return ERR_PTR(err);
}

/*
 * routine to check that the specified directory is empty (for rmdir)
 */
bool ext4_empty_dir(struct inode *inode)
{
        unsigned int offset;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        struct super_block *sb;

        if (ext4_has_inline_data(inode)) {
                int has_inline_data = 1;
                int ret;

                ret = empty_inline_dir(inode, &has_inline_data);
                if (has_inline_data)
                        return ret;
        }

        sb = inode->i_sb;
        if (inode->i_size < ext4_dir_rec_len(1, NULL) +
                                        ext4_dir_rec_len(2, NULL)) {
                EXT4_ERROR_INODE(inode, "invalid size");
                return false;
        }
        bh = ext4_read_dirblock(inode, 0, EITHER);
        if (IS_ERR(bh))
                return false;

        de = (struct ext4_dir_entry_2 *) bh->b_data;
        if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size,
                                 0) ||
            le32_to_cpu(de->inode) != inode->i_ino || de->name_len != 1 ||
            de->name[0] != '.') {
                ext4_warning_inode(inode, "directory missing '.'");
                brelse(bh);
                return false;
        }
        offset = ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        de = ext4_next_entry(de, sb->s_blocksize);
        if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size,
                                 offset) ||
            le32_to_cpu(de->inode) == 0 || de->name_len != 2 ||
            de->name[0] != '.' || de->name[1] != '.') {
                ext4_warning_inode(inode, "directory missing '..'");
                brelse(bh);
                return false;
        }
        offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        while (offset < inode->i_size) {
                if (!(offset & (sb->s_blocksize - 1))) {
                        unsigned int lblock;
                        brelse(bh);
                        lblock = offset >> EXT4_BLOCK_SIZE_BITS(sb);
                        bh = ext4_read_dirblock(inode, lblock, EITHER);
                        if (bh == NULL) {
                                offset += sb->s_blocksize;
                                continue;
                        }
                        if (IS_ERR(bh))
                                return false;
                }
                de = (struct ext4_dir_entry_2 *) (bh->b_data +
                                        (offset & (sb->s_blocksize - 1)));
                if (ext4_check_dir_entry(inode, NULL, de, bh,
                                         bh->b_data, bh->b_size, offset) ||
                    le32_to_cpu(de->inode)) {
                        brelse(bh);
                        return false;
                }
                offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        }
        brelse(bh);
        return true;
}

static int ext4_rmdir(struct inode *dir, struct dentry *dentry)
{
        int retval;
        struct inode *inode;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        handle_t *handle = NULL;

        retval = ext4_emergency_state(dir->i_sb);
        if (unlikely(retval))
                return retval;

        /* Initialize quotas before so that eventual writes go in
         * separate transaction */
        retval = dquot_initialize(dir);
        if (retval)
                return retval;
        retval = dquot_initialize(d_inode(dentry));
        if (retval)
                return retval;

        retval = -ENOENT;
        bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        if (!bh)
                goto end_rmdir;

        inode = d_inode(dentry);

        retval = -EFSCORRUPTED;
        if (le32_to_cpu(de->inode) != inode->i_ino)
                goto end_rmdir;

        retval = -ENOTEMPTY;
        if (!ext4_empty_dir(inode))
                goto end_rmdir;

        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                                    EXT4_DATA_TRANS_BLOCKS(dir->i_sb));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                handle = NULL;
                goto end_rmdir;
        }

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        retval = ext4_delete_entry(handle, dir, de, bh);
        if (retval)
                goto end_rmdir;
        if (!EXT4_DIR_LINK_EMPTY(inode))
                ext4_warning_inode(inode,
                             "empty directory '%.*s' has too many links (%u)",
                             dentry->d_name.len, dentry->d_name.name,
                             inode->i_nlink);
        inode_inc_iversion(inode);
        clear_nlink(inode);
        /* There's no need to set i_disksize: the fact that i_nlink is
         * zero will ensure that the right thing happens during any
         * recovery. */
        inode->i_size = 0;
        ext4_orphan_add(handle, inode);
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        inode_set_ctime_current(inode);
        retval = ext4_mark_inode_dirty(handle, inode);
        if (retval)
                goto end_rmdir;
        ext4_dec_count(dir);
        ext4_update_dx_flag(dir);
        ext4_fc_track_unlink(handle, dentry);
        retval = ext4_mark_inode_dirty(handle, dir);

        /* VFS negative dentries are incompatible with Encoding and
         * Case-insensitiveness. Eventually we'll want avoid
         * invalidating the dentries here, alongside with returning the
         * negative dentries at ext4_lookup(), when it is better
         * supported by the VFS for the CI case.
         */
        if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
                d_invalidate(dentry);

end_rmdir:
        brelse(bh);
        if (handle)
                ext4_journal_stop(handle);
        return retval;
}

int __ext4_unlink(struct inode *dir, const struct qstr *d_name,
                  struct inode *inode,
                  struct dentry *dentry /* NULL during fast_commit recovery */)
{
        int retval = -ENOENT;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        handle_t *handle;
        int skip_remove_dentry = 0;

        /*
         * Keep this outside the transaction; it may have to set up the
         * directory's encryption key, which isn't GFP_NOFS-safe.
         */
        bh = ext4_find_entry(dir, d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        if (!bh)
                return -ENOENT;

        if (le32_to_cpu(de->inode) != inode->i_ino) {
                /*
                 * It's okay if we find dont find dentry which matches
                 * the inode. That's because it might have gotten
                 * renamed to a different inode number
                 */
                if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
                        skip_remove_dentry = 1;
                else
                        goto out_bh;
        }

        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                                    EXT4_DATA_TRANS_BLOCKS(dir->i_sb));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                goto out_bh;
        }

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        if (!skip_remove_dentry) {
                retval = ext4_delete_entry(handle, dir, de, bh);
                if (retval)
                        goto out_handle;
                inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
                ext4_update_dx_flag(dir);
                retval = ext4_mark_inode_dirty(handle, dir);
                if (retval)
                        goto out_handle;
        } else {
                retval = 0;
        }
        if (inode->i_nlink == 0)
                ext4_warning_inode(inode, "Deleting file '%.*s' with no links",
                                   d_name->len, d_name->name);
        else
                drop_nlink(inode);
        if (!inode->i_nlink)
                ext4_orphan_add(handle, inode);
        inode_set_ctime_current(inode);
        retval = ext4_mark_inode_dirty(handle, inode);
        if (dentry && !retval)
                ext4_fc_track_unlink(handle, dentry);
out_handle:
        ext4_journal_stop(handle);
out_bh:
        brelse(bh);
        return retval;
}

static int ext4_unlink(struct inode *dir, struct dentry *dentry)
{
        int retval;

        retval = ext4_emergency_state(dir->i_sb);
        if (unlikely(retval))
                return retval;

        trace_ext4_unlink_enter(dir, dentry);
        /*
         * Initialize quotas before so that eventual writes go
         * in separate transaction
         */
        retval = dquot_initialize(dir);
        if (retval)
                goto out_trace;
        retval = dquot_initialize(d_inode(dentry));
        if (retval)
                goto out_trace;

        retval = __ext4_unlink(dir, &dentry->d_name, d_inode(dentry), dentry);

        /* VFS negative dentries are incompatible with Encoding and
         * Case-insensitiveness. Eventually we'll want avoid
         * invalidating the dentries here, alongside with returning the
         * negative dentries at ext4_lookup(), when it is  better
         * supported by the VFS for the CI case.
         */
        if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
                d_invalidate(dentry);

out_trace:
        trace_ext4_unlink_exit(dentry, retval);
        return retval;
}

static int ext4_init_symlink_block(handle_t *handle, struct inode *inode,
                                   struct fscrypt_str *disk_link)
{
        struct buffer_head *bh;
        char *kaddr;
        int err = 0;

        bh = ext4_bread(handle, inode, 0, EXT4_GET_BLOCKS_CREATE);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, inode->i_sb, bh, EXT4_JTR_NONE);
        if (err)
                goto out;

        kaddr = (char *)bh->b_data;
        memcpy(kaddr, disk_link->name, disk_link->len);
        inode->i_size = disk_link->len - 1;
        EXT4_I(inode)->i_disksize = inode->i_size;
        err = ext4_handle_dirty_metadata(handle, inode, bh);
out:
        brelse(bh);
        return err;
}

static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir,
                        struct dentry *dentry, const char *symname)
{
        handle_t *handle;
        struct inode *inode;
        int err, len = strlen(symname);
        int credits;
        struct fscrypt_str disk_link;
        int retries = 0;

        err = ext4_emergency_state(dir->i_sb);
        if (unlikely(err))
                return err;

        err = fscrypt_prepare_symlink(dir, symname, len, dir->i_sb->s_blocksize,
                                      &disk_link);
        if (err)
                return err;

        err = dquot_initialize(dir);
        if (err)
                return err;

        /*
         * EXT4_INDEX_EXTRA_TRANS_BLOCKS for addition of entry into the
         * directory. +3 for inode, inode bitmap, group descriptor allocation.
         * EXT4_DATA_TRANS_BLOCKS for the data block allocation and
         * modification.
         */
        credits = EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                  EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3;
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, S_IFLNK|S_IRWXUGO,
                                            &dentry->d_name, 0, NULL,
                                            EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        if (IS_ERR(inode)) {
                if (handle)
                        ext4_journal_stop(handle);
                err = PTR_ERR(inode);
                goto out_retry;
        }

        if (IS_ENCRYPTED(inode)) {
                err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link);
                if (err)
                        goto err_drop_inode;
                inode->i_op = &ext4_encrypted_symlink_inode_operations;
        } else {
                if ((disk_link.len > EXT4_N_BLOCKS * 4)) {
                        inode->i_op = &ext4_symlink_inode_operations;
                } else {
                        inode->i_op = &ext4_fast_symlink_inode_operations;
                }
        }

        if ((disk_link.len > EXT4_N_BLOCKS * 4)) {
                /* alloc symlink block and fill it */
                err = ext4_init_symlink_block(handle, inode, &disk_link);
                if (err)
                        goto err_drop_inode;
        } else {
                /* clear the extent format for fast symlink */
                ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS);
                memcpy((char *)&EXT4_I(inode)->i_data, disk_link.name,
                       disk_link.len);
                inode->i_size = disk_link.len - 1;
                EXT4_I(inode)->i_disksize = inode->i_size;
                if (!IS_ENCRYPTED(inode))
                        inode_set_cached_link(inode, (char *)&EXT4_I(inode)->i_data,
                                              inode->i_size);
        }
        err = ext4_add_nondir(handle, dentry, &inode);
        if (handle)
                ext4_journal_stop(handle);
        iput(inode);
        goto out_retry;

err_drop_inode:
        clear_nlink(inode);
        ext4_mark_inode_dirty(handle, inode);
        ext4_orphan_add(handle, inode);
        unlock_new_inode(inode);
        if (handle)
                ext4_journal_stop(handle);
        iput(inode);
out_retry:
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        if (disk_link.name != (unsigned char *)symname)
                kfree(disk_link.name);
        return err;
}

int __ext4_link(struct inode *dir, struct inode *inode, struct dentry *dentry)
{
        handle_t *handle;
        int err, retries = 0;
retry:
        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                 EXT4_INDEX_EXTRA_TRANS_BLOCKS) + 1);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        inode_set_ctime_current(inode);
        ext4_inc_count(inode);
        ihold(inode);

        err = ext4_add_entry(handle, dentry, inode);
        if (!err) {
                err = ext4_mark_inode_dirty(handle, inode);
                /* this can happen only for tmpfile being
                 * linked the first time
                 */
                if (inode->i_nlink == 1)
                        ext4_orphan_del(handle, inode);
                d_instantiate(dentry, inode);
                ext4_fc_track_link(handle, dentry);
        } else {
                drop_nlink(inode);
                iput(inode);
        }
        ext4_journal_stop(handle);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_link(struct dentry *old_dentry,
                     struct inode *dir, struct dentry *dentry)
{
        struct inode *inode = d_inode(old_dentry);
        int err;

        if (inode->i_nlink >= EXT4_LINK_MAX)
                return -EMLINK;

        err = fscrypt_prepare_link(old_dentry, dir, dentry);
        if (err)
                return err;

        if ((ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) &&
            (!projid_eq(EXT4_I(dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)))
                return -EXDEV;

        err = dquot_initialize(dir);
        if (err)
                return err;
        return __ext4_link(dir, inode, dentry);
}

/*
 * Try to find buffer head where contains the parent block.
 * It should be the inode block if it is inlined or the 1st block
 * if it is a normal dir.
 */
static struct buffer_head *ext4_get_first_dir_block(handle_t *handle,
                                        struct inode *inode,
                                        int *retval,
                                        struct ext4_dir_entry_2 **parent_de,
                                        int *inlined)
{
        struct buffer_head *bh;

        if (!ext4_has_inline_data(inode)) {
                struct ext4_dir_entry_2 *de;
                unsigned int offset;

                bh = ext4_read_dirblock(inode, 0, EITHER);
                if (IS_ERR(bh)) {
                        *retval = PTR_ERR(bh);
                        return NULL;
                }

                de = (struct ext4_dir_entry_2 *) bh->b_data;
                if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data,
                                         bh->b_size, 0) ||
                    le32_to_cpu(de->inode) != inode->i_ino ||
                    de->name_len != 1 || de->name[0] != '.') {
                        EXT4_ERROR_INODE(inode, "directory missing '.'");
                        brelse(bh);
                        *retval = -EFSCORRUPTED;
                        return NULL;
                }
                offset = ext4_rec_len_from_disk(de->rec_len,
                                                inode->i_sb->s_blocksize);
                de = ext4_next_entry(de, inode->i_sb->s_blocksize);
                if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data,
                                         bh->b_size, offset) ||
                    le32_to_cpu(de->inode) == 0 || de->name_len != 2 ||
                    de->name[0] != '.' || de->name[1] != '.') {
                        EXT4_ERROR_INODE(inode, "directory missing '..'");
                        brelse(bh);
                        *retval = -EFSCORRUPTED;
                        return NULL;
                }
                *parent_de = de;

                return bh;
        }

        *inlined = 1;
        return ext4_get_first_inline_block(inode, parent_de, retval);
}

struct ext4_renament {
        struct inode *dir;
        struct dentry *dentry;
        struct inode *inode;
        bool is_dir;
        int dir_nlink_delta;

        /* entry for "dentry" */
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        int inlined;

        /* entry for ".." in inode if it's a directory */
        struct buffer_head *dir_bh;
        struct ext4_dir_entry_2 *parent_de;
        int dir_inlined;
};

static int ext4_rename_dir_prepare(handle_t *handle, struct ext4_renament *ent, bool is_cross)
{
        int retval;

        ent->is_dir = true;
        if (!is_cross)
                return 0;

        ent->dir_bh = ext4_get_first_dir_block(handle, ent->inode,
                                              &retval, &ent->parent_de,
                                              &ent->dir_inlined);
        if (!ent->dir_bh)
                return retval;
        if (le32_to_cpu(ent->parent_de->inode) != ent->dir->i_ino)
                return -EFSCORRUPTED;
        BUFFER_TRACE(ent->dir_bh, "get_write_access");
        return ext4_journal_get_write_access(handle, ent->dir->i_sb,
                                             ent->dir_bh, EXT4_JTR_NONE);
}

static int ext4_rename_dir_finish(handle_t *handle, struct ext4_renament *ent,
                                  unsigned dir_ino)
{
        int retval;

        if (!ent->dir_bh)
                return 0;

        ent->parent_de->inode = cpu_to_le32(dir_ino);
        BUFFER_TRACE(ent->dir_bh, "call ext4_handle_dirty_metadata");
        if (!ent->dir_inlined) {
                if (is_dx(ent->inode)) {
                        retval = ext4_handle_dirty_dx_node(handle,
                                                           ent->inode,
                                                           ent->dir_bh);
                } else {
                        retval = ext4_handle_dirty_dirblock(handle, ent->inode,
                                                            ent->dir_bh);
                }
        } else {
                retval = ext4_mark_inode_dirty(handle, ent->inode);
        }
        if (retval) {
                ext4_std_error(ent->dir->i_sb, retval);
                return retval;
        }
        return 0;
}

static int ext4_setent(handle_t *handle, struct ext4_renament *ent,
                       unsigned ino, unsigned file_type)
{
        int retval, retval2;

        BUFFER_TRACE(ent->bh, "get write access");
        retval = ext4_journal_get_write_access(handle, ent->dir->i_sb, ent->bh,
                                               EXT4_JTR_NONE);
        if (retval)
                return retval;
        ent->de->inode = cpu_to_le32(ino);
        if (ext4_has_feature_filetype(ent->dir->i_sb))
                ent->de->file_type = file_type;
        inode_inc_iversion(ent->dir);
        inode_set_mtime_to_ts(ent->dir, inode_set_ctime_current(ent->dir));
        retval = ext4_mark_inode_dirty(handle, ent->dir);
        BUFFER_TRACE(ent->bh, "call ext4_handle_dirty_metadata");
        if (!ent->inlined) {
                retval2 = ext4_handle_dirty_dirblock(handle, ent->dir, ent->bh);
                if (unlikely(retval2)) {
                        ext4_std_error(ent->dir->i_sb, retval2);
                        return retval2;
                }
        }
        return retval;
}

static void ext4_resetent(handle_t *handle, struct ext4_renament *ent,
                          unsigned ino, unsigned file_type)
{
        struct ext4_renament old = *ent;
        int retval = 0;

        /*
         * old->de could have moved from under us during make indexed dir,
         * so the old->de may no longer valid and need to find it again
         * before reset old inode info.
         */
        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de,
                                 &old.inlined);
        if (IS_ERR(old.bh))
                retval = PTR_ERR(old.bh);
        if (!old.bh)
                retval = -ENOENT;
        if (retval) {
                ext4_std_error(old.dir->i_sb, retval);
                return;
        }

        ext4_setent(handle, &old, ino, file_type);
        brelse(old.bh);
}

static int ext4_find_delete_entry(handle_t *handle, struct inode *dir,
                                  const struct qstr *d_name)
{
        int retval = -ENOENT;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;

        bh = ext4_find_entry(dir, d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        if (bh) {
                retval = ext4_delete_entry(handle, dir, de, bh);
                brelse(bh);
        }
        return retval;
}

static void ext4_rename_delete(handle_t *handle, struct ext4_renament *ent,
                               int force_reread)
{
        int retval;
        /*
         * ent->de could have moved from under us during htree split, so make
         * sure that we are deleting the right entry.  We might also be pointing
         * to a stale entry in the unused part of ent->bh so just checking inum
         * and the name isn't enough.
         */
        if (le32_to_cpu(ent->de->inode) != ent->inode->i_ino ||
            ent->de->name_len != ent->dentry->d_name.len ||
            strncmp(ent->de->name, ent->dentry->d_name.name,
                    ent->de->name_len) ||
            force_reread) {
                retval = ext4_find_delete_entry(handle, ent->dir,
                                                &ent->dentry->d_name);
        } else {
                retval = ext4_delete_entry(handle, ent->dir, ent->de, ent->bh);
                if (retval == -ENOENT) {
                        retval = ext4_find_delete_entry(handle, ent->dir,
                                                        &ent->dentry->d_name);
                }
        }

        if (retval) {
                ext4_warning_inode(ent->dir,
                                   "Deleting old file: nlink %d, error=%d",
                                   ent->dir->i_nlink, retval);
        }
}

static void ext4_update_dir_count(handle_t *handle, struct ext4_renament *ent)
{
        if (ent->dir_nlink_delta) {
                if (ent->dir_nlink_delta == -1)
                        ext4_dec_count(ent->dir);
                else
                        ext4_inc_count(ent->dir);
                ext4_mark_inode_dirty(handle, ent->dir);
        }
}

static struct inode *ext4_whiteout_for_rename(struct mnt_idmap *idmap,
                                              struct ext4_renament *ent,
                                              int credits, handle_t **h)
{
        struct inode *wh;
        handle_t *handle;
        int retries = 0;

        /*
         * for inode block, sb block, group summaries,
         * and inode bitmap
         */
        credits += (EXT4_MAXQUOTAS_TRANS_BLOCKS(ent->dir->i_sb) +
                    EXT4_XATTR_TRANS_BLOCKS + 4);
retry:
        wh = ext4_new_inode_start_handle(idmap, ent->dir,
                                         S_IFCHR | WHITEOUT_MODE,
                                         &ent->dentry->d_name, 0, NULL,
                                         EXT4_HT_DIR, credits);

        handle = ext4_journal_current_handle();
        if (IS_ERR(wh)) {
                if (handle)
                        ext4_journal_stop(handle);
                if (PTR_ERR(wh) == -ENOSPC &&
                    ext4_should_retry_alloc(ent->dir->i_sb, &retries))
                        goto retry;
        } else {
                *h = handle;
                init_special_inode(wh, wh->i_mode, WHITEOUT_DEV);
                wh->i_op = &ext4_special_inode_operations;
        }
        return wh;
}

/*
 * Anybody can rename anything with this: the permission checks are left to the
 * higher-level routines.
 *
 * n.b.  old_{dentry,inode) refers to the source dentry/inode
 * while new_{dentry,inode) refers to the destination dentry/inode
 * This comes from rename(const char *oldpath, const char *newpath)
 */
static int ext4_rename(struct mnt_idmap *idmap, struct inode *old_dir,
                       struct dentry *old_dentry, struct inode *new_dir,
                       struct dentry *new_dentry, unsigned int flags)
{
        handle_t *handle = NULL;
        struct ext4_renament old = {
                .dir = old_dir,
                .dentry = old_dentry,
                .inode = d_inode(old_dentry),
        };
        struct ext4_renament new = {
                .dir = new_dir,
                .dentry = new_dentry,
                .inode = d_inode(new_dentry),
        };
        int force_reread;
        int retval;
        struct inode *whiteout = NULL;
        int credits;
        u8 old_file_type;

        if (new.inode && new.inode->i_nlink == 0) {
                EXT4_ERROR_INODE(new.inode,
                                 "target of rename is already freed");
                return -EFSCORRUPTED;
        }

        if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT)) &&
            (!projid_eq(EXT4_I(new_dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)))
                return -EXDEV;

        retval = dquot_initialize(old.dir);
        if (retval)
                return retval;
        retval = dquot_initialize(old.inode);
        if (retval)
                return retval;
        retval = dquot_initialize(new.dir);
        if (retval)
                return retval;

        /* Initialize quotas before so that eventual writes go
         * in separate transaction */
        if (new.inode) {
                retval = dquot_initialize(new.inode);
                if (retval)
                        return retval;
        }

        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de,
                                 &old.inlined);
        if (IS_ERR(old.bh))
                return PTR_ERR(old.bh);

        /*
         *  Check for inode number is _not_ due to possible IO errors.
         *  We might rmdir the source, keep it as pwd of some process
         *  and merrily kill the link to whatever was created under the
         *  same name. Goodbye sticky bit ;-<
         */
        retval = -ENOENT;
        if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino)
                goto release_bh;

        new.bh = ext4_find_entry(new.dir, &new.dentry->d_name,
                                 &new.de, &new.inlined);
        if (IS_ERR(new.bh)) {
                retval = PTR_ERR(new.bh);
                new.bh = NULL;
                goto release_bh;
        }
        if (new.bh) {
                if (!new.inode) {
                        brelse(new.bh);
                        new.bh = NULL;
                }
        }
        if (new.inode && !test_opt(new.dir->i_sb, NO_AUTO_DA_ALLOC))
                ext4_alloc_da_blocks(old.inode);

        credits = (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2);
        if (!(flags & RENAME_WHITEOUT)) {
                handle = ext4_journal_start(old.dir, EXT4_HT_DIR, credits);
                if (IS_ERR(handle)) {
                        retval = PTR_ERR(handle);
                        goto release_bh;
                }
        } else {
                whiteout = ext4_whiteout_for_rename(idmap, &old, credits, &handle);
                if (IS_ERR(whiteout)) {
                        retval = PTR_ERR(whiteout);
                        goto release_bh;
                }
        }

        old_file_type = old.de->file_type;
        if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir))
                ext4_handle_sync(handle);

        if (S_ISDIR(old.inode->i_mode)) {
                if (new.inode) {
                        retval = -ENOTEMPTY;
                        if (!ext4_empty_dir(new.inode))
                                goto end_rename;
                } else {
                        retval = -EMLINK;
                        if (new.dir != old.dir && EXT4_DIR_LINK_MAX(new.dir))
                                goto end_rename;
                }
                retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }
        /*
         * If we're renaming a file within an inline_data dir and adding or
         * setting the new dirent causes a conversion from inline_data to
         * extents/blockmap, we need to force the dirent delete code to
         * re-read the directory, or else we end up trying to delete a dirent
         * from what is now the extent tree root (or a block map).
         */
        force_reread = (new.dir->i_ino == old.dir->i_ino &&
                        ext4_test_inode_flag(new.dir, EXT4_INODE_INLINE_DATA));

        if (whiteout) {
                /*
                 * Do this before adding a new entry, so the old entry is sure
                 * to be still pointing to the valid old entry.
                 */
                retval = ext4_setent(handle, &old, whiteout->i_ino,
                                     EXT4_FT_CHRDEV);
                if (retval)
                        goto end_rename;
                retval = ext4_mark_inode_dirty(handle, whiteout);
                if (unlikely(retval))
                        goto end_rename;

        }
        if (!new.bh) {
                retval = ext4_add_entry(handle, new.dentry, old.inode);
                if (retval)
                        goto end_rename;
        } else {
                retval = ext4_setent(handle, &new,
                                     old.inode->i_ino, old_file_type);
                if (retval)
                        goto end_rename;
        }
        if (force_reread)
                force_reread = !ext4_test_inode_flag(new.dir,
                                                     EXT4_INODE_INLINE_DATA);

        /*
         * Like most other Unix systems, set the ctime for inodes on a
         * rename.
         */
        inode_set_ctime_current(old.inode);
        retval = ext4_mark_inode_dirty(handle, old.inode);
        if (unlikely(retval))
                goto end_rename;

        if (!whiteout) {
                /*
                 * ok, that's it
                 */
                ext4_rename_delete(handle, &old, force_reread);
        }

        if (new.inode) {
                ext4_dec_count(new.inode);
                inode_set_ctime_current(new.inode);
        }
        inode_set_mtime_to_ts(old.dir, inode_set_ctime_current(old.dir));
        ext4_update_dx_flag(old.dir);
        if (old.is_dir) {
                retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
                if (retval)
                        goto end_rename;

                ext4_dec_count(old.dir);
                if (new.inode) {
                        /* checked ext4_empty_dir above, can't have another
                         * parent, ext4_dec_count() won't work for many-linked
                         * dirs */
                        clear_nlink(new.inode);
                } else {
                        ext4_inc_count(new.dir);
                        ext4_update_dx_flag(new.dir);
                        retval = ext4_mark_inode_dirty(handle, new.dir);
                        if (unlikely(retval))
                                goto end_rename;
                }
        }
        retval = ext4_mark_inode_dirty(handle, old.dir);
        if (unlikely(retval))
                goto end_rename;

        if (old.is_dir) {
                /*
                 * We disable fast commits here that's because the
                 * replay code is not yet capable of changing dot dot
                 * dirents in directories.
                 */
                ext4_fc_mark_ineligible(old.inode->i_sb,
                        EXT4_FC_REASON_RENAME_DIR, handle);
        } else {
                struct super_block *sb = old.inode->i_sb;

                if (new.inode)
                        ext4_fc_track_unlink(handle, new.dentry);
                if (test_opt2(sb, JOURNAL_FAST_COMMIT) &&
                    !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
                    !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE))) {
                        __ext4_fc_track_link(handle, old.inode, new.dentry);
                        __ext4_fc_track_unlink(handle, old.inode, old.dentry);
                        if (whiteout)
                                __ext4_fc_track_create(handle, whiteout,
                                                       old.dentry);
                }
        }

        if (new.inode) {
                retval = ext4_mark_inode_dirty(handle, new.inode);
                if (unlikely(retval))
                        goto end_rename;
                if (!new.inode->i_nlink)
                        ext4_orphan_add(handle, new.inode);
        }
        retval = 0;

end_rename:
        if (whiteout) {
                if (retval) {
                        ext4_resetent(handle, &old,
                                      old.inode->i_ino, old_file_type);
                        drop_nlink(whiteout);
                        ext4_mark_inode_dirty(handle, whiteout);
                        ext4_orphan_add(handle, whiteout);
                }
                unlock_new_inode(whiteout);
                ext4_journal_stop(handle);
                iput(whiteout);
        } else {
                ext4_journal_stop(handle);
        }
release_bh:
        brelse(old.dir_bh);
        brelse(old.bh);
        brelse(new.bh);

        return retval;
}

static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
                             struct inode *new_dir, struct dentry *new_dentry)
{
        handle_t *handle = NULL;
        struct ext4_renament old = {
                .dir = old_dir,
                .dentry = old_dentry,
                .inode = d_inode(old_dentry),
        };
        struct ext4_renament new = {
                .dir = new_dir,
                .dentry = new_dentry,
                .inode = d_inode(new_dentry),
        };
        u8 new_file_type;
        int retval;

        if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT) &&
             !projid_eq(EXT4_I(new_dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)) ||
            (ext4_test_inode_flag(old_dir, EXT4_INODE_PROJINHERIT) &&
             !projid_eq(EXT4_I(old_dir)->i_projid,
                        EXT4_I(new_dentry->d_inode)->i_projid)))
                return -EXDEV;

        retval = dquot_initialize(old.dir);
        if (retval)
                return retval;
        retval = dquot_initialize(new.dir);
        if (retval)
                return retval;

        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name,
                                 &old.de, &old.inlined);
        if (IS_ERR(old.bh))
                return PTR_ERR(old.bh);
        /*
         *  Check for inode number is _not_ due to possible IO errors.
         *  We might rmdir the source, keep it as pwd of some process
         *  and merrily kill the link to whatever was created under the
         *  same name. Goodbye sticky bit ;-<
         */
        retval = -ENOENT;
        if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino)
                goto end_rename;

        new.bh = ext4_find_entry(new.dir, &new.dentry->d_name,
                                 &new.de, &new.inlined);
        if (IS_ERR(new.bh)) {
                retval = PTR_ERR(new.bh);
                new.bh = NULL;
                goto end_rename;
        }

        /* RENAME_EXCHANGE case: old *and* new must both exist */
        if (!new.bh || le32_to_cpu(new.de->inode) != new.inode->i_ino)
                goto end_rename;

        handle = ext4_journal_start(old.dir, EXT4_HT_DIR,
                (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) +
                 2 * EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                handle = NULL;
                goto end_rename;
        }

        if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir))
                ext4_handle_sync(handle);

        if (S_ISDIR(old.inode->i_mode)) {
                retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }
        if (S_ISDIR(new.inode->i_mode)) {
                retval = ext4_rename_dir_prepare(handle, &new, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }

        /*
         * Other than the special case of overwriting a directory, parents'
         * nlink only needs to be modified if this is a cross directory rename.
         */
        if (old.dir != new.dir && old.is_dir != new.is_dir) {
                old.dir_nlink_delta = old.is_dir ? -1 : 1;
                new.dir_nlink_delta = -old.dir_nlink_delta;
                retval = -EMLINK;
                if ((old.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(old.dir)) ||
                    (new.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(new.dir)))
                        goto end_rename;
        }

        new_file_type = new.de->file_type;
        retval = ext4_setent(handle, &new, old.inode->i_ino, old.de->file_type);
        if (retval)
                goto end_rename;

        retval = ext4_setent(handle, &old, new.inode->i_ino, new_file_type);
        if (retval)
                goto end_rename;

        /*
         * Like most other Unix systems, set the ctime for inodes on a
         * rename.
         */
        inode_set_ctime_current(old.inode);
        inode_set_ctime_current(new.inode);
        retval = ext4_mark_inode_dirty(handle, old.inode);
        if (unlikely(retval))
                goto end_rename;
        retval = ext4_mark_inode_dirty(handle, new.inode);
        if (unlikely(retval))
                goto end_rename;
        ext4_fc_mark_ineligible(new.inode->i_sb,
                                EXT4_FC_REASON_CROSS_RENAME, handle);
        if (old.dir_bh) {
                retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
                if (retval)
                        goto end_rename;
        }
        if (new.dir_bh) {
                retval = ext4_rename_dir_finish(handle, &new, old.dir->i_ino);
                if (retval)
                        goto end_rename;
        }
        ext4_update_dir_count(handle, &old);
        ext4_update_dir_count(handle, &new);
        retval = 0;

end_rename:
        brelse(old.dir_bh);
        brelse(new.dir_bh);
        brelse(old.bh);
        brelse(new.bh);
        if (handle)
                ext4_journal_stop(handle);
        return retval;
}

static int ext4_rename2(struct mnt_idmap *idmap,
                        struct inode *old_dir, struct dentry *old_dentry,
                        struct inode *new_dir, struct dentry *new_dentry,
                        unsigned int flags)
{
        int err;

        err = ext4_emergency_state(old_dir->i_sb);
        if (unlikely(err))
                return err;

        if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
                return -EINVAL;

        err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry,
                                     flags);
        if (err)
                return err;

        if (flags & RENAME_EXCHANGE) {
                return ext4_cross_rename(old_dir, old_dentry,
                                         new_dir, new_dentry);
        }

        return ext4_rename(idmap, old_dir, old_dentry, new_dir, new_dentry, flags);
}

/*
 * directories can handle most operations...
 */
const struct inode_operations ext4_dir_inode_operations = {
        .create                = ext4_create,
        .lookup                = ext4_lookup,
        .link                = ext4_link,
        .unlink                = ext4_unlink,
        .symlink        = ext4_symlink,
        .mkdir                = ext4_mkdir,
        .rmdir                = ext4_rmdir,
        .mknod                = ext4_mknod,
        .tmpfile        = ext4_tmpfile,
        .rename                = ext4_rename2,
        .setattr        = ext4_setattr,
        .getattr        = ext4_getattr,
        .listxattr        = ext4_listxattr,
        .get_inode_acl        = ext4_get_acl,
        .set_acl        = ext4_set_acl,
        .fiemap         = ext4_fiemap,
        .fileattr_get        = ext4_fileattr_get,
        .fileattr_set        = ext4_fileattr_set,
};

const struct inode_operations ext4_special_inode_operations = {
        .setattr        = ext4_setattr,
        .getattr        = ext4_getattr,
        .listxattr        = ext4_listxattr,
        .get_inode_acl        = ext4_get_acl,
        .set_acl        = ext4_set_acl,
};




























    2 




























































































































































    3 














    1 
    1 





    1 























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    3 








    3 





    2 
    2 
















    1 
    1 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008, 2009 open80211s Ltd.
 * Copyright (C) 2018 - 2024 Intel Corporation
 * Authors:    Luis Carlos Cobo <luisca@cozybit.com>
 *                Javier Cardona <javier@cozybit.com>
 */

#include <linux/slab.h>
#include <linux/unaligned.h>
#include <net/sock.h>
#include "ieee80211_i.h"
#include "mesh.h"
#include "wme.h"
#include "driver-ops.h"

static int mesh_allocated;
static struct kmem_cache *rm_cache;

bool mesh_action_is_path_sel(struct ieee80211_mgmt *mgmt)
{
        return (mgmt->u.action.u.mesh_action.action_code ==
                        WLAN_MESH_ACTION_HWMP_PATH_SELECTION);
}

void ieee80211s_init(void)
{
        mesh_allocated = 1;
        rm_cache = kmem_cache_create("mesh_rmc", sizeof(struct rmc_entry),
                                     0, 0, NULL);
}

void ieee80211s_stop(void)
{
        if (!mesh_allocated)
                return;
        kmem_cache_destroy(rm_cache);
}

static void ieee80211_mesh_housekeeping_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.housekeeping_timer);
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        set_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags);

        wiphy_work_queue(local->hw.wiphy, &sdata->work);
}

/**
 * mesh_matches_local - check if the config of a mesh point matches ours
 *
 * @sdata: local mesh subif
 * @ie: information elements of a management frame from the mesh peer
 *
 * This function checks if the mesh configuration of a mesh point matches the
 * local mesh configuration, i.e. if both nodes belong to the same mesh network.
 *
 * Returns: %true if both nodes belong to the same mesh
 */
bool mesh_matches_local(struct ieee80211_sub_if_data *sdata,
                        struct ieee802_11_elems *ie)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 basic_rates = 0;
        struct cfg80211_chan_def sta_chan_def;
        struct ieee80211_supported_band *sband;
        u32 vht_cap_info = 0;

        /*
         * As support for each feature is added, check for matching
         * - On mesh config capabilities
         *   - Power Save Support En
         *   - Sync support enabled
         *   - Sync support active
         *   - Sync support required from peer
         *   - MDA enabled
         * - Power management control on fc
         */
        if (!(ifmsh->mesh_id_len == ie->mesh_id_len &&
             memcmp(ifmsh->mesh_id, ie->mesh_id, ie->mesh_id_len) == 0 &&
             (ifmsh->mesh_pp_id == ie->mesh_config->meshconf_psel) &&
             (ifmsh->mesh_pm_id == ie->mesh_config->meshconf_pmetric) &&
             (ifmsh->mesh_cc_id == ie->mesh_config->meshconf_congest) &&
             (ifmsh->mesh_sp_id == ie->mesh_config->meshconf_synch) &&
             (ifmsh->mesh_auth_id == ie->mesh_config->meshconf_auth)))
                return false;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return false;

        ieee80211_sta_get_rates(sdata, ie, sband->band,
                                &basic_rates);

        if (sdata->vif.bss_conf.basic_rates != basic_rates)
                return false;

        cfg80211_chandef_create(&sta_chan_def, sdata->vif.bss_conf.chanreq.oper.chan,
                                NL80211_CHAN_NO_HT);
        ieee80211_chandef_ht_oper(ie->ht_operation, &sta_chan_def);

        if (ie->vht_cap_elem)
                vht_cap_info = le32_to_cpu(ie->vht_cap_elem->vht_cap_info);

        ieee80211_chandef_vht_oper(&sdata->local->hw, vht_cap_info,
                                   ie->vht_operation, ie->ht_operation,
                                   &sta_chan_def);
        ieee80211_chandef_he_6ghz_oper(sdata->local, ie->he_operation,
                                       ie->eht_operation,
                                       &sta_chan_def);

        if (!cfg80211_chandef_compatible(&sdata->vif.bss_conf.chanreq.oper,
                                         &sta_chan_def))
                return false;

        return true;
}

/**
 * mesh_peer_accepts_plinks - check if an mp is willing to establish peer links
 *
 * @ie: information elements of a management frame from the mesh peer
 *
 * Returns: %true if the mesh peer is willing to establish peer links
 */
bool mesh_peer_accepts_plinks(struct ieee802_11_elems *ie)
{
        return (ie->mesh_config->meshconf_cap &
                        IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS) != 0;
}

/**
 * mesh_accept_plinks_update - update accepting_plink in local mesh beacons
 *
 * @sdata: mesh interface in which mesh beacons are going to be updated
 *
 * Returns: beacon changed flag if the beacon content changed.
 */
u64 mesh_accept_plinks_update(struct ieee80211_sub_if_data *sdata)
{
        bool free_plinks;
        u64 changed = 0;

        /* In case mesh_plink_free_count > 0 and mesh_plinktbl_capacity == 0,
         * the mesh interface might be able to establish plinks with peers that
         * are already on the table but are not on PLINK_ESTAB state. However,
         * in general the mesh interface is not accepting peer link requests
         * from new peers, and that must be reflected in the beacon
         */
        free_plinks = mesh_plink_availables(sdata);

        if (free_plinks != sdata->u.mesh.accepting_plinks) {
                sdata->u.mesh.accepting_plinks = free_plinks;
                changed = BSS_CHANGED_BEACON;
        }

        return changed;
}

/*
 * mesh_sta_cleanup - clean up any mesh sta state
 *
 * @sta: mesh sta to clean up.
 */
void mesh_sta_cleanup(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        u64 changed = mesh_plink_deactivate(sta);

        if (changed)
                ieee80211_mbss_info_change_notify(sdata, changed);
}

int mesh_rmc_init(struct ieee80211_sub_if_data *sdata)
{
        int i;

        sdata->u.mesh.rmc = kmalloc(sizeof(struct mesh_rmc), GFP_KERNEL);
        if (!sdata->u.mesh.rmc)
                return -ENOMEM;
        sdata->u.mesh.rmc->idx_mask = RMC_BUCKETS - 1;
        for (i = 0; i < RMC_BUCKETS; i++)
                INIT_HLIST_HEAD(&sdata->u.mesh.rmc->bucket[i]);
        return 0;
}

void mesh_rmc_free(struct ieee80211_sub_if_data *sdata)
{
        struct mesh_rmc *rmc = sdata->u.mesh.rmc;
        struct rmc_entry *p;
        struct hlist_node *n;
        int i;

        if (!sdata->u.mesh.rmc)
                return;

        for (i = 0; i < RMC_BUCKETS; i++) {
                hlist_for_each_entry_safe(p, n, &rmc->bucket[i], list) {
                        hlist_del(&p->list);
                        kmem_cache_free(rm_cache, p);
                }
        }

        kfree(rmc);
        sdata->u.mesh.rmc = NULL;
}

/**
 * mesh_rmc_check - Check frame in recent multicast cache and add if absent.
 *
 * @sdata:        interface
 * @sa:                source address
 * @mesh_hdr:        mesh_header
 *
 * Returns: 0 if the frame is not in the cache, nonzero otherwise.
 *
 * Checks using the source address and the mesh sequence number if we have
 * received this frame lately. If the frame is not in the cache, it is added to
 * it.
 */
int mesh_rmc_check(struct ieee80211_sub_if_data *sdata,
                   const u8 *sa, struct ieee80211s_hdr *mesh_hdr)
{
        struct mesh_rmc *rmc = sdata->u.mesh.rmc;
        u32 seqnum = 0;
        int entries = 0;
        u8 idx;
        struct rmc_entry *p;
        struct hlist_node *n;

        if (!rmc)
                return -1;

        /* Don't care about endianness since only match matters */
        memcpy(&seqnum, &mesh_hdr->seqnum, sizeof(mesh_hdr->seqnum));
        idx = le32_to_cpu(mesh_hdr->seqnum) & rmc->idx_mask;
        hlist_for_each_entry_safe(p, n, &rmc->bucket[idx], list) {
                ++entries;
                if (time_after(jiffies, p->exp_time) ||
                    entries == RMC_QUEUE_MAX_LEN) {
                        hlist_del(&p->list);
                        kmem_cache_free(rm_cache, p);
                        --entries;
                } else if ((seqnum == p->seqnum) && ether_addr_equal(sa, p->sa))
                        return -1;
        }

        p = kmem_cache_alloc(rm_cache, GFP_ATOMIC);
        if (!p)
                return 0;

        p->seqnum = seqnum;
        p->exp_time = jiffies + RMC_TIMEOUT;
        memcpy(p->sa, sa, ETH_ALEN);
        hlist_add_head(&p->list, &rmc->bucket[idx]);
        return 0;
}

int mesh_add_meshconf_ie(struct ieee80211_sub_if_data *sdata,
                         struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos, neighbors;
        u8 meshconf_len = sizeof(struct ieee80211_meshconf_ie);
        bool is_connected_to_gate = ifmsh->num_gates > 0 ||
                ifmsh->mshcfg.dot11MeshGateAnnouncementProtocol ||
                ifmsh->mshcfg.dot11MeshConnectedToMeshGate;
        bool is_connected_to_as = ifmsh->mshcfg.dot11MeshConnectedToAuthServer;

        if (skb_tailroom(skb) < 2 + meshconf_len)
                return -ENOMEM;

        pos = skb_put(skb, 2 + meshconf_len);
        *pos++ = WLAN_EID_MESH_CONFIG;
        *pos++ = meshconf_len;

        /* save a pointer for quick updates in pre-tbtt */
        ifmsh->meshconf_offset = pos - skb->data;

        /* Active path selection protocol ID */
        *pos++ = ifmsh->mesh_pp_id;
        /* Active path selection metric ID   */
        *pos++ = ifmsh->mesh_pm_id;
        /* Congestion control mode identifier */
        *pos++ = ifmsh->mesh_cc_id;
        /* Synchronization protocol identifier */
        *pos++ = ifmsh->mesh_sp_id;
        /* Authentication Protocol identifier */
        *pos++ = ifmsh->mesh_auth_id;
        /* Mesh Formation Info - number of neighbors */
        neighbors = atomic_read(&ifmsh->estab_plinks);
        neighbors = min_t(int, neighbors, IEEE80211_MAX_MESH_PEERINGS);
        *pos++ = (is_connected_to_as << 7) |
                 (neighbors << 1) |
                 is_connected_to_gate;
        /* Mesh capability */
        *pos = 0x00;
        *pos |= ifmsh->mshcfg.dot11MeshForwarding ?
                        IEEE80211_MESHCONF_CAPAB_FORWARDING : 0x00;
        *pos |= ifmsh->accepting_plinks ?
                        IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS : 0x00;
        /* Mesh PS mode. See IEEE802.11-2012 8.4.2.100.8 */
        *pos |= ifmsh->ps_peers_deep_sleep ?
                        IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL : 0x00;
        return 0;
}

int mesh_add_meshid_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos;

        if (skb_tailroom(skb) < 2 + ifmsh->mesh_id_len)
                return -ENOMEM;

        pos = skb_put(skb, 2 + ifmsh->mesh_id_len);
        *pos++ = WLAN_EID_MESH_ID;
        *pos++ = ifmsh->mesh_id_len;
        if (ifmsh->mesh_id_len)
                memcpy(pos, ifmsh->mesh_id, ifmsh->mesh_id_len);

        return 0;
}

static int mesh_add_awake_window_ie(struct ieee80211_sub_if_data *sdata,
                                    struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos;

        /* see IEEE802.11-2012 13.14.6 */
        if (ifmsh->ps_peers_light_sleep == 0 &&
            ifmsh->ps_peers_deep_sleep == 0 &&
            ifmsh->nonpeer_pm == NL80211_MESH_POWER_ACTIVE)
                return 0;

        if (skb_tailroom(skb) < 4)
                return -ENOMEM;

        pos = skb_put(skb, 2 + 2);
        *pos++ = WLAN_EID_MESH_AWAKE_WINDOW;
        *pos++ = 2;
        put_unaligned_le16(ifmsh->mshcfg.dot11MeshAwakeWindowDuration, pos);

        return 0;
}

int mesh_add_vendor_ies(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 offset, len;
        const u8 *data;

        if (!ifmsh->ie || !ifmsh->ie_len)
                return 0;

        /* fast-forward to vendor IEs */
        offset = ieee80211_ie_split_vendor(ifmsh->ie, ifmsh->ie_len, 0);

        if (offset < ifmsh->ie_len) {
                len = ifmsh->ie_len - offset;
                data = ifmsh->ie + offset;
                if (skb_tailroom(skb) < len)
                        return -ENOMEM;
                skb_put_data(skb, data, len);
        }

        return 0;
}

int mesh_add_rsn_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 len = 0;
        const u8 *data;

        if (!ifmsh->ie || !ifmsh->ie_len)
                return 0;

        /* find RSN IE */
        data = cfg80211_find_ie(WLAN_EID_RSN, ifmsh->ie, ifmsh->ie_len);
        if (!data)
                return 0;

        len = data[1] + 2;

        if (skb_tailroom(skb) < len)
                return -ENOMEM;
        skb_put_data(skb, data, len);

        return 0;
}

static int mesh_add_ds_params_ie(struct ieee80211_sub_if_data *sdata,
                                 struct sk_buff *skb)
{
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *chan;
        u8 *pos;

        if (skb_tailroom(skb) < 3)
                return -ENOMEM;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        chan = chanctx_conf->def.chan;
        rcu_read_unlock();

        pos = skb_put(skb, 2 + 1);
        *pos++ = WLAN_EID_DS_PARAMS;
        *pos++ = 1;
        *pos++ = ieee80211_frequency_to_channel(chan->center_freq);

        return 0;
}

int mesh_add_ht_cap_ie(struct ieee80211_sub_if_data *sdata,
                       struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        /* HT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!sband->ht_cap.ht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_ht_cap))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_cap));
        ieee80211_ie_build_ht_cap(pos, &sband->ht_cap, sband->ht_cap.cap);

        return 0;
}

int mesh_add_ht_oper_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *channel;
        struct ieee80211_supported_band *sband;
        struct ieee80211_sta_ht_cap *ht_cap;
        u8 *pos;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        channel = chanctx_conf->def.chan;
        rcu_read_unlock();

        sband = local->hw.wiphy->bands[channel->band];
        ht_cap = &sband->ht_cap;

        /* HT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!ht_cap->ht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_ht_operation))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_operation));
        ieee80211_ie_build_ht_oper(pos, ht_cap, &sdata->vif.bss_conf.chanreq.oper,
                                   sdata->vif.bss_conf.ht_operation_mode,
                                   false);

        return 0;
}

int mesh_add_vht_cap_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        /* VHT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!sband->vht_cap.vht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_vht_cap))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_cap));
        ieee80211_ie_build_vht_cap(pos, &sband->vht_cap, sband->vht_cap.cap);

        return 0;
}

int mesh_add_vht_oper_ie(struct ieee80211_sub_if_data *sdata,
                         struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *channel;
        struct ieee80211_supported_band *sband;
        struct ieee80211_sta_vht_cap *vht_cap;
        u8 *pos;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        channel = chanctx_conf->def.chan;
        rcu_read_unlock();

        sband = local->hw.wiphy->bands[channel->band];
        vht_cap = &sband->vht_cap;

        /* VHT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!vht_cap->vht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_vht_operation))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_operation));
        ieee80211_ie_build_vht_oper(pos, vht_cap,
                                    &sdata->vif.bss_conf.chanreq.oper);

        return 0;
}

int mesh_add_he_cap_ie(struct ieee80211_sub_if_data *sdata,
                       struct sk_buff *skb, u8 ie_len)
{
        struct ieee80211_supported_band *sband;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        if (sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        return ieee80211_put_he_cap(skb, sdata, sband, NULL);
}

int mesh_add_he_oper_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        const struct ieee80211_sta_he_cap *he_cap;
        struct ieee80211_supported_band *sband;
        u32 len;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        he_cap = ieee80211_get_he_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
        if (!he_cap ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        len = 2 + 1 + sizeof(struct ieee80211_he_operation);
        if (sdata->vif.bss_conf.chanreq.oper.chan->band == NL80211_BAND_6GHZ)
                len += sizeof(struct ieee80211_he_6ghz_oper);

        if (skb_tailroom(skb) < len)
                return -ENOMEM;

        pos = skb_put(skb, len);
        ieee80211_ie_build_he_oper(pos, &sdata->vif.bss_conf.chanreq.oper);

        return 0;
}

int mesh_add_he_6ghz_cap_ie(struct ieee80211_sub_if_data *sdata,
                            struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        const struct ieee80211_sband_iftype_data *iftd;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        if (sband->band != NL80211_BAND_6GHZ)
                return 0;

        iftd = ieee80211_get_sband_iftype_data(sband,
                                               NL80211_IFTYPE_MESH_POINT);
        /* The device doesn't support HE in mesh mode or at all */
        if (!iftd)
                return 0;

        ieee80211_put_he_6ghz_cap(skb, sdata, sdata->deflink.smps_mode);
        return 0;
}

int mesh_add_eht_cap_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb, u8 ie_len)
{
        struct ieee80211_supported_band *sband;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        if (sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        return ieee80211_put_eht_cap(skb, sdata, sband, NULL);
}

int mesh_add_eht_oper_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        const struct ieee80211_sta_eht_cap *eht_cap;
        struct ieee80211_supported_band *sband;
        u32 len;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        eht_cap = ieee80211_get_eht_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
        if (!eht_cap ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        len = 2 + 1 + offsetof(struct ieee80211_eht_operation, optional) +
                      offsetof(struct ieee80211_eht_operation_info, optional);

        if (skb_tailroom(skb) < len)
                return -ENOMEM;

        pos = skb_put(skb, len);
        ieee80211_ie_build_eht_oper(pos, &sdata->vif.bss_conf.chanreq.oper, eht_cap);

        return 0;
}

static void ieee80211_mesh_path_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.mesh_path_timer);

        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

static void ieee80211_mesh_path_root_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.mesh_path_root_timer);
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        set_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);

        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

void ieee80211_mesh_root_setup(struct ieee80211_if_mesh *ifmsh)
{
        if (ifmsh->mshcfg.dot11MeshHWMPRootMode > IEEE80211_ROOTMODE_ROOT)
                set_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);
        else {
                clear_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);
                /* stop running timer */
                timer_delete_sync(&ifmsh->mesh_path_root_timer);
        }
}

static void
ieee80211_mesh_update_bss_params(struct ieee80211_sub_if_data *sdata,
                                 u8 *ie, u8 ie_len)
{
        struct ieee80211_supported_band *sband;
        const struct element *cap;
        const struct ieee80211_he_operation *he_oper = NULL;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return;

        if (!ieee80211_get_he_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT) ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return;

        sdata->vif.bss_conf.he_support = true;

        cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ie, ie_len);
        if (cap && cap->datalen >= 1 + sizeof(*he_oper) &&
            cap->datalen >= 1 + ieee80211_he_oper_size(cap->data + 1))
                he_oper = (void *)(cap->data + 1);

        if (he_oper)
                sdata->vif.bss_conf.he_oper.params =
                        __le32_to_cpu(he_oper->he_oper_params);

        sdata->vif.bss_conf.eht_support =
                !!ieee80211_get_eht_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
}

bool ieee80211_mesh_xmit_fast(struct ieee80211_sub_if_data *sdata,
                              struct sk_buff *skb, u32 ctrl_flags)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_mesh_fast_tx_key key = {
                .type = MESH_FAST_TX_TYPE_LOCAL
        };
        struct ieee80211_mesh_fast_tx *entry;
        struct ieee80211s_hdr *meshhdr;
        u8 sa[ETH_ALEN] __aligned(2);
        struct tid_ampdu_tx *tid_tx;
        struct sta_info *sta;
        bool copy_sa = false;
        u16 ethertype;
        u8 tid;

        if (ctrl_flags & IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP)
                return false;

        if (ifmsh->mshcfg.dot11MeshNolearn)
                return false;

        /* Add support for these cases later */
        if (ifmsh->ps_peers_light_sleep || ifmsh->ps_peers_deep_sleep)
                return false;

        if (is_multicast_ether_addr(skb->data))
                return false;

        ethertype = (skb->data[12] << 8) | skb->data[13];
        if (ethertype < ETH_P_802_3_MIN)
                return false;

        if (sk_requests_wifi_status(skb->sk))
                return false;

        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                skb_set_transport_header(skb, skb_checksum_start_offset(skb));
                if (skb_checksum_help(skb))
                        return false;
        }

        ether_addr_copy(key.addr, skb->data);
        if (!ether_addr_equal(skb->data + ETH_ALEN, sdata->vif.addr))
                key.type = MESH_FAST_TX_TYPE_PROXIED;
        entry = mesh_fast_tx_get(sdata, &key);
        if (!entry)
                return false;

        if (skb_headroom(skb) < entry->hdrlen + entry->fast_tx.hdr_len)
                return false;

        sta = rcu_dereference(entry->mpath->next_hop);
        if (!sta)
                return false;

        tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK;
        tid_tx = rcu_dereference(sta->ampdu_mlme.tid_tx[tid]);
        if (tid_tx) {
                if (!test_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state))
                        return false;
                if (tid_tx->timeout)
                        tid_tx->last_tx = jiffies;
        }

        skb = skb_share_check(skb, GFP_ATOMIC);
        if (!skb)
                return true;

        skb_set_queue_mapping(skb, ieee80211_select_queue(sdata, sta, skb));

        meshhdr = (struct ieee80211s_hdr *)entry->hdr;
        if ((meshhdr->flags & MESH_FLAGS_AE) == MESH_FLAGS_AE_A5_A6) {
                /* preserve SA from eth header for 6-addr frames */
                ether_addr_copy(sa, skb->data + ETH_ALEN);
                copy_sa = true;
        }

        memcpy(skb_push(skb, entry->hdrlen - 2 * ETH_ALEN), entry->hdr,
               entry->hdrlen);

        meshhdr = (struct ieee80211s_hdr *)skb->data;
        put_unaligned_le32(atomic_inc_return(&sdata->u.mesh.mesh_seqnum),
                           &meshhdr->seqnum);
        meshhdr->ttl = sdata->u.mesh.mshcfg.dot11MeshTTL;
        if (copy_sa)
            ether_addr_copy(meshhdr->eaddr2, sa);

        skb_push(skb, 2 * ETH_ALEN);
        __ieee80211_xmit_fast(sdata, sta, &entry->fast_tx, skb, tid_tx,
                              entry->mpath->dst, sdata->vif.addr);

        return true;
}

/**
 * ieee80211_fill_mesh_addresses - fill addresses of a locally originated mesh frame
 * @hdr:        802.11 frame header
 * @fc:                frame control field
 * @meshda:        destination address in the mesh
 * @meshsa:        source address in the mesh.  Same as TA, as frame is
 *              locally originated.
 *
 * Returns: the length of the 802.11 frame header (excludes mesh control header)
 */
int ieee80211_fill_mesh_addresses(struct ieee80211_hdr *hdr, __le16 *fc,
                                  const u8 *meshda, const u8 *meshsa)
{
        if (is_multicast_ether_addr(meshda)) {
                *fc |= cpu_to_le16(IEEE80211_FCTL_FROMDS);
                /* DA TA SA */
                memcpy(hdr->addr1, meshda, ETH_ALEN);
                memcpy(hdr->addr2, meshsa, ETH_ALEN);
                memcpy(hdr->addr3, meshsa, ETH_ALEN);
                return 24;
        } else {
                *fc |= cpu_to_le16(IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS);
                /* RA TA DA SA */
                eth_zero_addr(hdr->addr1);   /* RA is resolved later */
                memcpy(hdr->addr2, meshsa, ETH_ALEN);
                memcpy(hdr->addr3, meshda, ETH_ALEN);
                memcpy(hdr->addr4, meshsa, ETH_ALEN);
                return 30;
        }
}

/**
 * ieee80211_new_mesh_header - create a new mesh header
 * @sdata:        mesh interface to be used
 * @meshhdr:    uninitialized mesh header
 * @addr4or5:   1st address in the ae header, which may correspond to address 4
 *              (if addr6 is NULL) or address 5 (if addr6 is present). It may
 *              be NULL.
 * @addr6:        2nd address in the ae header, which corresponds to addr6 of the
 *              mesh frame
 *
 * Returns: the header length
 */
unsigned int ieee80211_new_mesh_header(struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211s_hdr *meshhdr,
                                       const char *addr4or5, const char *addr6)
{
        if (WARN_ON(!addr4or5 && addr6))
                return 0;

        memset(meshhdr, 0, sizeof(*meshhdr));

        meshhdr->ttl = sdata->u.mesh.mshcfg.dot11MeshTTL;

        put_unaligned_le32(atomic_inc_return(&sdata->u.mesh.mesh_seqnum),
                           &meshhdr->seqnum);
        if (addr4or5 && !addr6) {
                meshhdr->flags |= MESH_FLAGS_AE_A4;
                memcpy(meshhdr->eaddr1, addr4or5, ETH_ALEN);
                return 2 * ETH_ALEN;
        } else if (addr4or5 && addr6) {
                meshhdr->flags |= MESH_FLAGS_AE_A5_A6;
                memcpy(meshhdr->eaddr1, addr4or5, ETH_ALEN);
                memcpy(meshhdr->eaddr2, addr6, ETH_ALEN);
                return 3 * ETH_ALEN;
        }

        return ETH_ALEN;
}

static void ieee80211_mesh_housekeeping(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u64 changed;

        if (ifmsh->mshcfg.plink_timeout > 0)
                ieee80211_sta_expire(sdata, ifmsh->mshcfg.plink_timeout * HZ);
        mesh_path_expire(sdata);

        changed = mesh_accept_plinks_update(sdata);
        ieee80211_mbss_info_change_notify(sdata, changed);

        mesh_fast_tx_gc(sdata);

        mod_timer(&ifmsh->housekeeping_timer,
                  round_jiffies(jiffies +
                                IEEE80211_MESH_HOUSEKEEPING_INTERVAL));
}

static void ieee80211_mesh_rootpath(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 interval;

        mesh_path_tx_root_frame(sdata);

        if (ifmsh->mshcfg.dot11MeshHWMPRootMode == IEEE80211_PROACTIVE_RANN)
                interval = ifmsh->mshcfg.dot11MeshHWMPRannInterval;
        else
                interval = ifmsh->mshcfg.dot11MeshHWMProotInterval;

        mod_timer(&ifmsh->mesh_path_root_timer,
                  round_jiffies(TU_TO_EXP_TIME(interval)));
}

static int
ieee80211_mesh_build_beacon(struct ieee80211_if_mesh *ifmsh)
{
        struct beacon_data *bcn;
        int head_len, tail_len;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        struct mesh_csa_settings *csa;
        const struct ieee80211_supported_band *sband;
        u8 ie_len_he_cap, ie_len_eht_cap;
        u8 *pos;
        struct ieee80211_sub_if_data *sdata;
        int hdr_len = offsetofend(struct ieee80211_mgmt, u.beacon);

        sdata = container_of(ifmsh, struct ieee80211_sub_if_data, u.mesh);

        sband = ieee80211_get_sband(sdata);

        ie_len_he_cap = ieee80211_ie_len_he_cap(sdata);
        ie_len_eht_cap = ieee80211_ie_len_eht_cap(sdata);
        head_len = hdr_len +
                   2 + /* NULL SSID */
                   /* Channel Switch Announcement */
                   2 + sizeof(struct ieee80211_channel_sw_ie) +
                   /* Mesh Channel Switch Parameters */
                   2 + sizeof(struct ieee80211_mesh_chansw_params_ie) +
                   /* Channel Switch Wrapper + Wide Bandwidth CSA IE */
                   2 + 2 + sizeof(struct ieee80211_wide_bw_chansw_ie) +
                   2 + sizeof(struct ieee80211_sec_chan_offs_ie) +
                   2 + 8 + /* supported rates */
                   2 + 3; /* DS params */
        tail_len = 2 + (IEEE80211_MAX_SUPP_RATES - 8) +
                   2 + sizeof(struct ieee80211_ht_cap) +
                   2 + sizeof(struct ieee80211_ht_operation) +
                   2 + ifmsh->mesh_id_len +
                   2 + sizeof(struct ieee80211_meshconf_ie) +
                   2 + sizeof(__le16) + /* awake window */
                   2 + sizeof(struct ieee80211_vht_cap) +
                   2 + sizeof(struct ieee80211_vht_operation) +
                   ie_len_he_cap +
                   2 + 1 + sizeof(struct ieee80211_he_operation) +
                           sizeof(struct ieee80211_he_6ghz_oper) +
                   2 + 1 + sizeof(struct ieee80211_he_6ghz_capa) +
                   ie_len_eht_cap +
                   2 + 1 + offsetof(struct ieee80211_eht_operation, optional) +
                           offsetof(struct ieee80211_eht_operation_info, optional) +
                   ifmsh->ie_len;

        bcn = kzalloc(sizeof(*bcn) + head_len + tail_len, GFP_KERNEL);
        /* need an skb for IE builders to operate on */
        skb = __dev_alloc_skb(max(head_len, tail_len), GFP_KERNEL);

        if (!bcn || !skb)
                goto out_free;

        /*
         * pointers go into the block we allocated,
         * memory is | beacon_data | head | tail |
         */
        bcn->head = ((u8 *) bcn) + sizeof(*bcn);

        /* fill in the head */
        mgmt = skb_put_zero(skb, hdr_len);
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                          IEEE80211_STYPE_BEACON);
        eth_broadcast_addr(mgmt->da);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, sdata->vif.addr, ETH_ALEN);
        ieee80211_mps_set_frame_flags(sdata, NULL, (void *) mgmt);
        mgmt->u.beacon.beacon_int =
                cpu_to_le16(sdata->vif.bss_conf.beacon_int);
        mgmt->u.beacon.capab_info |= cpu_to_le16(
                sdata->u.mesh.security ? WLAN_CAPABILITY_PRIVACY : 0);

        pos = skb_put(skb, 2);
        *pos++ = WLAN_EID_SSID;
        *pos++ = 0x0;

        rcu_read_lock();
        csa = rcu_dereference(ifmsh->csa);
        if (csa) {
                enum nl80211_channel_type ct;
                struct cfg80211_chan_def *chandef;
                int ie_len = 2 + sizeof(struct ieee80211_channel_sw_ie) +
                             2 + sizeof(struct ieee80211_mesh_chansw_params_ie);

                pos = skb_put_zero(skb, ie_len);
                *pos++ = WLAN_EID_CHANNEL_SWITCH;
                *pos++ = 3;
                *pos++ = 0x0;
                *pos++ = ieee80211_frequency_to_channel(
                                csa->settings.chandef.chan->center_freq);
                bcn->cntdwn_current_counter = csa->settings.count;
                bcn->cntdwn_counter_offsets[0] = hdr_len + 6;
                *pos++ = csa->settings.count;
                *pos++ = WLAN_EID_CHAN_SWITCH_PARAM;
                *pos++ = 6;
                if (ifmsh->csa_role == IEEE80211_MESH_CSA_ROLE_INIT) {
                        *pos++ = ifmsh->mshcfg.dot11MeshTTL;
                        *pos |= WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR;
                } else {
                        *pos++ = ifmsh->chsw_ttl;
                }
                *pos++ |= csa->settings.block_tx ?
                          WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT : 0x00;
                put_unaligned_le16(WLAN_REASON_MESH_CHAN, pos);
                pos += 2;
                put_unaligned_le16(ifmsh->pre_value, pos);
                pos += 2;

                switch (csa->settings.chandef.width) {
                case NL80211_CHAN_WIDTH_40:
                        ie_len = 2 + sizeof(struct ieee80211_sec_chan_offs_ie);
                        pos = skb_put_zero(skb, ie_len);

                        *pos++ = WLAN_EID_SECONDARY_CHANNEL_OFFSET; /* EID */
                        *pos++ = 1;                                    /* len */
                        ct = cfg80211_get_chandef_type(&csa->settings.chandef);
                        if (ct == NL80211_CHAN_HT40PLUS)
                                *pos++ = IEEE80211_HT_PARAM_CHA_SEC_ABOVE;
                        else
                                *pos++ = IEEE80211_HT_PARAM_CHA_SEC_BELOW;
                        break;
                case NL80211_CHAN_WIDTH_80:
                case NL80211_CHAN_WIDTH_80P80:
                case NL80211_CHAN_WIDTH_160:
                        /* Channel Switch Wrapper + Wide Bandwidth CSA IE */
                        ie_len = 2 + 2 +
                                 sizeof(struct ieee80211_wide_bw_chansw_ie);
                        pos = skb_put_zero(skb, ie_len);

                        *pos++ = WLAN_EID_CHANNEL_SWITCH_WRAPPER; /* EID */
                        *pos++ = 5;                                  /* len */
                        /* put sub IE */
                        chandef = &csa->settings.chandef;
                        ieee80211_ie_build_wide_bw_cs(pos, chandef);
                        break;
                default:
                        break;
                }
        }
        rcu_read_unlock();

        if (ieee80211_put_srates_elem(skb, sband,
                                      sdata->vif.bss_conf.basic_rates,
                                      0, WLAN_EID_SUPP_RATES) ||
            mesh_add_ds_params_ie(sdata, skb))
                goto out_free;

        bcn->head_len = skb->len;
        memcpy(bcn->head, skb->data, bcn->head_len);

        /* now the tail */
        skb_trim(skb, 0);
        bcn->tail = bcn->head + bcn->head_len;

        if (ieee80211_put_srates_elem(skb, sband,
                                      sdata->vif.bss_conf.basic_rates,
                                      0, WLAN_EID_EXT_SUPP_RATES) ||
            mesh_add_rsn_ie(sdata, skb) ||
            mesh_add_ht_cap_ie(sdata, skb) ||
            mesh_add_ht_oper_ie(sdata, skb) ||
            mesh_add_meshid_ie(sdata, skb) ||
            mesh_add_meshconf_ie(sdata, skb) ||
            mesh_add_awake_window_ie(sdata, skb) ||
            mesh_add_vht_cap_ie(sdata, skb) ||
            mesh_add_vht_oper_ie(sdata, skb) ||
            mesh_add_he_cap_ie(sdata, skb, ie_len_he_cap) ||
            mesh_add_he_oper_ie(sdata, skb) ||
            mesh_add_he_6ghz_cap_ie(sdata, skb) ||
            mesh_add_eht_cap_ie(sdata, skb, ie_len_eht_cap) ||
            mesh_add_eht_oper_ie(sdata, skb) ||
            mesh_add_vendor_ies(sdata, skb))
                goto out_free;

        bcn->tail_len = skb->len;
        memcpy(bcn->tail, skb->data, bcn->tail_len);
        ieee80211_mesh_update_bss_params(sdata, bcn->tail, bcn->tail_len);
        bcn->meshconf = (struct ieee80211_meshconf_ie *)
                                        (bcn->tail + ifmsh->meshconf_offset);

        dev_kfree_skb(skb);
        rcu_assign_pointer(ifmsh->beacon, bcn);
        return 0;
out_free:
        kfree(bcn);
        dev_kfree_skb(skb);
        return -ENOMEM;
}

static int
ieee80211_mesh_rebuild_beacon(struct ieee80211_sub_if_data *sdata)
{
        struct beacon_data *old_bcn;
        int ret;

        old_bcn = sdata_dereference(sdata->u.mesh.beacon, sdata);
        ret = ieee80211_mesh_build_beacon(&sdata->u.mesh);
        if (ret)
                /* just reuse old beacon */
                return ret;

        if (old_bcn)
                kfree_rcu(old_bcn, rcu_head);
        return 0;
}

void ieee80211_mbss_info_change_notify(struct ieee80211_sub_if_data *sdata,
                                       u64 changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        unsigned long bits[] = { BITMAP_FROM_U64(changed) };
        u32 bit;

        if (!changed)
                return;

        /* if we race with running work, worst case this work becomes a noop */
        for_each_set_bit(bit, bits, sizeof(changed) * BITS_PER_BYTE)
                set_bit(bit, ifmsh->mbss_changed);
        set_bit(MESH_WORK_MBSS_CHANGED, &ifmsh->wrkq_flags);
        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

int ieee80211_start_mesh(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_local *local = sdata->local;
        u64 changed = BSS_CHANGED_BEACON |
                      BSS_CHANGED_BEACON_ENABLED |
                      BSS_CHANGED_HT |
                      BSS_CHANGED_BASIC_RATES |
                      BSS_CHANGED_BEACON_INT |
                      BSS_CHANGED_MCAST_RATE;

        local->fif_other_bss++;
        /* mesh ifaces must set allmulti to forward mcast traffic */
        atomic_inc(&local->iff_allmultis);
        ieee80211_configure_filter(local);

        ifmsh->mesh_cc_id = 0;        /* Disabled */
        /* register sync ops from extensible synchronization framework */
        ifmsh->sync_ops = ieee80211_mesh_sync_ops_get(ifmsh->mesh_sp_id);
        ifmsh->sync_offset_clockdrift_max = 0;
        set_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags);
        ieee80211_mesh_root_setup(ifmsh);
        wiphy_work_queue(local->hw.wiphy, &sdata->work);
        sdata->vif.bss_conf.ht_operation_mode =
                                ifmsh->mshcfg.ht_opmode;
        sdata->vif.bss_conf.enable_beacon = true;

        changed |= ieee80211_mps_local_status_update(sdata);

        if (ieee80211_mesh_build_beacon(ifmsh)) {
                ieee80211_stop_mesh(sdata);
                return -ENOMEM;
        }

        ieee80211_recalc_dtim(sdata, drv_get_tsf(local, sdata));
        ieee80211_link_info_change_notify(sdata, &sdata->deflink, changed);

        netif_carrier_on(sdata->dev);
        return 0;
}

void ieee80211_stop_mesh(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct beacon_data *bcn;

        netif_carrier_off(sdata->dev);

        /* flush STAs and mpaths on this iface */
        sta_info_flush(sdata, -1);
        ieee80211_free_keys(sdata, true);
        mesh_path_flush_by_iface(sdata);

        /* stop the beacon */
        ifmsh->mesh_id_len = 0;
        sdata->vif.bss_conf.enable_beacon = false;
        sdata->beacon_rate_set = false;
        clear_bit(SDATA_STATE_OFFCHANNEL_BEACON_STOPPED, &sdata->state);
        ieee80211_link_info_change_notify(sdata, &sdata->deflink,
                                          BSS_CHANGED_BEACON_ENABLED);

        /* remove beacon */
        bcn = sdata_dereference(ifmsh->beacon, sdata);
        RCU_INIT_POINTER(ifmsh->beacon, NULL);
        kfree_rcu(bcn, rcu_head);

        /* free all potentially still buffered group-addressed frames */
        local->total_ps_buffered -= skb_queue_len(&ifmsh->ps.bc_buf);
        skb_queue_purge(&ifmsh->ps.bc_buf);

        timer_delete_sync(&sdata->u.mesh.housekeeping_timer);
        timer_delete_sync(&sdata->u.mesh.mesh_path_root_timer);
        timer_delete_sync(&sdata->u.mesh.mesh_path_timer);

        /* clear any mesh work (for next join) we may have accrued */
        ifmsh->wrkq_flags = 0;
        memset(ifmsh->mbss_changed, 0, sizeof(ifmsh->mbss_changed));

        local->fif_other_bss--;
        atomic_dec(&local->iff_allmultis);
        ieee80211_configure_filter(local);
}

static void ieee80211_mesh_csa_mark_radar(struct ieee80211_sub_if_data *sdata)
{
        int err;

        /* if the current channel is a DFS channel, mark the channel as
         * unavailable.
         */
        err = cfg80211_chandef_dfs_required(sdata->local->hw.wiphy,
                                            &sdata->vif.bss_conf.chanreq.oper,
                                            NL80211_IFTYPE_MESH_POINT);
        if (err > 0)
                cfg80211_radar_event(sdata->local->hw.wiphy,
                                     &sdata->vif.bss_conf.chanreq.oper,
                                     GFP_ATOMIC);
}

static bool
ieee80211_mesh_process_chnswitch(struct ieee80211_sub_if_data *sdata,
                                 struct ieee802_11_elems *elems, bool beacon)
{
        struct cfg80211_csa_settings params;
        struct ieee80211_csa_ie csa_ie;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_supported_band *sband;
        int err;
        struct ieee80211_conn_settings conn = ieee80211_conn_settings_unlimited;
        u32 vht_cap_info = 0;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return false;

        switch (sdata->vif.bss_conf.chanreq.oper.width) {
        case NL80211_CHAN_WIDTH_20_NOHT:
                conn.mode = IEEE80211_CONN_MODE_LEGACY;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        case NL80211_CHAN_WIDTH_20:
                conn.mode = IEEE80211_CONN_MODE_HT;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        case NL80211_CHAN_WIDTH_40:
                conn.mode = IEEE80211_CONN_MODE_HT;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_40;
                break;
        default:
                break;
        }

        if (elems->vht_cap_elem)
                vht_cap_info =
                        le32_to_cpu(elems->vht_cap_elem->vht_cap_info);

        memset(&params, 0, sizeof(params));
        err = ieee80211_parse_ch_switch_ie(sdata, elems, sband->band,
                                           vht_cap_info, &conn,
                                           sdata->vif.addr, false,
                                           &csa_ie);
        if (err < 0)
                return false;
        if (err)
                return false;

        /* Mark the channel unavailable if the reason for the switch is
         * regulatory.
         */
        if (csa_ie.reason_code == WLAN_REASON_MESH_CHAN_REGULATORY)
                ieee80211_mesh_csa_mark_radar(sdata);

        params.chandef = csa_ie.chanreq.oper;
        params.count = csa_ie.count;

        if (!cfg80211_chandef_usable(sdata->local->hw.wiphy, &params.chandef,
                                     IEEE80211_CHAN_DISABLED) ||
            !cfg80211_reg_can_beacon(sdata->local->hw.wiphy, &params.chandef,
                                     NL80211_IFTYPE_MESH_POINT)) {
                sdata_info(sdata,
                           "mesh STA %pM switches to unsupported channel (%d MHz, width:%d, CF1/2: %d/%d MHz), aborting\n",
                           sdata->vif.addr,
                           params.chandef.chan->center_freq,
                           params.chandef.width,
                           params.chandef.center_freq1,
                           params.chandef.center_freq2);
                return false;
        }

        err = cfg80211_chandef_dfs_required(sdata->local->hw.wiphy,
                                            &params.chandef,
                                            NL80211_IFTYPE_MESH_POINT);
        if (err < 0)
                return false;
        if (err > 0 && !ifmsh->userspace_handles_dfs) {
                sdata_info(sdata,
                           "mesh STA %pM switches to channel requiring DFS (%d MHz, width:%d, CF1/2: %d/%d MHz), aborting\n",
                           sdata->vif.addr,
                           params.chandef.chan->center_freq,
                           params.chandef.width,
                           params.chandef.center_freq1,
                           params.chandef.center_freq2);
                return false;
        }

        params.radar_required = err;

        if (cfg80211_chandef_identical(&params.chandef,
                                       &sdata->vif.bss_conf.chanreq.oper)) {
                mcsa_dbg(sdata,
                         "received csa with an identical chandef, ignoring\n");
                return true;
        }

        mcsa_dbg(sdata,
                 "received channel switch announcement to go to channel %d MHz\n",
                 params.chandef.chan->center_freq);

        params.block_tx = csa_ie.mode & WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT;
        if (beacon) {
                ifmsh->chsw_ttl = csa_ie.ttl - 1;
                if (ifmsh->pre_value >= csa_ie.pre_value)
                        return false;
                ifmsh->pre_value = csa_ie.pre_value;
        }

        if (ifmsh->chsw_ttl >= ifmsh->mshcfg.dot11MeshTTL)
                return false;

        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_REPEATER;

        if (ieee80211_channel_switch(sdata->local->hw.wiphy, sdata->dev,
                                     &params) < 0)
                return false;

        return true;
}

static void
ieee80211_mesh_rx_probe_req(struct ieee80211_sub_if_data *sdata,
                            struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct sk_buff *presp;
        struct beacon_data *bcn;
        struct ieee80211_mgmt *hdr;
        struct ieee802_11_elems *elems;
        size_t baselen;
        u8 *pos;

        pos = mgmt->u.probe_req.variable;
        baselen = (u8 *) pos - (u8 *) mgmt;
        if (baselen > len)
                return;

        elems = ieee802_11_parse_elems(pos, len - baselen, false, NULL);
        if (!elems)
                return;

        if (!elems->mesh_id)
                goto free;

        /* 802.11-2012 10.1.4.3.2 */
        if ((!ether_addr_equal(mgmt->da, sdata->vif.addr) &&
             !is_broadcast_ether_addr(mgmt->da)) ||
            elems->ssid_len != 0)
                goto free;

        if (elems->mesh_id_len != 0 &&
            (elems->mesh_id_len != ifmsh->mesh_id_len ||
             memcmp(elems->mesh_id, ifmsh->mesh_id, ifmsh->mesh_id_len)))
                goto free;

        rcu_read_lock();
        bcn = rcu_dereference(ifmsh->beacon);

        if (!bcn)
                goto out;

        presp = dev_alloc_skb(local->tx_headroom +
                              bcn->head_len + bcn->tail_len);
        if (!presp)
                goto out;

        skb_reserve(presp, local->tx_headroom);
        skb_put_data(presp, bcn->head, bcn->head_len);
        skb_put_data(presp, bcn->tail, bcn->tail_len);
        hdr = (struct ieee80211_mgmt *) presp->data;
        hdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                         IEEE80211_STYPE_PROBE_RESP);
        memcpy(hdr->da, mgmt->sa, ETH_ALEN);
        IEEE80211_SKB_CB(presp)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
        ieee80211_tx_skb(sdata, presp);
out:
        rcu_read_unlock();
free:
        kfree(elems);
}

static void ieee80211_mesh_rx_bcn_presp(struct ieee80211_sub_if_data *sdata,
                                        u16 stype,
                                        struct ieee80211_mgmt *mgmt,
                                        size_t len,
                                        struct ieee80211_rx_status *rx_status)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee802_11_elems *elems;
        struct ieee80211_channel *channel;
        size_t baselen;
        int freq;
        enum nl80211_band band = rx_status->band;

        /* ignore ProbeResp to foreign address */
        if (stype == IEEE80211_STYPE_PROBE_RESP &&
            !ether_addr_equal(mgmt->da, sdata->vif.addr))
                return;

        baselen = (u8 *) mgmt->u.probe_resp.variable - (u8 *) mgmt;
        if (baselen > len)
                return;

        elems = ieee802_11_parse_elems(mgmt->u.probe_resp.variable,
                                       len - baselen,
                                       false, NULL);
        if (!elems)
                return;

        /* ignore non-mesh or secure / insecure mismatch */
        if ((!elems->mesh_id || !elems->mesh_config) ||
            (elems->rsn && sdata->u.mesh.security == IEEE80211_MESH_SEC_NONE) ||
            (!elems->rsn && sdata->u.mesh.security != IEEE80211_MESH_SEC_NONE))
                goto free;

        if (elems->ds_params)
                freq = ieee80211_channel_to_frequency(elems->ds_params[0], band);
        else
                freq = rx_status->freq;

        channel = ieee80211_get_channel(local->hw.wiphy, freq);

        if (!channel || channel->flags & IEEE80211_CHAN_DISABLED)
                goto free;

        if (mesh_matches_local(sdata, elems)) {
                mpl_dbg(sdata, "rssi_threshold=%d,rx_status->signal=%d\n",
                        sdata->u.mesh.mshcfg.rssi_threshold, rx_status->signal);
                if (!sdata->u.mesh.user_mpm ||
                    sdata->u.mesh.mshcfg.rssi_threshold == 0 ||
                    sdata->u.mesh.mshcfg.rssi_threshold < rx_status->signal)
                        mesh_neighbour_update(sdata, mgmt->sa, elems,
                                              rx_status);

                if (ifmsh->csa_role != IEEE80211_MESH_CSA_ROLE_INIT &&
                    !sdata->vif.bss_conf.csa_active)
                        ieee80211_mesh_process_chnswitch(sdata, elems, true);
        }

        if (ifmsh->sync_ops)
                ifmsh->sync_ops->rx_bcn_presp(sdata, stype, mgmt, len,
                                              elems->mesh_config, rx_status);
free:
        kfree(elems);
}

int ieee80211_mesh_finish_csa(struct ieee80211_sub_if_data *sdata, u64 *changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct mesh_csa_settings *tmp_csa_settings;
        int ret = 0;

        /* Reset the TTL value and Initiator flag */
        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE;
        ifmsh->chsw_ttl = 0;

        /* Remove the CSA and MCSP elements from the beacon */
        tmp_csa_settings = sdata_dereference(ifmsh->csa, sdata);
        RCU_INIT_POINTER(ifmsh->csa, NULL);
        if (tmp_csa_settings)
                kfree_rcu(tmp_csa_settings, rcu_head);
        ret = ieee80211_mesh_rebuild_beacon(sdata);
        if (ret)
                return -EINVAL;

        *changed |= BSS_CHANGED_BEACON;

        mcsa_dbg(sdata, "complete switching to center freq %d MHz",
                 sdata->vif.bss_conf.chanreq.oper.chan->center_freq);
        return 0;
}

int ieee80211_mesh_csa_beacon(struct ieee80211_sub_if_data *sdata,
                              struct cfg80211_csa_settings *csa_settings,
                              u64 *changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct mesh_csa_settings *tmp_csa_settings;
        int ret = 0;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        tmp_csa_settings = kmalloc(sizeof(*tmp_csa_settings),
                                   GFP_ATOMIC);
        if (!tmp_csa_settings)
                return -ENOMEM;

        memcpy(&tmp_csa_settings->settings, csa_settings,
               sizeof(struct cfg80211_csa_settings));

        rcu_assign_pointer(ifmsh->csa, tmp_csa_settings);

        ret = ieee80211_mesh_rebuild_beacon(sdata);
        if (ret) {
                tmp_csa_settings = rcu_dereference(ifmsh->csa);
                RCU_INIT_POINTER(ifmsh->csa, NULL);
                kfree_rcu(tmp_csa_settings, rcu_head);
                return ret;
        }

        *changed |= BSS_CHANGED_BEACON;
        return 0;
}

static int mesh_fwd_csa_frame(struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_mgmt *mgmt, size_t len,
                               struct ieee802_11_elems *elems)
{
        struct ieee80211_mgmt *mgmt_fwd;
        struct sk_buff *skb;
        struct ieee80211_local *local = sdata->local;

        skb = dev_alloc_skb(local->tx_headroom + len);
        if (!skb)
                return -ENOMEM;
        skb_reserve(skb, local->tx_headroom);
        mgmt_fwd = skb_put(skb, len);

        elems->mesh_chansw_params_ie->mesh_ttl--;
        elems->mesh_chansw_params_ie->mesh_flags &=
                ~WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR;

        memcpy(mgmt_fwd, mgmt, len);
        eth_broadcast_addr(mgmt_fwd->da);
        memcpy(mgmt_fwd->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt_fwd->bssid, sdata->vif.addr, ETH_ALEN);

        ieee80211_tx_skb(sdata, skb);
        return 0;
}

static void mesh_rx_csa_frame(struct ieee80211_sub_if_data *sdata,
                              struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee802_11_elems *elems;
        u16 pre_value;
        bool fwd_csa = true;
        size_t baselen;
        u8 *pos;

        if (mgmt->u.action.u.measurement.action_code !=
            WLAN_ACTION_SPCT_CHL_SWITCH)
                return;

        pos = mgmt->u.action.u.chan_switch.variable;
        baselen = offsetof(struct ieee80211_mgmt,
                           u.action.u.chan_switch.variable);
        elems = ieee802_11_parse_elems(pos, len - baselen, true, NULL);
        if (!elems)
                return;

        if (!mesh_matches_local(sdata, elems))
                goto free;

        ifmsh->chsw_ttl = elems->mesh_chansw_params_ie->mesh_ttl;
        if (!--ifmsh->chsw_ttl)
                fwd_csa = false;

        pre_value = le16_to_cpu(elems->mesh_chansw_params_ie->mesh_pre_value);
        if (ifmsh->pre_value >= pre_value)
                goto free;

        ifmsh->pre_value = pre_value;

        if (!sdata->vif.bss_conf.csa_active &&
            !ieee80211_mesh_process_chnswitch(sdata, elems, false)) {
                mcsa_dbg(sdata, "Failed to process CSA action frame");
                goto free;
        }

        /* forward or re-broadcast the CSA frame */
        if (fwd_csa) {
                if (mesh_fwd_csa_frame(sdata, mgmt, len, elems) < 0)
                        mcsa_dbg(sdata, "Failed to forward the CSA frame");
        }
free:
        kfree(elems);
}

static void ieee80211_mesh_rx_mgmt_action(struct ieee80211_sub_if_data *sdata,
                                          struct ieee80211_mgmt *mgmt,
                                          size_t len,
                                          struct ieee80211_rx_status *rx_status)
{
        switch (mgmt->u.action.category) {
        case WLAN_CATEGORY_SELF_PROTECTED:
                switch (mgmt->u.action.u.self_prot.action_code) {
                case WLAN_SP_MESH_PEERING_OPEN:
                case WLAN_SP_MESH_PEERING_CLOSE:
                case WLAN_SP_MESH_PEERING_CONFIRM:
                        mesh_rx_plink_frame(sdata, mgmt, len, rx_status);
                        break;
                }
                break;
        case WLAN_CATEGORY_MESH_ACTION:
                if (mesh_action_is_path_sel(mgmt))
                        mesh_rx_path_sel_frame(sdata, mgmt, len);
                break;
        case WLAN_CATEGORY_SPECTRUM_MGMT:
                mesh_rx_csa_frame(sdata, mgmt, len);
                break;
        }
}

void ieee80211_mesh_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
                                   struct sk_buff *skb)
{
        struct ieee80211_rx_status *rx_status;
        struct ieee80211_mgmt *mgmt;
        u16 stype;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        /* mesh already went down */
        if (!sdata->u.mesh.mesh_id_len)
                return;

        rx_status = IEEE80211_SKB_RXCB(skb);
        mgmt = (struct ieee80211_mgmt *) skb->data;
        stype = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE;

        switch (stype) {
        case IEEE80211_STYPE_PROBE_RESP:
        case IEEE80211_STYPE_BEACON:
                ieee80211_mesh_rx_bcn_presp(sdata, stype, mgmt, skb->len,
                                            rx_status);
                break;
        case IEEE80211_STYPE_PROBE_REQ:
                ieee80211_mesh_rx_probe_req(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_ACTION:
                ieee80211_mesh_rx_mgmt_action(sdata, mgmt, skb->len, rx_status);
                break;
        }
}

static void mesh_bss_info_changed(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 bit;
        u64 changed = 0;

        for_each_set_bit(bit, ifmsh->mbss_changed,
                         sizeof(changed) * BITS_PER_BYTE) {
                clear_bit(bit, ifmsh->mbss_changed);
                changed |= BIT(bit);
        }

        if (sdata->vif.bss_conf.enable_beacon &&
            (changed & (BSS_CHANGED_BEACON |
                        BSS_CHANGED_HT |
                        BSS_CHANGED_BASIC_RATES |
                        BSS_CHANGED_BEACON_INT)))
                if (ieee80211_mesh_rebuild_beacon(sdata))
                        return;

        ieee80211_link_info_change_notify(sdata, &sdata->deflink, changed);
}

void ieee80211_mesh_work(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        /* mesh already went down */
        if (!sdata->u.mesh.mesh_id_len)
                return;

        if (ifmsh->preq_queue_len &&
            time_after(jiffies,
                       ifmsh->last_preq + msecs_to_jiffies(ifmsh->mshcfg.dot11MeshHWMPpreqMinInterval)))
                mesh_path_start_discovery(sdata);

        if (test_and_clear_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags))
                ieee80211_mesh_housekeeping(sdata);

        if (test_and_clear_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags))
                ieee80211_mesh_rootpath(sdata);

        if (test_and_clear_bit(MESH_WORK_DRIFT_ADJUST, &ifmsh->wrkq_flags))
                mesh_sync_adjust_tsf(sdata);

        if (test_and_clear_bit(MESH_WORK_MBSS_CHANGED, &ifmsh->wrkq_flags))
                mesh_bss_info_changed(sdata);
}


void ieee80211_mesh_init_sdata(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        static u8 zero_addr[ETH_ALEN] = {};

        timer_setup(&ifmsh->housekeeping_timer,
                    ieee80211_mesh_housekeeping_timer, 0);

        ifmsh->accepting_plinks = true;
        atomic_set(&ifmsh->mpaths, 0);
        mesh_rmc_init(sdata);
        ifmsh->last_preq = jiffies;
        ifmsh->next_perr = jiffies;
        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE;
        ifmsh->nonpeer_pm = NL80211_MESH_POWER_ACTIVE;
        /* Allocate all mesh structures when creating the first mesh interface. */
        if (!mesh_allocated)
                ieee80211s_init();

        mesh_pathtbl_init(sdata);

        timer_setup(&ifmsh->mesh_path_timer, ieee80211_mesh_path_timer, 0);
        timer_setup(&ifmsh->mesh_path_root_timer,
                    ieee80211_mesh_path_root_timer, 0);
        INIT_LIST_HEAD(&ifmsh->preq_queue.list);
        skb_queue_head_init(&ifmsh->ps.bc_buf);
        spin_lock_init(&ifmsh->mesh_preq_queue_lock);
        spin_lock_init(&ifmsh->sync_offset_lock);
        RCU_INIT_POINTER(ifmsh->beacon, NULL);

        sdata->vif.bss_conf.bssid = zero_addr;
}

void ieee80211_mesh_teardown_sdata(struct ieee80211_sub_if_data *sdata)
{
        mesh_rmc_free(sdata);
        mesh_pathtbl_unregister(sdata);
}


























    3 
    3 
    3 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_TC_CSUM_H
#define __NET_TC_CSUM_H

#include <linux/types.h>
#include <net/act_api.h>
#include <linux/tc_act/tc_csum.h>

struct tcf_csum_params {
        u32 update_flags;
        int action;
        struct rcu_head rcu;
};

struct tcf_csum {
        struct tc_action common;

        struct tcf_csum_params __rcu *params;
};
#define to_tcf_csum(a) ((struct tcf_csum *)a)

static inline u32 tcf_csum_update_flags(const struct tc_action *a)
{
        u32 update_flags;

        rcu_read_lock();
        update_flags = rcu_dereference(to_tcf_csum(a)->params)->update_flags;
        rcu_read_unlock();

        return update_flags;
}

#endif /* __NET_TC_CSUM_H */
















































   22 



















   23 
   23 
   23 
    4 




























































































































































































































   22 

























   23 



   23 









    2 













































































   23 



    2 


   22 










































































   23 












   23 







































   23 



























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2013 Politecnico di Torino, Italy
 *                    TORSEC group -- https://security.polito.it
 *
 * Author: Roberto Sassu <roberto.sassu@polito.it>
 *
 * File: ima_template_lib.c
 *      Library of supported template fields.
 */

#include "ima_template_lib.h"
#include <linux/xattr.h>
#include <linux/evm.h>

static bool ima_template_hash_algo_allowed(u8 algo)
{
        if (algo == HASH_ALGO_SHA1 || algo == HASH_ALGO_MD5)
                return true;

        return false;
}

enum data_formats {
        DATA_FMT_DIGEST = 0,
        DATA_FMT_DIGEST_WITH_ALGO,
        DATA_FMT_DIGEST_WITH_TYPE_AND_ALGO,
        DATA_FMT_STRING,
        DATA_FMT_HEX,
        DATA_FMT_UINT
};

enum digest_type {
        DIGEST_TYPE_IMA,
        DIGEST_TYPE_VERITY,
        DIGEST_TYPE__LAST
};

#define DIGEST_TYPE_NAME_LEN_MAX 7        /* including NUL */
static const char * const digest_type_name[DIGEST_TYPE__LAST] = {
        [DIGEST_TYPE_IMA] = "ima",
        [DIGEST_TYPE_VERITY] = "verity"
};

static int ima_write_template_field_data(const void *data, const u32 datalen,
                                         enum data_formats datafmt,
                                         struct ima_field_data *field_data)
{
        u8 *buf, *buf_ptr;
        u32 buflen = datalen;

        if (datafmt == DATA_FMT_STRING)
                buflen = datalen + 1;

        buf = kzalloc(buflen, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        memcpy(buf, data, datalen);

        /*
         * Replace all space characters with underscore for event names and
         * strings. This avoid that, during the parsing of a measurements list,
         * filenames with spaces or that end with the suffix ' (deleted)' are
         * split into multiple template fields (the space is the delimitator
         * character for measurements lists in ASCII format).
         */
        if (datafmt == DATA_FMT_STRING) {
                for (buf_ptr = buf; buf_ptr - buf < datalen; buf_ptr++)
                        if (*buf_ptr == ' ')
                                *buf_ptr = '_';
        }

        field_data->data = buf;
        field_data->len = buflen;
        return 0;
}

static void ima_show_template_data_ascii(struct seq_file *m,
                                         enum ima_show_type show,
                                         enum data_formats datafmt,
                                         struct ima_field_data *field_data)
{
        u8 *buf_ptr = field_data->data;
        u32 buflen = field_data->len;

        switch (datafmt) {
        case DATA_FMT_DIGEST_WITH_TYPE_AND_ALGO:
        case DATA_FMT_DIGEST_WITH_ALGO:
                buf_ptr = strrchr(field_data->data, ':');
                if (buf_ptr != field_data->data)
                        seq_printf(m, "%s", field_data->data);

                /* skip ':' and '\0' */
                buf_ptr += 2;
                buflen -= buf_ptr - field_data->data;
                fallthrough;
        case DATA_FMT_DIGEST:
        case DATA_FMT_HEX:
                if (!buflen)
                        break;
                ima_print_digest(m, buf_ptr, buflen);
                break;
        case DATA_FMT_STRING:
                seq_printf(m, "%s", buf_ptr);
                break;
        case DATA_FMT_UINT:
                switch (field_data->len) {
                case sizeof(u8):
                        seq_printf(m, "%u", *(u8 *)buf_ptr);
                        break;
                case sizeof(u16):
                        if (ima_canonical_fmt)
                                seq_printf(m, "%u",
                                           le16_to_cpu(*(__le16 *)buf_ptr));
                        else
                                seq_printf(m, "%u", *(u16 *)buf_ptr);
                        break;
                case sizeof(u32):
                        if (ima_canonical_fmt)
                                seq_printf(m, "%u",
                                           le32_to_cpu(*(__le32 *)buf_ptr));
                        else
                                seq_printf(m, "%u", *(u32 *)buf_ptr);
                        break;
                case sizeof(u64):
                        if (ima_canonical_fmt)
                                seq_printf(m, "%llu",
                                           le64_to_cpu(*(__le64 *)buf_ptr));
                        else
                                seq_printf(m, "%llu", *(u64 *)buf_ptr);
                        break;
                default:
                        break;
                }
                break;
        default:
                break;
        }
}

static void ima_show_template_data_binary(struct seq_file *m,
                                          enum ima_show_type show,
                                          enum data_formats datafmt,
                                          struct ima_field_data *field_data)
{
        u32 len = (show == IMA_SHOW_BINARY_OLD_STRING_FMT) ?
            strlen(field_data->data) : field_data->len;

        if (show != IMA_SHOW_BINARY_NO_FIELD_LEN) {
                u32 field_len = !ima_canonical_fmt ?
                                len : (__force u32)cpu_to_le32(len);

                ima_putc(m, &field_len, sizeof(field_len));
        }

        if (!len)
                return;

        ima_putc(m, field_data->data, len);
}

static void ima_show_template_field_data(struct seq_file *m,
                                         enum ima_show_type show,
                                         enum data_formats datafmt,
                                         struct ima_field_data *field_data)
{
        switch (show) {
        case IMA_SHOW_ASCII:
                ima_show_template_data_ascii(m, show, datafmt, field_data);
                break;
        case IMA_SHOW_BINARY:
        case IMA_SHOW_BINARY_NO_FIELD_LEN:
        case IMA_SHOW_BINARY_OLD_STRING_FMT:
                ima_show_template_data_binary(m, show, datafmt, field_data);
                break;
        default:
                break;
        }
}

void ima_show_template_digest(struct seq_file *m, enum ima_show_type show,
                              struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_DIGEST, field_data);
}

void ima_show_template_digest_ng(struct seq_file *m, enum ima_show_type show,
                                 struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_DIGEST_WITH_ALGO,
                                     field_data);
}

void ima_show_template_digest_ngv2(struct seq_file *m, enum ima_show_type show,
                                   struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show,
                                     DATA_FMT_DIGEST_WITH_TYPE_AND_ALGO,
                                     field_data);
}

void ima_show_template_string(struct seq_file *m, enum ima_show_type show,
                              struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_STRING, field_data);
}

void ima_show_template_sig(struct seq_file *m, enum ima_show_type show,
                           struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_HEX, field_data);
}

void ima_show_template_buf(struct seq_file *m, enum ima_show_type show,
                           struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_HEX, field_data);
}

void ima_show_template_uint(struct seq_file *m, enum ima_show_type show,
                            struct ima_field_data *field_data)
{
        ima_show_template_field_data(m, show, DATA_FMT_UINT, field_data);
}

/**
 * ima_parse_buf() - Parses lengths and data from an input buffer
 * @bufstartp:       Buffer start address.
 * @bufendp:         Buffer end address.
 * @bufcurp:         Pointer to remaining (non-parsed) data.
 * @maxfields:       Length of fields array.
 * @fields:          Array containing lengths and pointers of parsed data.
 * @curfields:       Number of array items containing parsed data.
 * @len_mask:        Bitmap (if bit is set, data length should not be parsed).
 * @enforce_mask:    Check if curfields == maxfields and/or bufcurp == bufendp.
 * @bufname:         String identifier of the input buffer.
 *
 * Return: 0 on success, -EINVAL on error.
 */
int ima_parse_buf(void *bufstartp, void *bufendp, void **bufcurp,
                  int maxfields, struct ima_field_data *fields, int *curfields,
                  unsigned long *len_mask, int enforce_mask, char *bufname)
{
        void *bufp = bufstartp;
        int i;

        for (i = 0; i < maxfields; i++) {
                if (len_mask == NULL || !test_bit(i, len_mask)) {
                        if (bufp > (bufendp - sizeof(u32)))
                                break;

                        if (ima_canonical_fmt)
                                fields[i].len = le32_to_cpu(*(__le32 *)bufp);
                        else
                                fields[i].len = *(u32 *)bufp;

                        bufp += sizeof(u32);
                }

                if (bufp > (bufendp - fields[i].len))
                        break;

                fields[i].data = bufp;
                bufp += fields[i].len;
        }

        if ((enforce_mask & ENFORCE_FIELDS) && i != maxfields) {
                pr_err("%s: nr of fields mismatch: expected: %d, current: %d\n",
                       bufname, maxfields, i);
                return -EINVAL;
        }

        if ((enforce_mask & ENFORCE_BUFEND) && bufp != bufendp) {
                pr_err("%s: buf end mismatch: expected: %p, current: %p\n",
                       bufname, bufendp, bufp);
                return -EINVAL;
        }

        if (curfields)
                *curfields = i;

        if (bufcurp)
                *bufcurp = bufp;

        return 0;
}

static int ima_eventdigest_init_common(const u8 *digest, u32 digestsize,
                                       u8 digest_type, u8 hash_algo,
                                       struct ima_field_data *field_data)
{
        /*
         * digest formats:
         *  - DATA_FMT_DIGEST: digest
         *  - DATA_FMT_DIGEST_WITH_ALGO: <hash algo> + ':' + '\0' + digest,
         *  - DATA_FMT_DIGEST_WITH_TYPE_AND_ALGO:
         *        <digest type> + ':' + <hash algo> + ':' + '\0' + digest,
         *
         *    where 'DATA_FMT_DIGEST' is the original digest format ('d')
         *      with a hash size limitation of 20 bytes,
         *    where <digest type> is either "ima" or "verity",
         *    where <hash algo> is the hash_algo_name[] string.
         */
        u8 buffer[DIGEST_TYPE_NAME_LEN_MAX + CRYPTO_MAX_ALG_NAME + 2 +
                IMA_MAX_DIGEST_SIZE] = { 0 };
        enum data_formats fmt = DATA_FMT_DIGEST;
        u32 offset = 0;

        if (digest_type < DIGEST_TYPE__LAST && hash_algo < HASH_ALGO__LAST) {
                fmt = DATA_FMT_DIGEST_WITH_TYPE_AND_ALGO;
                offset += 1 + sprintf(buffer, "%s:%s:",
                                      digest_type_name[digest_type],
                                      hash_algo_name[hash_algo]);
        } else if (hash_algo < HASH_ALGO__LAST) {
                fmt = DATA_FMT_DIGEST_WITH_ALGO;
                offset += 1 + sprintf(buffer, "%s:",
                                      hash_algo_name[hash_algo]);
        }

        if (digest) {
                memcpy(buffer + offset, digest, digestsize);
        } else {
                /*
                 * If digest is NULL, the event being recorded is a violation.
                 * Make room for the digest by increasing the offset by the
                 * hash algorithm digest size. If the hash algorithm is not
                 * specified increase the offset by IMA_DIGEST_SIZE which
                 * fits SHA1 or MD5
                 */
                if (hash_algo < HASH_ALGO__LAST)
                        offset += hash_digest_size[hash_algo];
                else
                        offset += IMA_DIGEST_SIZE;
        }

        return ima_write_template_field_data(buffer, offset + digestsize,
                                             fmt, field_data);
}

/*
 * This function writes the digest of an event (with size limit).
 */
int ima_eventdigest_init(struct ima_event_data *event_data,
                         struct ima_field_data *field_data)
{
        struct ima_max_digest_data hash;
        struct ima_digest_data *hash_hdr = container_of(&hash.hdr,
                                                struct ima_digest_data, hdr);
        u8 *cur_digest = NULL;
        u32 cur_digestsize = 0;
        struct inode *inode;
        int result;

        memset(&hash, 0, sizeof(hash));

        if (event_data->violation)        /* recording a violation. */
                goto out;

        if (ima_template_hash_algo_allowed(event_data->iint->ima_hash->algo)) {
                cur_digest = event_data->iint->ima_hash->digest;
                cur_digestsize = event_data->iint->ima_hash->length;
                goto out;
        }

        if ((const char *)event_data->filename == boot_aggregate_name) {
                if (ima_tpm_chip) {
                        hash.hdr.algo = HASH_ALGO_SHA1;
                        result = ima_calc_boot_aggregate(hash_hdr);

                        /* algo can change depending on available PCR banks */
                        if (!result && hash.hdr.algo != HASH_ALGO_SHA1)
                                result = -EINVAL;

                        if (result < 0)
                                memset(&hash, 0, sizeof(hash));
                }

                cur_digest = hash_hdr->digest;
                cur_digestsize = hash_digest_size[HASH_ALGO_SHA1];
                goto out;
        }

        if (!event_data->file)        /* missing info to re-calculate the digest */
                return -EINVAL;

        inode = file_inode(event_data->file);
        hash.hdr.algo = ima_template_hash_algo_allowed(ima_hash_algo) ?
            ima_hash_algo : HASH_ALGO_SHA1;
        result = ima_calc_file_hash(event_data->file, hash_hdr);
        if (result) {
                integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
                                    event_data->filename, "collect_data",
                                    "failed", result, 0);
                return result;
        }
        cur_digest = hash_hdr->digest;
        cur_digestsize = hash.hdr.length;
out:
        return ima_eventdigest_init_common(cur_digest, cur_digestsize,
                                           DIGEST_TYPE__LAST, HASH_ALGO__LAST,
                                           field_data);
}

/*
 * This function writes the digest of an event (without size limit).
 */
int ima_eventdigest_ng_init(struct ima_event_data *event_data,
                            struct ima_field_data *field_data)
{
        u8 *cur_digest = NULL, hash_algo = ima_hash_algo;
        u32 cur_digestsize = 0;

        if (event_data->violation)        /* recording a violation. */
                goto out;

        cur_digest = event_data->iint->ima_hash->digest;
        cur_digestsize = event_data->iint->ima_hash->length;

        hash_algo = event_data->iint->ima_hash->algo;
out:
        return ima_eventdigest_init_common(cur_digest, cur_digestsize,
                                           DIGEST_TYPE__LAST, hash_algo,
                                           field_data);
}

/*
 * This function writes the digest of an event (without size limit),
 * prefixed with both the digest type and hash algorithm.
 */
int ima_eventdigest_ngv2_init(struct ima_event_data *event_data,
                              struct ima_field_data *field_data)
{
        u8 *cur_digest = NULL, hash_algo = ima_hash_algo;
        u32 cur_digestsize = 0;
        u8 digest_type = DIGEST_TYPE_IMA;

        if (event_data->violation)        /* recording a violation. */
                goto out;

        cur_digest = event_data->iint->ima_hash->digest;
        cur_digestsize = event_data->iint->ima_hash->length;

        hash_algo = event_data->iint->ima_hash->algo;
        if (event_data->iint->flags & IMA_VERITY_REQUIRED)
                digest_type = DIGEST_TYPE_VERITY;
out:
        return ima_eventdigest_init_common(cur_digest, cur_digestsize,
                                           digest_type, hash_algo,
                                           field_data);
}

/*
 * This function writes the digest of the file which is expected to match the
 * digest contained in the file's appended signature.
 */
int ima_eventdigest_modsig_init(struct ima_event_data *event_data,
                                struct ima_field_data *field_data)
{
        enum hash_algo hash_algo;
        const u8 *cur_digest;
        u32 cur_digestsize;

        if (!event_data->modsig)
                return 0;

        if (event_data->violation) {
                /* Recording a violation. */
                hash_algo = HASH_ALGO_SHA1;
                cur_digest = NULL;
                cur_digestsize = 0;
        } else {
                int rc;

                rc = ima_get_modsig_digest(event_data->modsig, &hash_algo,
                                           &cur_digest, &cur_digestsize);
                if (rc)
                        return rc;
                else if (hash_algo == HASH_ALGO__LAST || cur_digestsize == 0)
                        /* There was some error collecting the digest. */
                        return -EINVAL;
        }

        return ima_eventdigest_init_common(cur_digest, cur_digestsize,
                                           DIGEST_TYPE__LAST, hash_algo,
                                           field_data);
}

static int ima_eventname_init_common(struct ima_event_data *event_data,
                                     struct ima_field_data *field_data,
                                     bool size_limit)
{
        const char *cur_filename = NULL;
        struct name_snapshot filename;
        u32 cur_filename_len = 0;
        bool snapshot = false;
        int ret;

        BUG_ON(event_data->filename == NULL && event_data->file == NULL);

        if (event_data->filename) {
                cur_filename = event_data->filename;
                cur_filename_len = strlen(event_data->filename);

                if (!size_limit || cur_filename_len <= IMA_EVENT_NAME_LEN_MAX)
                        goto out;
        }

        if (event_data->file) {
                take_dentry_name_snapshot(&filename,
                                          event_data->file->f_path.dentry);
                snapshot = true;
                cur_filename = filename.name.name;
                cur_filename_len = strlen(cur_filename);
        } else
                /*
                 * Truncate filename if the latter is too long and
                 * the file descriptor is not available.
                 */
                cur_filename_len = IMA_EVENT_NAME_LEN_MAX;
out:
        ret = ima_write_template_field_data(cur_filename, cur_filename_len,
                                            DATA_FMT_STRING, field_data);

        if (snapshot)
                release_dentry_name_snapshot(&filename);

        return ret;
}

/*
 * This function writes the name of an event (with size limit).
 */
int ima_eventname_init(struct ima_event_data *event_data,
                       struct ima_field_data *field_data)
{
        return ima_eventname_init_common(event_data, field_data, true);
}

/*
 * This function writes the name of an event (without size limit).
 */
int ima_eventname_ng_init(struct ima_event_data *event_data,
                          struct ima_field_data *field_data)
{
        return ima_eventname_init_common(event_data, field_data, false);
}

/*
 *  ima_eventsig_init - include the file signature as part of the template data
 */
int ima_eventsig_init(struct ima_event_data *event_data,
                      struct ima_field_data *field_data)
{
        struct evm_ima_xattr_data *xattr_value = event_data->xattr_value;

        if (!xattr_value ||
            (xattr_value->type != EVM_IMA_XATTR_DIGSIG &&
             xattr_value->type != IMA_VERITY_DIGSIG))
                return ima_eventevmsig_init(event_data, field_data);

        return ima_write_template_field_data(xattr_value, event_data->xattr_len,
                                             DATA_FMT_HEX, field_data);
}

/*
 *  ima_eventbuf_init - include the buffer(kexec-cmldine) as part of the
 *  template data.
 */
int ima_eventbuf_init(struct ima_event_data *event_data,
                      struct ima_field_data *field_data)
{
        if ((!event_data->buf) || (event_data->buf_len == 0))
                return 0;

        return ima_write_template_field_data(event_data->buf,
                                             event_data->buf_len, DATA_FMT_HEX,
                                             field_data);
}

/*
 *  ima_eventmodsig_init - include the appended file signature as part of the
 *  template data
 */
int ima_eventmodsig_init(struct ima_event_data *event_data,
                         struct ima_field_data *field_data)
{
        const void *data;
        u32 data_len;
        int rc;

        if (!event_data->modsig)
                return 0;

        /*
         * modsig is a runtime structure containing pointers. Get its raw data
         * instead.
         */
        rc = ima_get_raw_modsig(event_data->modsig, &data, &data_len);
        if (rc)
                return rc;

        return ima_write_template_field_data(data, data_len, DATA_FMT_HEX,
                                             field_data);
}

/*
 *  ima_eventevmsig_init - include the EVM portable signature as part of the
 *  template data
 */
int ima_eventevmsig_init(struct ima_event_data *event_data,
                         struct ima_field_data *field_data)
{
        struct evm_ima_xattr_data *xattr_data = NULL;
        int rc = 0;

        if (!event_data->file)
                return 0;

        rc = vfs_getxattr_alloc(&nop_mnt_idmap, file_dentry(event_data->file),
                                XATTR_NAME_EVM, (char **)&xattr_data, 0,
                                GFP_NOFS);
        if (rc <= 0 || xattr_data->type != EVM_XATTR_PORTABLE_DIGSIG) {
                rc = 0;
                goto out;
        }

        rc = ima_write_template_field_data((char *)xattr_data, rc, DATA_FMT_HEX,
                                           field_data);

out:
        kfree(xattr_data);
        return rc;
}

static int ima_eventinodedac_init_common(struct ima_event_data *event_data,
                                         struct ima_field_data *field_data,
                                         bool get_uid)
{
        unsigned int id;

        if (!event_data->file)
                return 0;

        if (get_uid)
                id = i_uid_read(file_inode(event_data->file));
        else
                id = i_gid_read(file_inode(event_data->file));

        if (ima_canonical_fmt) {
                if (sizeof(id) == sizeof(u16))
                        id = (__force u16)cpu_to_le16(id);
                else
                        id = (__force u32)cpu_to_le32(id);
        }

        return ima_write_template_field_data((void *)&id, sizeof(id),
                                             DATA_FMT_UINT, field_data);
}

/*
 *  ima_eventinodeuid_init - include the inode UID as part of the template
 *  data
 */
int ima_eventinodeuid_init(struct ima_event_data *event_data,
                           struct ima_field_data *field_data)
{
        return ima_eventinodedac_init_common(event_data, field_data, true);
}

/*
 *  ima_eventinodegid_init - include the inode GID as part of the template
 *  data
 */
int ima_eventinodegid_init(struct ima_event_data *event_data,
                           struct ima_field_data *field_data)
{
        return ima_eventinodedac_init_common(event_data, field_data, false);
}

/*
 *  ima_eventinodemode_init - include the inode mode as part of the template
 *  data
 */
int ima_eventinodemode_init(struct ima_event_data *event_data,
                            struct ima_field_data *field_data)
{
        struct inode *inode;
        u16 mode;

        if (!event_data->file)
                return 0;

        inode = file_inode(event_data->file);
        mode = inode->i_mode;
        if (ima_canonical_fmt)
                mode = (__force u16)cpu_to_le16(mode);

        return ima_write_template_field_data((char *)&mode, sizeof(mode),
                                             DATA_FMT_UINT, field_data);
}

static int ima_eventinodexattrs_init_common(struct ima_event_data *event_data,
                                            struct ima_field_data *field_data,
                                            char type)
{
        u8 *buffer = NULL;
        int rc;

        if (!event_data->file)
                return 0;

        rc = evm_read_protected_xattrs(file_dentry(event_data->file), NULL, 0,
                                       type, ima_canonical_fmt);
        if (rc < 0)
                return 0;

        buffer = kmalloc(rc, GFP_KERNEL);
        if (!buffer)
                return 0;

        rc = evm_read_protected_xattrs(file_dentry(event_data->file), buffer,
                                       rc, type, ima_canonical_fmt);
        if (rc < 0) {
                rc = 0;
                goto out;
        }

        rc = ima_write_template_field_data((char *)buffer, rc, DATA_FMT_HEX,
                                           field_data);
out:
        kfree(buffer);
        return rc;
}

/*
 *  ima_eventinodexattrnames_init - include a list of xattr names as part of the
 *  template data
 */
int ima_eventinodexattrnames_init(struct ima_event_data *event_data,
                                  struct ima_field_data *field_data)
{
        return ima_eventinodexattrs_init_common(event_data, field_data, 'n');
}

/*
 *  ima_eventinodexattrlengths_init - include a list of xattr lengths as part of
 *  the template data
 */
int ima_eventinodexattrlengths_init(struct ima_event_data *event_data,
                                    struct ima_field_data *field_data)
{
        return ima_eventinodexattrs_init_common(event_data, field_data, 'l');
}

/*
 *  ima_eventinodexattrvalues_init - include a list of xattr values as part of
 *  the template data
 */
int ima_eventinodexattrvalues_init(struct ima_event_data *event_data,
                                   struct ima_field_data *field_data)
{
        return ima_eventinodexattrs_init_common(event_data, field_data, 'v');
}




















 2377 












 2158 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BH_H
#define _LINUX_BH_H

#include <linux/instruction_pointer.h>
#include <linux/preempt.h>

#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_TRACE_IRQFLAGS)
extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
#else
static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
{
        preempt_count_add(cnt);
        barrier();
}
#endif

static inline void local_bh_disable(void)
{
        __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

extern void _local_bh_enable(void);
extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt);

static inline void local_bh_enable_ip(unsigned long ip)
{
        __local_bh_enable_ip(ip, SOFTIRQ_DISABLE_OFFSET);
}

static inline void local_bh_enable(void)
{
        __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

#ifdef CONFIG_PREEMPT_RT
extern bool local_bh_blocked(void);
#else
static inline bool local_bh_blocked(void) { return false; }
#endif

#endif /* _LINUX_BH_H */





































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * linux/ipc/util.h
 * Copyright (C) 1999 Christoph Rohland
 *
 * ipc helper functions (c) 1999 Manfred Spraul <manfred@colorfullife.com>
 * namespaces support.      2006 OpenVZ, SWsoft Inc.
 *                               Pavel Emelianov <xemul@openvz.org>
 */

#ifndef _IPC_UTIL_H
#define _IPC_UTIL_H

#include <linux/unistd.h>
#include <linux/err.h>
#include <linux/ipc_namespace.h>
#include <linux/pid.h>

/*
 * The IPC ID contains 2 separate numbers - index and sequence number.
 * By default,
 *   bits  0-14: index (32k, 15 bits)
 *   bits 15-30: sequence number (64k, 16 bits)
 *
 * When IPCMNI extension mode is turned on, the composition changes:
 *   bits  0-23: index (16M, 24 bits)
 *   bits 24-30: sequence number (128, 7 bits)
 */
#define IPCMNI_SHIFT                15
#define IPCMNI_EXTEND_SHIFT        24
#define IPCMNI_EXTEND_MIN_CYCLE        (RADIX_TREE_MAP_SIZE * RADIX_TREE_MAP_SIZE)
#define IPCMNI                        (1 << IPCMNI_SHIFT)
#define IPCMNI_EXTEND                (1 << IPCMNI_EXTEND_SHIFT)

#ifdef CONFIG_SYSVIPC_SYSCTL
extern int ipc_mni;
extern int ipc_mni_shift;
extern int ipc_min_cycle;

#define ipcmni_seq_shift()        ipc_mni_shift
#define IPCMNI_IDX_MASK                ((1 << ipc_mni_shift) - 1)

#else /* CONFIG_SYSVIPC_SYSCTL */

#define ipc_mni                        IPCMNI
#define ipc_min_cycle                ((int)RADIX_TREE_MAP_SIZE)
#define ipcmni_seq_shift()        IPCMNI_SHIFT
#define IPCMNI_IDX_MASK                ((1 << IPCMNI_SHIFT) - 1)
#endif /* CONFIG_SYSVIPC_SYSCTL */

void sem_init(void);
void msg_init(void);
void shm_init(void);

struct ipc_namespace;
struct pid_namespace;

#ifdef CONFIG_POSIX_MQUEUE
extern void mq_clear_sbinfo(struct ipc_namespace *ns);
#else
static inline void mq_clear_sbinfo(struct ipc_namespace *ns) { }
#endif

#ifdef CONFIG_SYSVIPC
void sem_init_ns(struct ipc_namespace *ns);
int msg_init_ns(struct ipc_namespace *ns);
void shm_init_ns(struct ipc_namespace *ns);

void sem_exit_ns(struct ipc_namespace *ns);
void msg_exit_ns(struct ipc_namespace *ns);
void shm_exit_ns(struct ipc_namespace *ns);
#else
static inline void sem_init_ns(struct ipc_namespace *ns) { }
static inline int msg_init_ns(struct ipc_namespace *ns) { return 0; }
static inline void shm_init_ns(struct ipc_namespace *ns) { }

static inline void sem_exit_ns(struct ipc_namespace *ns) { }
static inline void msg_exit_ns(struct ipc_namespace *ns) { }
static inline void shm_exit_ns(struct ipc_namespace *ns) { }
#endif

/*
 * Structure that holds the parameters needed by the ipc operations
 * (see after)
 */
struct ipc_params {
        key_t key;
        int flg;
        union {
                size_t size;        /* for shared memories */
                int nsems;        /* for semaphores */
        } u;                        /* holds the getnew() specific param */
};

/*
 * Structure that holds some ipc operations. This structure is used to unify
 * the calls to sys_msgget(), sys_semget(), sys_shmget()
 *      . routine to call to create a new ipc object. Can be one of newque,
 *        newary, newseg
 *      . routine to call to check permissions for a new ipc object.
 *        Can be one of security_msg_associate, security_sem_associate,
 *        security_shm_associate
 *      . routine to call for an extra check if needed
 */
struct ipc_ops {
        int (*getnew)(struct ipc_namespace *, struct ipc_params *);
        int (*associate)(struct kern_ipc_perm *, int);
        int (*more_checks)(struct kern_ipc_perm *, struct ipc_params *);
};

struct seq_file;
struct ipc_ids;

void ipc_init_ids(struct ipc_ids *ids);
#ifdef CONFIG_PROC_FS
void __init ipc_init_proc_interface(const char *path, const char *header,
                int ids, int (*show)(struct seq_file *, void *));
struct pid_namespace *ipc_seq_pid_ns(struct seq_file *);
#else
#define ipc_init_proc_interface(path, header, ids, show) do {} while (0)
#endif

#define IPC_SEM_IDS        0
#define IPC_MSG_IDS        1
#define IPC_SHM_IDS        2

#define ipcid_to_idx(id)  ((id) & IPCMNI_IDX_MASK)
#define ipcid_to_seqx(id) ((id) >> ipcmni_seq_shift())
#define ipcid_seq_max()          (INT_MAX >> ipcmni_seq_shift())

/* must be called with ids->rwsem acquired for writing */
int ipc_addid(struct ipc_ids *, struct kern_ipc_perm *, int);

/* must be called with both locks acquired. */
void ipc_rmid(struct ipc_ids *, struct kern_ipc_perm *);

/* must be called with both locks acquired. */
void ipc_set_key_private(struct ipc_ids *, struct kern_ipc_perm *);

/* must be called with ipcp locked */
int ipcperms(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp, short flg);

/**
 * ipc_get_maxidx - get the highest assigned index
 * @ids: ipc identifier set
 *
 * The function returns the highest assigned index for @ids. The function
 * doesn't scan the idr tree, it uses a cached value.
 *
 * Called with ipc_ids.rwsem held for reading.
 */
static inline int ipc_get_maxidx(struct ipc_ids *ids)
{
        if (ids->in_use == 0)
                return -1;

        if (ids->in_use == ipc_mni)
                return ipc_mni - 1;

        return ids->max_idx;
}

/*
 * For allocation that need to be freed by RCU.
 * Objects are reference counted, they start with reference count 1.
 * getref increases the refcount, the putref call that reduces the recount
 * to 0 schedules the rcu destruction. Caller must guarantee locking.
 *
 * refcount is initialized by ipc_addid(), before that point call_rcu()
 * must be used.
 */
bool ipc_rcu_getref(struct kern_ipc_perm *ptr);
void ipc_rcu_putref(struct kern_ipc_perm *ptr,
                        void (*func)(struct rcu_head *head));

struct kern_ipc_perm *ipc_obtain_object_idr(struct ipc_ids *ids, int id);

void kernel_to_ipc64_perm(struct kern_ipc_perm *in, struct ipc64_perm *out);
void ipc64_perm_to_ipc_perm(struct ipc64_perm *in, struct ipc_perm *out);
int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out);
struct kern_ipc_perm *ipcctl_obtain_check(struct ipc_namespace *ns,
                                             struct ipc_ids *ids, int id, int cmd,
                                             struct ipc64_perm *perm, int extra_perm);

static inline void ipc_update_pid(struct pid **pos, struct pid *pid)
{
        struct pid *old = *pos;
        if (old != pid) {
                *pos = get_pid(pid);
                put_pid(old);
        }
}

#ifdef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
int ipc_parse_version(int *cmd);
#endif

extern void free_msg(struct msg_msg *msg);
extern struct msg_msg *load_msg(const void __user *src, size_t len);
extern struct msg_msg *copy_msg(struct msg_msg *src, struct msg_msg *dst);
extern int store_msg(void __user *dest, struct msg_msg *msg, size_t len);

static inline int ipc_checkid(struct kern_ipc_perm *ipcp, int id)
{
        return ipcid_to_seqx(id) != ipcp->seq;
}

static inline void ipc_lock_object(struct kern_ipc_perm *perm)
{
        spin_lock(&perm->lock);
}

static inline void ipc_unlock_object(struct kern_ipc_perm *perm)
{
        spin_unlock(&perm->lock);
}

static inline void ipc_assert_locked_object(struct kern_ipc_perm *perm)
{
        assert_spin_locked(&perm->lock);
}

static inline void ipc_unlock(struct kern_ipc_perm *perm)
{
        ipc_unlock_object(perm);
        rcu_read_unlock();
}

/*
 * ipc_valid_object() - helper to sort out IPC_RMID races for codepaths
 * where the respective ipc_ids.rwsem is not being held down.
 * Checks whether the ipc object is still around or if it's gone already, as
 * ipc_rmid() may have already freed the ID while the ipc lock was spinning.
 * Needs to be called with kern_ipc_perm.lock held -- exception made for one
 * checkpoint case at sys_semtimedop() as noted in code commentary.
 */
static inline bool ipc_valid_object(struct kern_ipc_perm *perm)
{
        return !perm->deleted;
}

struct kern_ipc_perm *ipc_obtain_object_check(struct ipc_ids *ids, int id);
int ipcget(struct ipc_namespace *ns, struct ipc_ids *ids,
                        const struct ipc_ops *ops, struct ipc_params *params);
void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
                void (*free)(struct ipc_namespace *, struct kern_ipc_perm *));

static inline int sem_check_semmni(struct ipc_namespace *ns) {
        /*
         * Check semmni range [0, ipc_mni]
         * semmni is the last element of sem_ctls[4] array
         */
        return ((ns->sem_ctls[3] < 0) || (ns->sem_ctls[3] > ipc_mni))
                ? -ERANGE : 0;
}

#ifdef CONFIG_COMPAT
#include <linux/compat.h>
struct compat_ipc_perm {
        key_t key;
        __compat_uid_t uid;
        __compat_gid_t gid;
        __compat_uid_t cuid;
        __compat_gid_t cgid;
        compat_mode_t mode;
        unsigned short seq;
};

void to_compat_ipc_perm(struct compat_ipc_perm *, struct ipc64_perm *);
void to_compat_ipc64_perm(struct compat_ipc64_perm *, struct ipc64_perm *);
int get_compat_ipc_perm(struct ipc64_perm *, struct compat_ipc_perm __user *);
int get_compat_ipc64_perm(struct ipc64_perm *,
                          struct compat_ipc64_perm __user *);

static inline int compat_ipc_parse_version(int *cmd)
{
        int version = *cmd & IPC_64;
        *cmd &= ~IPC_64;
        return version;
}

long compat_ksys_old_semctl(int semid, int semnum, int cmd, int arg);
long compat_ksys_old_msgctl(int msqid, int cmd, void __user *uptr);
long compat_ksys_msgrcv(int msqid, compat_uptr_t msgp, compat_ssize_t msgsz,
                        compat_long_t msgtyp, int msgflg);
long compat_ksys_msgsnd(int msqid, compat_uptr_t msgp,
                       compat_ssize_t msgsz, int msgflg);
long compat_ksys_old_shmctl(int shmid, int cmd, void __user *uptr);

#endif

#endif















































































































































































































































































































































































































































































































































































































    5 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
/*
 *
 * Author        Karsten Keil <kkeil@novell.com>
 *
 * Copyright 2008  by Karsten Keil <kkeil@novell.com>
 *
 * This code is free software; you can redistribute it and/or modify
 * it under the terms of the GNU LESSER GENERAL PUBLIC LICENSE
 * version 2.1 as published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU LESSER GENERAL PUBLIC LICENSE for more details.
 *
 */

#ifndef mISDNIF_H
#define mISDNIF_H

#include <linux/types.h>
#include <linux/errno.h>
#include <linux/socket.h>

/*
 * ABI Version 32 bit
 *
 * <8 bit> Major version
 *                - changed if any interface become backwards incompatible
 *
 * <8 bit> Minor version
 *              - changed if any interface is extended but backwards compatible
 *
 * <16 bit> Release number
 *              - should be incremented on every checkin
 */
#define        MISDN_MAJOR_VERSION        1
#define        MISDN_MINOR_VERSION        1
#define MISDN_RELEASE                29

/* primitives for information exchange
 * generell format
 * <16  bit  0 >
 * <8  bit command>
 *    BIT 8 = 1 LAYER private
 *    BIT 7 = 1 answer
 *    BIT 6 = 1 DATA
 * <8  bit target layer mask>
 *
 * Layer = 00 is reserved for general commands
   Layer = 01  L2 -> HW
   Layer = 02  HW -> L2
   Layer = 04  L3 -> L2
   Layer = 08  L2 -> L3
 * Layer = FF is reserved for broadcast commands
 */

#define MISDN_CMDMASK                0xff00
#define MISDN_LAYERMASK                0x00ff

/* generell commands */
#define OPEN_CHANNEL                0x0100
#define CLOSE_CHANNEL                0x0200
#define CONTROL_CHANNEL                0x0300
#define CHECK_DATA                0x0400

/* layer 2 -> layer 1 */
#define PH_ACTIVATE_REQ                0x0101
#define PH_DEACTIVATE_REQ        0x0201
#define PH_DATA_REQ                0x2001
#define MPH_ACTIVATE_REQ        0x0501
#define MPH_DEACTIVATE_REQ        0x0601
#define MPH_INFORMATION_REQ        0x0701
#define PH_CONTROL_REQ                0x0801

/* layer 1 -> layer 2 */
#define PH_ACTIVATE_IND                0x0102
#define PH_ACTIVATE_CNF                0x4102
#define PH_DEACTIVATE_IND        0x0202
#define PH_DEACTIVATE_CNF        0x4202
#define PH_DATA_IND                0x2002
#define PH_DATA_E_IND                0x3002
#define MPH_ACTIVATE_IND        0x0502
#define MPH_DEACTIVATE_IND        0x0602
#define MPH_INFORMATION_IND        0x0702
#define PH_DATA_CNF                0x6002
#define PH_CONTROL_IND                0x0802
#define PH_CONTROL_CNF                0x4802

/* layer 3 -> layer 2 */
#define DL_ESTABLISH_REQ        0x1004
#define DL_RELEASE_REQ                0x1104
#define DL_DATA_REQ                0x3004
#define DL_UNITDATA_REQ                0x3104
#define DL_INFORMATION_REQ        0x0004

/* layer 2 -> layer 3 */
#define DL_ESTABLISH_IND        0x1008
#define DL_ESTABLISH_CNF        0x5008
#define DL_RELEASE_IND                0x1108
#define DL_RELEASE_CNF                0x5108
#define DL_DATA_IND                0x3008
#define DL_UNITDATA_IND                0x3108
#define DL_INFORMATION_IND        0x0008

/* intern layer 2 management */
#define MDL_ASSIGN_REQ                0x1804
#define MDL_ASSIGN_IND                0x1904
#define MDL_REMOVE_REQ                0x1A04
#define MDL_REMOVE_IND                0x1B04
#define MDL_STATUS_UP_IND        0x1C04
#define MDL_STATUS_DOWN_IND        0x1D04
#define MDL_STATUS_UI_IND        0x1E04
#define MDL_ERROR_IND                0x1F04
#define MDL_ERROR_RSP                0x5F04

/* intern layer 2 */
#define DL_TIMER200_IND                0x7004
#define DL_TIMER203_IND                0x7304
#define DL_INTERN_MSG                0x7804

/* DL_INFORMATION_IND types */
#define DL_INFO_L2_CONNECT        0x0001
#define DL_INFO_L2_REMOVED        0x0002

/* PH_CONTROL types */
/* TOUCH TONE IS 0x20XX  XX "0"..."9", "A","B","C","D","*","#" */
#define DTMF_TONE_VAL                0x2000
#define DTMF_TONE_MASK                0x007F
#define DTMF_TONE_START                0x2100
#define DTMF_TONE_STOP                0x2200
#define DTMF_HFC_COEF                0x4000
#define DSP_CONF_JOIN                0x2403
#define DSP_CONF_SPLIT                0x2404
#define DSP_RECEIVE_OFF                0x2405
#define DSP_RECEIVE_ON                0x2406
#define DSP_ECHO_ON                0x2407
#define DSP_ECHO_OFF                0x2408
#define DSP_MIX_ON                0x2409
#define DSP_MIX_OFF                0x240a
#define DSP_DELAY                0x240b
#define DSP_JITTER                0x240c
#define DSP_TXDATA_ON                0x240d
#define DSP_TXDATA_OFF                0x240e
#define DSP_TX_DEJITTER                0x240f
#define DSP_TX_DEJ_OFF                0x2410
#define DSP_TONE_PATT_ON        0x2411
#define DSP_TONE_PATT_OFF        0x2412
#define DSP_VOL_CHANGE_TX        0x2413
#define DSP_VOL_CHANGE_RX        0x2414
#define DSP_BF_ENABLE_KEY        0x2415
#define DSP_BF_DISABLE                0x2416
#define DSP_BF_ACCEPT                0x2416
#define DSP_BF_REJECT                0x2417
#define DSP_PIPELINE_CFG        0x2418
#define HFC_VOL_CHANGE_TX        0x2601
#define HFC_VOL_CHANGE_RX        0x2602
#define HFC_SPL_LOOP_ON                0x2603
#define HFC_SPL_LOOP_OFF        0x2604
/* for T30 FAX and analog modem */
#define HW_MOD_FRM                0x4000
#define HW_MOD_FRH                0x4001
#define HW_MOD_FTM                0x4002
#define HW_MOD_FTH                0x4003
#define HW_MOD_FTS                0x4004
#define HW_MOD_CONNECT                0x4010
#define HW_MOD_OK                0x4011
#define HW_MOD_NOCARR                0x4012
#define HW_MOD_FCERROR                0x4013
#define HW_MOD_READY                0x4014
#define HW_MOD_LASTDATA                0x4015

/* DSP_TONE_PATT_ON parameter */
#define TONE_OFF                        0x0000
#define TONE_GERMAN_DIALTONE                0x0001
#define TONE_GERMAN_OLDDIALTONE                0x0002
#define TONE_AMERICAN_DIALTONE                0x0003
#define TONE_GERMAN_DIALPBX                0x0004
#define TONE_GERMAN_OLDDIALPBX                0x0005
#define TONE_AMERICAN_DIALPBX                0x0006
#define TONE_GERMAN_RINGING                0x0007
#define TONE_GERMAN_OLDRINGING                0x0008
#define TONE_AMERICAN_RINGPBX                0x000b
#define TONE_GERMAN_RINGPBX                0x000c
#define TONE_GERMAN_OLDRINGPBX                0x000d
#define TONE_AMERICAN_RINGING                0x000e
#define TONE_GERMAN_BUSY                0x000f
#define TONE_GERMAN_OLDBUSY                0x0010
#define TONE_AMERICAN_BUSY                0x0011
#define TONE_GERMAN_HANGUP                0x0012
#define TONE_GERMAN_OLDHANGUP                0x0013
#define TONE_AMERICAN_HANGUP                0x0014
#define TONE_SPECIAL_INFO                0x0015
#define TONE_GERMAN_GASSENBESETZT        0x0016
#define TONE_GERMAN_AUFSCHALTTON        0x0016

/* MPH_INFORMATION_IND */
#define L1_SIGNAL_LOS_OFF        0x0010
#define L1_SIGNAL_LOS_ON        0x0011
#define L1_SIGNAL_AIS_OFF        0x0012
#define L1_SIGNAL_AIS_ON        0x0013
#define L1_SIGNAL_RDI_OFF        0x0014
#define L1_SIGNAL_RDI_ON        0x0015
#define L1_SIGNAL_SLIP_RX        0x0020
#define L1_SIGNAL_SLIP_TX        0x0021

/*
 * protocol ids
 * D channel 1-31
 * B channel 33 - 63
 */

#define ISDN_P_NONE                0
#define ISDN_P_BASE                0
#define ISDN_P_TE_S0                0x01
#define ISDN_P_NT_S0                  0x02
#define ISDN_P_TE_E1                0x03
#define ISDN_P_NT_E1                  0x04
#define ISDN_P_TE_UP0                0x05
#define ISDN_P_NT_UP0                0x06

#define IS_ISDN_P_TE(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_TE_E1) || \
                                (p == ISDN_P_TE_UP0) || (p == ISDN_P_LAPD_TE))
#define IS_ISDN_P_NT(p) ((p == ISDN_P_NT_S0) || (p == ISDN_P_NT_E1) || \
                                (p == ISDN_P_NT_UP0) || (p == ISDN_P_LAPD_NT))
#define IS_ISDN_P_S0(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_NT_S0))
#define IS_ISDN_P_E1(p) ((p == ISDN_P_TE_E1) || (p == ISDN_P_NT_E1))
#define IS_ISDN_P_UP0(p) ((p == ISDN_P_TE_UP0) || (p == ISDN_P_NT_UP0))


#define ISDN_P_LAPD_TE                0x10
#define        ISDN_P_LAPD_NT                0x11

#define ISDN_P_B_MASK                0x1f
#define ISDN_P_B_START                0x20

#define ISDN_P_B_RAW                0x21
#define ISDN_P_B_HDLC                0x22
#define ISDN_P_B_X75SLP                0x23
#define ISDN_P_B_L2DTMF                0x24
#define ISDN_P_B_L2DSP                0x25
#define ISDN_P_B_L2DSPHDLC        0x26
#define ISDN_P_B_T30_FAX        0x27
#define ISDN_P_B_MODEM_ASYNC        0x28

#define OPTION_L2_PMX                1
#define OPTION_L2_PTP                2
#define OPTION_L2_FIXEDTEI        3
#define OPTION_L2_CLEANUP        4
#define OPTION_L1_HOLD                5

/* should be in sync with linux/kobject.h:KOBJ_NAME_LEN */
#define MISDN_MAX_IDLEN                20

struct mISDNhead {
        unsigned int        prim;
        unsigned int        id;
}  __packed;

#define MISDN_HEADER_LEN        sizeof(struct mISDNhead)
#define MAX_DATA_SIZE                2048
#define MAX_DATA_MEM                (MAX_DATA_SIZE + MISDN_HEADER_LEN)
#define MAX_DFRAME_LEN                260

#define MISDN_ID_ADDR_MASK        0xFFFF
#define MISDN_ID_TEI_MASK        0xFF00
#define MISDN_ID_SAPI_MASK        0x00FF
#define MISDN_ID_TEI_ANY        0x7F00

#define MISDN_ID_ANY                0xFFFF
#define MISDN_ID_NONE                0xFFFE

#define GROUP_TEI                127
#define TEI_SAPI                63
#define CTRL_SAPI                0

#define MISDN_MAX_CHANNEL        127
#define MISDN_CHMAP_SIZE        ((MISDN_MAX_CHANNEL + 1) >> 3)

#define SOL_MISDN        0

struct sockaddr_mISDN {
        sa_family_t    family;
        unsigned char        dev;
        unsigned char        channel;
        unsigned char        sapi;
        unsigned char        tei;
};

struct mISDNversion {
        unsigned char        major;
        unsigned char        minor;
        unsigned short        release;
};

struct mISDN_devinfo {
        u_int                        id;
        u_int                        Dprotocols;
        u_int                        Bprotocols;
        u_int                        protocol;
        u_char                        channelmap[MISDN_CHMAP_SIZE];
        u_int                        nrbchan;
        char                        name[MISDN_MAX_IDLEN];
};

struct mISDN_devrename {
        u_int                        id;
        char                        name[MISDN_MAX_IDLEN]; /* new name */
};

/* MPH_INFORMATION_REQ payload */
struct ph_info_ch {
        __u32 protocol;
        __u64 Flags;
};

struct ph_info_dch {
        struct ph_info_ch ch;
        __u16 state;
        __u16 num_bch;
};

struct ph_info {
        struct ph_info_dch dch;
        struct ph_info_ch  bch[];
};

/* timer device ioctl */
#define IMADDTIMER        _IOR('I', 64, int)
#define IMDELTIMER        _IOR('I', 65, int)

/* socket ioctls */
#define        IMGETVERSION        _IOR('I', 66, int)
#define        IMGETCOUNT        _IOR('I', 67, int)
#define IMGETDEVINFO        _IOR('I', 68, int)
#define IMCTRLREQ        _IOR('I', 69, int)
#define IMCLEAR_L2        _IOR('I', 70, int)
#define IMSETDEVNAME        _IOR('I', 71, struct mISDN_devrename)
#define IMHOLD_L1        _IOR('I', 72, int)

static inline int
test_channelmap(u_int nr, u_char *map)
{
        if (nr <= MISDN_MAX_CHANNEL)
                return map[nr >> 3] & (1 << (nr & 7));
        else
                return 0;
}

static inline void
set_channelmap(u_int nr, u_char *map)
{
        map[nr >> 3] |= (1 << (nr & 7));
}

static inline void
clear_channelmap(u_int nr, u_char *map)
{
        map[nr >> 3] &= ~(1 << (nr & 7));
}

/* CONTROL_CHANNEL parameters */
#define MISDN_CTRL_GETOP                0x0000
#define MISDN_CTRL_LOOP                        0x0001
#define MISDN_CTRL_CONNECT                0x0002
#define MISDN_CTRL_DISCONNECT                0x0004
#define MISDN_CTRL_RX_BUFFER                0x0008
#define MISDN_CTRL_PCMCONNECT                0x0010
#define MISDN_CTRL_PCMDISCONNECT        0x0020
#define MISDN_CTRL_SETPEER                0x0040
#define MISDN_CTRL_UNSETPEER                0x0080
#define MISDN_CTRL_RX_OFF                0x0100
#define MISDN_CTRL_FILL_EMPTY                0x0200
#define MISDN_CTRL_GETPEER                0x0400
#define MISDN_CTRL_L1_TIMER3                0x0800
#define MISDN_CTRL_HW_FEATURES_OP        0x2000
#define MISDN_CTRL_HW_FEATURES                0x2001
#define MISDN_CTRL_HFC_OP                0x4000
#define MISDN_CTRL_HFC_PCM_CONN                0x4001
#define MISDN_CTRL_HFC_PCM_DISC                0x4002
#define MISDN_CTRL_HFC_CONF_JOIN        0x4003
#define MISDN_CTRL_HFC_CONF_SPLIT        0x4004
#define MISDN_CTRL_HFC_RECEIVE_OFF        0x4005
#define MISDN_CTRL_HFC_RECEIVE_ON        0x4006
#define MISDN_CTRL_HFC_ECHOCAN_ON         0x4007
#define MISDN_CTRL_HFC_ECHOCAN_OFF         0x4008
#define MISDN_CTRL_HFC_WD_INIT                0x4009
#define MISDN_CTRL_HFC_WD_RESET                0x400A

/* special RX buffer value for MISDN_CTRL_RX_BUFFER request.p1 is the minimum
 * buffer size request.p2 the maximum. Using  MISDN_CTRL_RX_SIZE_IGNORE will
 * not change the value, but still read back the actual stetting.
 */
#define MISDN_CTRL_RX_SIZE_IGNORE        -1

/* socket options */
#define MISDN_TIME_STAMP                0x0001

struct mISDN_ctrl_req {
        int                op;
        int                channel;
        int                p1;
        int                p2;
};

/* muxer options */
#define MISDN_OPT_ALL                1
#define MISDN_OPT_TEIMGR        2

#ifdef __KERNEL__
#include <linux/list.h>
#include <linux/skbuff.h>
#include <linux/net.h>
#include <net/sock.h>
#include <linux/completion.h>

#define DEBUG_CORE                0x000000ff
#define DEBUG_CORE_FUNC                0x00000002
#define DEBUG_SOCKET                0x00000004
#define DEBUG_MANAGER                0x00000008
#define DEBUG_SEND_ERR                0x00000010
#define DEBUG_MSG_THREAD        0x00000020
#define DEBUG_QUEUE_FUNC        0x00000040
#define DEBUG_L1                0x0000ff00
#define DEBUG_L1_FSM                0x00000200
#define DEBUG_L2                0x00ff0000
#define DEBUG_L2_FSM                0x00020000
#define DEBUG_L2_CTRL                0x00040000
#define DEBUG_L2_RECV                0x00080000
#define DEBUG_L2_TEI                0x00100000
#define DEBUG_L2_TEIFSM                0x00200000
#define DEBUG_TIMER                0x01000000
#define DEBUG_CLOCK                0x02000000

#define mISDN_HEAD_P(s)                ((struct mISDNhead *)&s->cb[0])
#define mISDN_HEAD_PRIM(s)        (((struct mISDNhead *)&s->cb[0])->prim)
#define mISDN_HEAD_ID(s)        (((struct mISDNhead *)&s->cb[0])->id)

/* socket states */
#define MISDN_OPEN        1
#define MISDN_BOUND        2
#define MISDN_CLOSED        3

struct mISDNchannel;
struct mISDNdevice;
struct mISDNstack;
struct mISDNclock;

struct channel_req {
        u_int                        protocol;
        struct sockaddr_mISDN        adr;
        struct mISDNchannel        *ch;
};

typedef        int        (ctrl_func_t)(struct mISDNchannel *, u_int, void *);
typedef        int        (send_func_t)(struct mISDNchannel *, struct sk_buff *);
typedef int        (create_func_t)(struct channel_req *);

struct Bprotocol {
        struct list_head        list;
        char                        *name;
        u_int                        Bprotocols;
        create_func_t                *create;
};

struct mISDNchannel {
        struct list_head        list;
        u_int                        protocol;
        u_int                        nr;
        u_long                        opt;
        u_int                        addr;
        struct mISDNstack        *st;
        struct mISDNchannel        *peer;
        send_func_t                *send;
        send_func_t                *recv;
        ctrl_func_t                *ctrl;
};

struct mISDN_sock_list {
        struct hlist_head        head;
        rwlock_t                lock;
};

struct mISDN_sock {
        struct sock                sk;
        struct mISDNchannel        ch;
        u_int                        cmask;
        struct mISDNdevice        *dev;
};



struct mISDNdevice {
        struct mISDNchannel        D;
        u_int                        id;
        u_int                        Dprotocols;
        u_int                        Bprotocols;
        u_int                        nrbchan;
        u_char                        channelmap[MISDN_CHMAP_SIZE];
        struct list_head        bchannels;
        struct mISDNchannel        *teimgr;
        struct device                dev;
};

struct mISDNstack {
        u_long                        status;
        struct mISDNdevice        *dev;
        struct task_struct        *thread;
        struct completion        *notify;
        wait_queue_head_t        workq;
        struct sk_buff_head        msgq;
        struct list_head        layer2;
        struct mISDNchannel        *layer1;
        struct mISDNchannel        own;
        struct mutex                lmutex; /* protect lists */
        struct mISDN_sock_list        l1sock;
#ifdef MISDN_MSG_STATS
        u_int                        msg_cnt;
        u_int                        sleep_cnt;
        u_int                        stopped_cnt;
#endif
};

typedef        int        (clockctl_func_t)(void *, int);

struct        mISDNclock {
        struct list_head        list;
        char                        name[64];
        int                        pri;
        clockctl_func_t                *ctl;
        void                        *priv;
};

/* global alloc/queue functions */

static inline struct sk_buff *
mI_alloc_skb(unsigned int len, gfp_t gfp_mask)
{
        struct sk_buff        *skb;

        skb = alloc_skb(len + MISDN_HEADER_LEN, gfp_mask);
        if (likely(skb))
                skb_reserve(skb, MISDN_HEADER_LEN);
        return skb;
}

static inline struct sk_buff *
_alloc_mISDN_skb(u_int prim, u_int id, u_int len, void *dp, gfp_t gfp_mask)
{
        struct sk_buff        *skb = mI_alloc_skb(len, gfp_mask);
        struct mISDNhead *hh;

        if (!skb)
                return NULL;
        if (len)
                skb_put_data(skb, dp, len);
        hh = mISDN_HEAD_P(skb);
        hh->prim = prim;
        hh->id = id;
        return skb;
}

static inline void
_queue_data(struct mISDNchannel *ch, u_int prim,
    u_int id, u_int len, void *dp, gfp_t gfp_mask)
{
        struct sk_buff                *skb;

        if (!ch->peer)
                return;
        skb = _alloc_mISDN_skb(prim, id, len, dp, gfp_mask);
        if (!skb)
                return;
        if (ch->recv(ch->peer, skb))
                dev_kfree_skb(skb);
}

/* global register/unregister functions */

extern int        mISDN_register_device(struct mISDNdevice *,
                                        struct device *parent, char *name);
extern void        mISDN_unregister_device(struct mISDNdevice *);
extern int        mISDN_register_Bprotocol(struct Bprotocol *);
extern void        mISDN_unregister_Bprotocol(struct Bprotocol *);
extern struct mISDNclock *mISDN_register_clock(char *, int, clockctl_func_t *,
                                                void *);
extern void        mISDN_unregister_clock(struct mISDNclock *);

static inline struct mISDNdevice *dev_to_mISDN(const struct device *dev)
{
        if (dev)
                return dev_get_drvdata(dev);
        else
                return NULL;
}

extern void        set_channel_address(struct mISDNchannel *, u_int, u_int);
extern void        mISDN_clock_update(struct mISDNclock *, int, ktime_t *);
extern unsigned short mISDN_clock_get(void);
extern const char *mISDNDevName4ch(struct mISDNchannel *);

#endif /* __KERNEL__ */
#endif /* mISDNIF_H */





































































   63 




   62 








  141 







    4 
  165 







   50 








   50 















   11 



















   11 


   11 
















    2 

    8 

    9 








    8 



   11 
























    3 






    3 



    3 



    3 












    3 





    1 
    2 














   18 





   18 










   18 



    1 

   15 







    1 



    1 


   11 














































































   74 





























   68 



   69 






























   12 


















   12 









   11 


    5 


























































































































































   47 




   48 





   30 

   12 




























   16 









    9 


   10 




















    9 







    2 

    7 




















































   31 












   26 









   11 

   11 












    1 


    1 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
/*
 * net/tipc/msg.c: TIPC message header routines
 *
 * Copyright (c) 2000-2006, 2014-2015, Ericsson AB
 * Copyright (c) 2005, 2010-2011, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <net/sock.h>
#include "core.h"
#include "msg.h"
#include "addr.h"
#include "name_table.h"
#include "crypto.h"

#define BUF_ALIGN(x) ALIGN(x, 4)
#define MAX_FORWARD_SIZE 1024
#ifdef CONFIG_TIPC_CRYPTO
#define BUF_HEADROOM ALIGN(((LL_MAX_HEADER + 48) + EHDR_MAX_SIZE), 16)
#define BUF_OVERHEAD (BUF_HEADROOM + TIPC_AES_GCM_TAG_SIZE)
#else
#define BUF_HEADROOM (LL_MAX_HEADER + 48)
#define BUF_OVERHEAD BUF_HEADROOM
#endif

const int one_page_mtu = PAGE_SIZE - SKB_DATA_ALIGN(BUF_OVERHEAD) -
                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

/**
 * tipc_buf_acquire - creates a TIPC message buffer
 * @size: message size (including TIPC header)
 * @gfp: memory allocation flags
 *
 * Return: a new buffer with data pointers set to the specified size.
 *
 * NOTE:
 * Headroom is reserved to allow prepending of a data link header.
 * There may also be unrequested tailroom present at the buffer's end.
 */
struct sk_buff *tipc_buf_acquire(u32 size, gfp_t gfp)
{
        struct sk_buff *skb;

        skb = alloc_skb_fclone(BUF_OVERHEAD + size, gfp);
        if (skb) {
                skb_reserve(skb, BUF_HEADROOM);
                skb_put(skb, size);
                skb->next = NULL;
        }
        return skb;
}

void tipc_msg_init(u32 own_node, struct tipc_msg *m, u32 user, u32 type,
                   u32 hsize, u32 dnode)
{
        memset(m, 0, hsize);
        msg_set_version(m);
        msg_set_user(m, user);
        msg_set_hdr_sz(m, hsize);
        msg_set_size(m, hsize);
        msg_set_prevnode(m, own_node);
        msg_set_type(m, type);
        if (hsize > SHORT_H_SIZE) {
                msg_set_orignode(m, own_node);
                msg_set_destnode(m, dnode);
        }
}

struct sk_buff *tipc_msg_create(uint user, uint type,
                                uint hdr_sz, uint data_sz, u32 dnode,
                                u32 onode, u32 dport, u32 oport, int errcode)
{
        struct tipc_msg *msg;
        struct sk_buff *buf;

        buf = tipc_buf_acquire(hdr_sz + data_sz, GFP_ATOMIC);
        if (unlikely(!buf))
                return NULL;

        msg = buf_msg(buf);
        tipc_msg_init(onode, msg, user, type, hdr_sz, dnode);
        msg_set_size(msg, hdr_sz + data_sz);
        msg_set_origport(msg, oport);
        msg_set_destport(msg, dport);
        msg_set_errcode(msg, errcode);
        return buf;
}

/* tipc_buf_append(): Append a buffer to the fragment list of another buffer
 * @*headbuf: in:  NULL for first frag, otherwise value returned from prev call
 *            out: set when successful non-complete reassembly, otherwise NULL
 * @*buf:     in:  the buffer to append. Always defined
 *            out: head buf after successful complete reassembly, otherwise NULL
 * Returns 1 when reassembly complete, otherwise 0
 */
int tipc_buf_append(struct sk_buff **headbuf, struct sk_buff **buf)
{
        struct sk_buff *head = *headbuf;
        struct sk_buff *frag = *buf;
        struct sk_buff *tail = NULL;
        struct tipc_msg *msg;
        u32 fragid;
        int delta;
        bool headstolen;

        if (!frag)
                goto err;

        msg = buf_msg(frag);
        fragid = msg_type(msg);
        frag->next = NULL;
        skb_pull(frag, msg_hdr_sz(msg));

        if (fragid == FIRST_FRAGMENT) {
                if (unlikely(head))
                        goto err;
                if (skb_has_frag_list(frag) && __skb_linearize(frag))
                        goto err;
                *buf = NULL;
                frag = skb_unshare(frag, GFP_ATOMIC);
                if (unlikely(!frag))
                        goto err;
                head = *headbuf = frag;
                TIPC_SKB_CB(head)->tail = NULL;
                return 0;
        }

        if (!head)
                goto err;

        /* Either the input skb ownership is transferred to headskb
         * or the input skb is freed, clear the reference to avoid
         * bad access on error path.
         */
        *buf = NULL;
        if (skb_try_coalesce(head, frag, &headstolen, &delta)) {
                kfree_skb_partial(frag, headstolen);
        } else {
                tail = TIPC_SKB_CB(head)->tail;
                if (!skb_has_frag_list(head))
                        skb_shinfo(head)->frag_list = frag;
                else
                        tail->next = frag;
                head->truesize += frag->truesize;
                head->data_len += frag->len;
                head->len += frag->len;
                TIPC_SKB_CB(head)->tail = frag;
        }

        if (fragid == LAST_FRAGMENT) {
                TIPC_SKB_CB(head)->validated = 0;
                if (unlikely(!tipc_msg_validate(&head)))
                        goto err;
                *buf = head;
                TIPC_SKB_CB(head)->tail = NULL;
                *headbuf = NULL;
                return 1;
        }
        return 0;
err:
        kfree_skb(*buf);
        kfree_skb(*headbuf);
        *buf = *headbuf = NULL;
        return 0;
}

/**
 * tipc_msg_append(): Append data to tail of an existing buffer queue
 * @_hdr: header to be used
 * @m: the data to be appended
 * @mss: max allowable size of buffer
 * @dlen: size of data to be appended
 * @txq: queue to append to
 *
 * Return: the number of 1k blocks appended or errno value
 */
int tipc_msg_append(struct tipc_msg *_hdr, struct msghdr *m, int dlen,
                    int mss, struct sk_buff_head *txq)
{
        struct sk_buff *skb;
        int accounted, total, curr;
        int mlen, cpy, rem = dlen;
        struct tipc_msg *hdr;

        skb = skb_peek_tail(txq);
        accounted = skb ? msg_blocks(buf_msg(skb)) : 0;
        total = accounted;

        do {
                if (!skb || skb->len >= mss) {
                        skb = tipc_buf_acquire(mss, GFP_KERNEL);
                        if (unlikely(!skb))
                                return -ENOMEM;
                        skb_orphan(skb);
                        skb_trim(skb, MIN_H_SIZE);
                        hdr = buf_msg(skb);
                        skb_copy_to_linear_data(skb, _hdr, MIN_H_SIZE);
                        msg_set_hdr_sz(hdr, MIN_H_SIZE);
                        msg_set_size(hdr, MIN_H_SIZE);
                        __skb_queue_tail(txq, skb);
                        total += 1;
                }
                hdr = buf_msg(skb);
                curr = msg_blocks(hdr);
                mlen = msg_size(hdr);
                cpy = min_t(size_t, rem, mss - mlen);
                if (cpy != copy_from_iter(skb->data + mlen, cpy, &m->msg_iter))
                        return -EFAULT;
                msg_set_size(hdr, mlen + cpy);
                skb_put(skb, cpy);
                rem -= cpy;
                total += msg_blocks(hdr) - curr;
        } while (rem > 0);
        return total - accounted;
}

/* tipc_msg_validate - validate basic format of received message
 *
 * This routine ensures a TIPC message has an acceptable header, and at least
 * as much data as the header indicates it should.  The routine also ensures
 * that the entire message header is stored in the main fragment of the message
 * buffer, to simplify future access to message header fields.
 *
 * Note: Having extra info present in the message header or data areas is OK.
 * TIPC will ignore the excess, under the assumption that it is optional info
 * introduced by a later release of the protocol.
 */
bool tipc_msg_validate(struct sk_buff **_skb)
{
        struct sk_buff *skb = *_skb;
        struct tipc_msg *hdr;
        int msz, hsz;

        /* Ensure that flow control ratio condition is satisfied */
        if (unlikely(skb->truesize / buf_roundup_len(skb) >= 4)) {
                skb = skb_copy_expand(skb, BUF_HEADROOM, 0, GFP_ATOMIC);
                if (!skb)
                        return false;
                kfree_skb(*_skb);
                *_skb = skb;
        }

        if (unlikely(TIPC_SKB_CB(skb)->validated))
                return true;

        if (unlikely(!pskb_may_pull(skb, MIN_H_SIZE)))
                return false;

        hsz = msg_hdr_sz(buf_msg(skb));
        if (unlikely(hsz < MIN_H_SIZE) || (hsz > MAX_H_SIZE))
                return false;
        if (unlikely(!pskb_may_pull(skb, hsz)))
                return false;

        hdr = buf_msg(skb);
        if (unlikely(msg_version(hdr) != TIPC_VERSION))
                return false;

        msz = msg_size(hdr);
        if (unlikely(msz < hsz))
                return false;
        if (unlikely((msz - hsz) > TIPC_MAX_USER_MSG_SIZE))
                return false;
        if (unlikely(skb->len < msz))
                return false;

        TIPC_SKB_CB(skb)->validated = 1;
        return true;
}

/**
 * tipc_msg_fragment - build a fragment skb list for TIPC message
 *
 * @skb: TIPC message skb
 * @hdr: internal msg header to be put on the top of the fragments
 * @pktmax: max size of a fragment incl. the header
 * @frags: returned fragment skb list
 *
 * Return: 0 if the fragmentation is successful, otherwise: -EINVAL
 * or -ENOMEM
 */
int tipc_msg_fragment(struct sk_buff *skb, const struct tipc_msg *hdr,
                      int pktmax, struct sk_buff_head *frags)
{
        int pktno, nof_fragms, dsz, dmax, eat;
        struct tipc_msg *_hdr;
        struct sk_buff *_skb;
        u8 *data;

        /* Non-linear buffer? */
        if (skb_linearize(skb))
                return -ENOMEM;

        data = (u8 *)skb->data;
        dsz = msg_size(buf_msg(skb));
        dmax = pktmax - INT_H_SIZE;
        if (dsz <= dmax || !dmax)
                return -EINVAL;

        nof_fragms = dsz / dmax + 1;
        for (pktno = 1; pktno <= nof_fragms; pktno++) {
                if (pktno < nof_fragms)
                        eat = dmax;
                else
                        eat = dsz % dmax;
                /* Allocate a new fragment */
                _skb = tipc_buf_acquire(INT_H_SIZE + eat, GFP_ATOMIC);
                if (!_skb)
                        goto error;
                skb_orphan(_skb);
                __skb_queue_tail(frags, _skb);
                /* Copy header & data to the fragment */
                skb_copy_to_linear_data(_skb, hdr, INT_H_SIZE);
                skb_copy_to_linear_data_offset(_skb, INT_H_SIZE, data, eat);
                data += eat;
                /* Update the fragment's header */
                _hdr = buf_msg(_skb);
                msg_set_fragm_no(_hdr, pktno);
                msg_set_nof_fragms(_hdr, nof_fragms);
                msg_set_size(_hdr, INT_H_SIZE + eat);
        }
        return 0;

error:
        __skb_queue_purge(frags);
        __skb_queue_head_init(frags);
        return -ENOMEM;
}

/**
 * tipc_msg_build - create buffer chain containing specified header and data
 * @mhdr: Message header, to be prepended to data
 * @m: User message
 * @offset: buffer offset for fragmented messages (FIXME)
 * @dsz: Total length of user data
 * @pktmax: Max packet size that can be used
 * @list: Buffer or chain of buffers to be returned to caller
 *
 * Note that the recursive call we are making here is safe, since it can
 * logically go only one further level down.
 *
 * Return: message data size or errno: -ENOMEM, -EFAULT
 */
int tipc_msg_build(struct tipc_msg *mhdr, struct msghdr *m, int offset,
                   int dsz, int pktmax, struct sk_buff_head *list)
{
        int mhsz = msg_hdr_sz(mhdr);
        struct tipc_msg pkthdr;
        int msz = mhsz + dsz;
        int pktrem = pktmax;
        struct sk_buff *skb;
        int drem = dsz;
        int pktno = 1;
        char *pktpos;
        int pktsz;
        int rc;

        msg_set_size(mhdr, msz);

        /* No fragmentation needed? */
        if (likely(msz <= pktmax)) {
                skb = tipc_buf_acquire(msz, GFP_KERNEL);

                /* Fall back to smaller MTU if node local message */
                if (unlikely(!skb)) {
                        if (pktmax != MAX_MSG_SIZE)
                                return -ENOMEM;
                        rc = tipc_msg_build(mhdr, m, offset, dsz,
                                            one_page_mtu, list);
                        if (rc != dsz)
                                return rc;
                        if (tipc_msg_assemble(list))
                                return dsz;
                        return -ENOMEM;
                }
                skb_orphan(skb);
                __skb_queue_tail(list, skb);
                skb_copy_to_linear_data(skb, mhdr, mhsz);
                pktpos = skb->data + mhsz;
                if (copy_from_iter_full(pktpos, dsz, &m->msg_iter))
                        return dsz;
                rc = -EFAULT;
                goto error;
        }

        /* Prepare reusable fragment header */
        tipc_msg_init(msg_prevnode(mhdr), &pkthdr, MSG_FRAGMENTER,
                      FIRST_FRAGMENT, INT_H_SIZE, msg_destnode(mhdr));
        msg_set_size(&pkthdr, pktmax);
        msg_set_fragm_no(&pkthdr, pktno);
        msg_set_importance(&pkthdr, msg_importance(mhdr));

        /* Prepare first fragment */
        skb = tipc_buf_acquire(pktmax, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;
        skb_orphan(skb);
        __skb_queue_tail(list, skb);
        pktpos = skb->data;
        skb_copy_to_linear_data(skb, &pkthdr, INT_H_SIZE);
        pktpos += INT_H_SIZE;
        pktrem -= INT_H_SIZE;
        skb_copy_to_linear_data_offset(skb, INT_H_SIZE, mhdr, mhsz);
        pktpos += mhsz;
        pktrem -= mhsz;

        do {
                if (drem < pktrem)
                        pktrem = drem;

                if (!copy_from_iter_full(pktpos, pktrem, &m->msg_iter)) {
                        rc = -EFAULT;
                        goto error;
                }
                drem -= pktrem;

                if (!drem)
                        break;

                /* Prepare new fragment: */
                if (drem < (pktmax - INT_H_SIZE))
                        pktsz = drem + INT_H_SIZE;
                else
                        pktsz = pktmax;
                skb = tipc_buf_acquire(pktsz, GFP_KERNEL);
                if (!skb) {
                        rc = -ENOMEM;
                        goto error;
                }
                skb_orphan(skb);
                __skb_queue_tail(list, skb);
                msg_set_type(&pkthdr, FRAGMENT);
                msg_set_size(&pkthdr, pktsz);
                msg_set_fragm_no(&pkthdr, ++pktno);
                skb_copy_to_linear_data(skb, &pkthdr, INT_H_SIZE);
                pktpos = skb->data + INT_H_SIZE;
                pktrem = pktsz - INT_H_SIZE;

        } while (1);
        msg_set_type(buf_msg(skb), LAST_FRAGMENT);
        return dsz;
error:
        __skb_queue_purge(list);
        __skb_queue_head_init(list);
        return rc;
}

/**
 * tipc_msg_bundle - Append contents of a buffer to tail of an existing one
 * @bskb: the bundle buffer to append to
 * @msg: message to be appended
 * @max: max allowable size for the bundle buffer
 *
 * Return: "true" if bundling has been performed, otherwise "false"
 */
static bool tipc_msg_bundle(struct sk_buff *bskb, struct tipc_msg *msg,
                            u32 max)
{
        struct tipc_msg *bmsg = buf_msg(bskb);
        u32 msz, bsz, offset, pad;

        msz = msg_size(msg);
        bsz = msg_size(bmsg);
        offset = BUF_ALIGN(bsz);
        pad = offset - bsz;

        if (unlikely(skb_tailroom(bskb) < (pad + msz)))
                return false;
        if (unlikely(max < (offset + msz)))
                return false;

        skb_put(bskb, pad + msz);
        skb_copy_to_linear_data_offset(bskb, offset, msg, msz);
        msg_set_size(bmsg, offset + msz);
        msg_set_msgcnt(bmsg, msg_msgcnt(bmsg) + 1);
        return true;
}

/**
 * tipc_msg_try_bundle - Try to bundle a new message to the last one
 * @tskb: the last/target message to which the new one will be appended
 * @skb: the new message skb pointer
 * @mss: max message size (header inclusive)
 * @dnode: destination node for the message
 * @new_bundle: if this call made a new bundle or not
 *
 * Return: "true" if the new message skb is potential for bundling this time or
 * later, in the case a bundling has been done this time, the skb is consumed
 * (the skb pointer = NULL).
 * Otherwise, "false" if the skb cannot be bundled at all.
 */
bool tipc_msg_try_bundle(struct sk_buff *tskb, struct sk_buff **skb, u32 mss,
                         u32 dnode, bool *new_bundle)
{
        struct tipc_msg *msg, *inner, *outer;
        u32 tsz;

        /* First, check if the new buffer is suitable for bundling */
        msg = buf_msg(*skb);
        if (msg_user(msg) == MSG_FRAGMENTER)
                return false;
        if (msg_user(msg) == TUNNEL_PROTOCOL)
                return false;
        if (msg_user(msg) == BCAST_PROTOCOL)
                return false;
        if (mss <= INT_H_SIZE + msg_size(msg))
                return false;

        /* Ok, but the last/target buffer can be empty? */
        if (unlikely(!tskb))
                return true;

        /* Is it a bundle already? Try to bundle the new message to it */
        if (msg_user(buf_msg(tskb)) == MSG_BUNDLER) {
                *new_bundle = false;
                goto bundle;
        }

        /* Make a new bundle of the two messages if possible */
        tsz = msg_size(buf_msg(tskb));
        if (unlikely(mss < BUF_ALIGN(INT_H_SIZE + tsz) + msg_size(msg)))
                return true;
        if (unlikely(pskb_expand_head(tskb, INT_H_SIZE, mss - tsz - INT_H_SIZE,
                                      GFP_ATOMIC)))
                return true;
        inner = buf_msg(tskb);
        skb_push(tskb, INT_H_SIZE);
        outer = buf_msg(tskb);
        tipc_msg_init(msg_prevnode(inner), outer, MSG_BUNDLER, 0, INT_H_SIZE,
                      dnode);
        msg_set_importance(outer, msg_importance(inner));
        msg_set_size(outer, INT_H_SIZE + tsz);
        msg_set_msgcnt(outer, 1);
        *new_bundle = true;

bundle:
        if (likely(tipc_msg_bundle(tskb, msg, mss))) {
                consume_skb(*skb);
                *skb = NULL;
        }
        return true;
}

/**
 *  tipc_msg_extract(): extract bundled inner packet from buffer
 *  @skb: buffer to be extracted from.
 *  @iskb: extracted inner buffer, to be returned
 *  @pos: position in outer message of msg to be extracted.
 *  Returns position of next msg.
 *  Consumes outer buffer when last packet extracted
 *  Return: true when there is an extracted buffer, otherwise false
 */
bool tipc_msg_extract(struct sk_buff *skb, struct sk_buff **iskb, int *pos)
{
        struct tipc_msg *hdr, *ihdr;
        int imsz;

        *iskb = NULL;
        if (unlikely(skb_linearize(skb)))
                goto none;

        hdr = buf_msg(skb);
        if (unlikely(*pos > (msg_data_sz(hdr) - MIN_H_SIZE)))
                goto none;

        ihdr = (struct tipc_msg *)(msg_data(hdr) + *pos);
        imsz = msg_size(ihdr);

        if ((*pos + imsz) > msg_data_sz(hdr))
                goto none;

        *iskb = tipc_buf_acquire(imsz, GFP_ATOMIC);
        if (!*iskb)
                goto none;

        skb_copy_to_linear_data(*iskb, ihdr, imsz);
        if (unlikely(!tipc_msg_validate(iskb)))
                goto none;

        *pos += BUF_ALIGN(imsz);
        return true;
none:
        kfree_skb(skb);
        kfree_skb(*iskb);
        *iskb = NULL;
        return false;
}

/**
 * tipc_msg_reverse(): swap source and destination addresses and add error code
 * @own_node: originating node id for reversed message
 * @skb:  buffer containing message to be reversed; will be consumed
 * @err:  error code to be set in message, if any
 * Replaces consumed buffer with new one when successful
 * Return: true if success, otherwise false
 */
bool tipc_msg_reverse(u32 own_node,  struct sk_buff **skb, int err)
{
        struct sk_buff *_skb = *skb;
        struct tipc_msg *_hdr, *hdr;
        int hlen, dlen;

        if (skb_linearize(_skb))
                goto exit;
        _hdr = buf_msg(_skb);
        dlen = min_t(uint, msg_data_sz(_hdr), MAX_FORWARD_SIZE);
        hlen = msg_hdr_sz(_hdr);

        if (msg_dest_droppable(_hdr))
                goto exit;
        if (msg_errcode(_hdr))
                goto exit;

        /* Never return SHORT header */
        if (hlen == SHORT_H_SIZE)
                hlen = BASIC_H_SIZE;

        /* Don't return data along with SYN+, - sender has a clone */
        if (msg_is_syn(_hdr) && err == TIPC_ERR_OVERLOAD)
                dlen = 0;

        /* Allocate new buffer to return */
        *skb = tipc_buf_acquire(hlen + dlen, GFP_ATOMIC);
        if (!*skb)
                goto exit;
        memcpy((*skb)->data, _skb->data, msg_hdr_sz(_hdr));
        memcpy((*skb)->data + hlen, msg_data(_hdr), dlen);

        /* Build reverse header in new buffer */
        hdr = buf_msg(*skb);
        msg_set_hdr_sz(hdr, hlen);
        msg_set_errcode(hdr, err);
        msg_set_non_seq(hdr, 0);
        msg_set_origport(hdr, msg_destport(_hdr));
        msg_set_destport(hdr, msg_origport(_hdr));
        msg_set_destnode(hdr, msg_prevnode(_hdr));
        msg_set_prevnode(hdr, own_node);
        msg_set_orignode(hdr, own_node);
        msg_set_size(hdr, hlen + dlen);
        skb_orphan(_skb);
        kfree_skb(_skb);
        return true;
exit:
        kfree_skb(_skb);
        *skb = NULL;
        return false;
}

bool tipc_msg_skb_clone(struct sk_buff_head *msg, struct sk_buff_head *cpy)
{
        struct sk_buff *skb, *_skb;

        skb_queue_walk(msg, skb) {
                _skb = skb_clone(skb, GFP_ATOMIC);
                if (!_skb) {
                        __skb_queue_purge(cpy);
                        pr_err_ratelimited("Failed to clone buffer chain\n");
                        return false;
                }
                __skb_queue_tail(cpy, _skb);
        }
        return true;
}

/**
 * tipc_msg_lookup_dest(): try to find new destination for named message
 * @net: pointer to associated network namespace
 * @skb: the buffer containing the message.
 * @err: error code to be used by caller if lookup fails
 * Does not consume buffer
 * Return: true if a destination is found, false otherwise
 */
bool tipc_msg_lookup_dest(struct net *net, struct sk_buff *skb, int *err)
{
        struct tipc_msg *msg = buf_msg(skb);
        u32 scope = msg_lookup_scope(msg);
        u32 self = tipc_own_addr(net);
        u32 inst = msg_nameinst(msg);
        struct tipc_socket_addr sk;
        struct tipc_uaddr ua;

        if (!msg_isdata(msg))
                return false;
        if (!msg_named(msg))
                return false;
        if (msg_errcode(msg))
                return false;
        *err = TIPC_ERR_NO_NAME;
        if (skb_linearize(skb))
                return false;
        msg = buf_msg(skb);
        if (msg_reroute_cnt(msg))
                return false;
        tipc_uaddr(&ua, TIPC_SERVICE_RANGE, scope,
                   msg_nametype(msg), inst, inst);
        sk.node = tipc_scope2node(net, scope);
        if (!tipc_nametbl_lookup_anycast(net, &ua, &sk))
                return false;
        msg_incr_reroute_cnt(msg);
        if (sk.node != self)
                msg_set_prevnode(msg, self);
        msg_set_destnode(msg, sk.node);
        msg_set_destport(msg, sk.ref);
        *err = TIPC_OK;

        return true;
}

/* tipc_msg_assemble() - assemble chain of fragments into one message
 */
bool tipc_msg_assemble(struct sk_buff_head *list)
{
        struct sk_buff *skb, *tmp = NULL;

        if (skb_queue_len(list) == 1)
                return true;

        while ((skb = __skb_dequeue(list))) {
                skb->next = NULL;
                if (tipc_buf_append(&tmp, &skb)) {
                        __skb_queue_tail(list, skb);
                        return true;
                }
                if (!tmp)
                        break;
        }
        __skb_queue_purge(list);
        __skb_queue_head_init(list);
        pr_warn("Failed do assemble buffer\n");
        return false;
}

/* tipc_msg_reassemble() - clone a buffer chain of fragments and
 *                         reassemble the clones into one message
 */
bool tipc_msg_reassemble(struct sk_buff_head *list, struct sk_buff_head *rcvq)
{
        struct sk_buff *skb, *_skb;
        struct sk_buff *frag = NULL;
        struct sk_buff *head = NULL;
        int hdr_len;

        /* Copy header if single buffer */
        if (skb_queue_len(list) == 1) {
                skb = skb_peek(list);
                hdr_len = skb_headroom(skb) + msg_hdr_sz(buf_msg(skb));
                _skb = __pskb_copy(skb, hdr_len, GFP_ATOMIC);
                if (!_skb)
                        return false;
                __skb_queue_tail(rcvq, _skb);
                return true;
        }

        /* Clone all fragments and reassemble */
        skb_queue_walk(list, skb) {
                frag = skb_clone(skb, GFP_ATOMIC);
                if (!frag)
                        goto error;
                frag->next = NULL;
                if (tipc_buf_append(&head, &frag))
                        break;
                if (!head)
                        goto error;
        }
        __skb_queue_tail(rcvq, frag);
        return true;
error:
        pr_warn("Failed do clone local mcast rcv buffer\n");
        kfree_skb(head);
        return false;
}

bool tipc_msg_pskb_copy(u32 dst, struct sk_buff_head *msg,
                        struct sk_buff_head *cpy)
{
        struct sk_buff *skb, *_skb;

        skb_queue_walk(msg, skb) {
                _skb = pskb_copy(skb, GFP_ATOMIC);
                if (!_skb) {
                        __skb_queue_purge(cpy);
                        return false;
                }
                msg_set_destnode(buf_msg(_skb), dst);
                __skb_queue_tail(cpy, _skb);
        }
        return true;
}

/* tipc_skb_queue_sorted(); sort pkt into list according to sequence number
 * @list: list to be appended to
 * @seqno: sequence number of buffer to add
 * @skb: buffer to add
 */
bool __tipc_skb_queue_sorted(struct sk_buff_head *list, u16 seqno,
                             struct sk_buff *skb)
{
        struct sk_buff *_skb, *tmp;

        if (skb_queue_empty(list) || less(seqno, buf_seqno(skb_peek(list)))) {
                __skb_queue_head(list, skb);
                return true;
        }

        if (more(seqno, buf_seqno(skb_peek_tail(list)))) {
                __skb_queue_tail(list, skb);
                return true;
        }

        skb_queue_walk_safe(list, _skb, tmp) {
                if (more(seqno, buf_seqno(_skb)))
                        continue;
                if (seqno == buf_seqno(_skb))
                        break;
                __skb_queue_before(list, _skb, skb);
                return true;
        }
        kfree_skb(skb);
        return false;
}

void tipc_skb_reject(struct net *net, int err, struct sk_buff *skb,
                     struct sk_buff_head *xmitq)
{
        if (tipc_msg_reverse(tipc_own_addr(net), &skb, err))
                __skb_queue_tail(xmitq, skb);
}





















































































































































































































































































































































































































































































































   48 




    1 
    3 
    2 




    1 






   38 

    1 


    2 



   33 

    6 

















    8 





    1 
    1 



    1 



    1 






    1 


    3 










    2 












    5 




    1 
    2 



    1 


    1 








    6 



    1 


    5 















    2 



    1 


    1 







   10 








   10 









    9 
    9 




















    1 




































































































   68 


   39 


   29 



    1 

    8 

    5 



    2 



    1 
    7 




    2 




    2 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/unaligned.h>
#include <linux/kernel.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <linux/dccp.h>
#include <linux/sctp.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <net/tcp.h>

struct nft_exthdr {
        u8                        type;
        u8                        offset;
        u8                        len;
        u8                        op;
        u8                        dreg;
        u8                        sreg;
        u8                        flags;
};

static unsigned int optlen(const u8 *opt, unsigned int offset)
{
        /* Beware zero-length options: make finite progress */
        if (opt[offset] <= TCPOPT_NOP || opt[offset + 1] == 0)
                return 1;
        else
                return opt[offset + 1];
}

static int nft_skb_copy_to_reg(const struct sk_buff *skb, int offset, u32 *dest, unsigned int len)
{
        if (len % NFT_REG32_SIZE)
                dest[len / NFT_REG32_SIZE] = 0;

        return skb_copy_bits(skb, offset, dest, len);
}

static void nft_exthdr_ipv6_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        u32 *dest = &regs->data[priv->dreg];
        unsigned int offset = 0;
        int err;

        if (pkt->skb->protocol != htons(ETH_P_IPV6))
                goto err;

        err = ipv6_find_hdr(pkt->skb, &offset, priv->type, NULL, NULL);
        if (priv->flags & NFT_EXTHDR_F_PRESENT) {
                nft_reg_store8(dest, err >= 0);
                return;
        } else if (err < 0) {
                goto err;
        }
        offset += priv->offset;

        if (nft_skb_copy_to_reg(pkt->skb, offset, dest, priv->len) < 0)
                goto err;
        return;
err:
        regs->verdict.code = NFT_BREAK;
}

/* find the offset to specified option.
 *
 * If target header is found, its offset is set in *offset and return option
 * number. Otherwise, return negative error.
 *
 * If the first fragment doesn't contain the End of Options it is considered
 * invalid.
 */
static int ipv4_find_option(struct net *net, struct sk_buff *skb,
                            unsigned int *offset, int target)
{
        unsigned char optbuf[sizeof(struct ip_options) + 40];
        struct ip_options *opt = (struct ip_options *)optbuf;
        struct iphdr *iph, _iph;
        bool found = false;
        __be32 info;
        int optlen;

        iph = skb_header_pointer(skb, 0, sizeof(_iph), &_iph);
        if (!iph)
                return -EBADMSG;

        optlen = iph->ihl * 4 - (int)sizeof(struct iphdr);
        if (optlen <= 0)
                return -ENOENT;

        memset(opt, 0, sizeof(struct ip_options));
        /* Copy the options since __ip_options_compile() modifies
         * the options.
         */
        if (skb_copy_bits(skb, sizeof(struct iphdr), opt->__data, optlen))
                return -EBADMSG;
        opt->optlen = optlen;

        if (__ip_options_compile(net, opt, NULL, &info))
                return -EBADMSG;

        switch (target) {
        case IPOPT_SSRR:
        case IPOPT_LSRR:
                if (!opt->srr)
                        break;
                found = target == IPOPT_SSRR ? opt->is_strictroute :
                                               !opt->is_strictroute;
                if (found)
                        *offset = opt->srr;
                break;
        case IPOPT_RR:
                if (!opt->rr)
                        break;
                *offset = opt->rr;
                found = true;
                break;
        case IPOPT_RA:
                if (!opt->router_alert)
                        break;
                *offset = opt->router_alert;
                found = true;
                break;
        default:
                return -EOPNOTSUPP;
        }
        return found ? target : -ENOENT;
}

static void nft_exthdr_ipv4_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        u32 *dest = &regs->data[priv->dreg];
        struct sk_buff *skb = pkt->skb;
        unsigned int offset;
        int err;

        if (skb->protocol != htons(ETH_P_IP))
                goto err;

        err = ipv4_find_option(nft_net(pkt), skb, &offset, priv->type);
        if (priv->flags & NFT_EXTHDR_F_PRESENT) {
                nft_reg_store8(dest, err >= 0);
                return;
        } else if (err < 0) {
                goto err;
        }
        offset += priv->offset;

        if (nft_skb_copy_to_reg(pkt->skb, offset, dest, priv->len) < 0)
                goto err;
        return;
err:
        regs->verdict.code = NFT_BREAK;
}

static void *
nft_tcp_header_pointer(const struct nft_pktinfo *pkt,
                       unsigned int len, void *buffer, unsigned int *tcphdr_len)
{
        struct tcphdr *tcph;

        if (pkt->tprot != IPPROTO_TCP || pkt->fragoff)
                return NULL;

        tcph = skb_header_pointer(pkt->skb, nft_thoff(pkt), sizeof(*tcph), buffer);
        if (!tcph)
                return NULL;

        *tcphdr_len = __tcp_hdrlen(tcph);
        if (*tcphdr_len < sizeof(*tcph) || *tcphdr_len > len)
                return NULL;

        return skb_header_pointer(pkt->skb, nft_thoff(pkt), *tcphdr_len, buffer);
}

static void nft_exthdr_tcp_eval(const struct nft_expr *expr,
                                struct nft_regs *regs,
                                const struct nft_pktinfo *pkt)
{
        u8 buff[sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE];
        struct nft_exthdr *priv = nft_expr_priv(expr);
        unsigned int i, optl, tcphdr_len, offset;
        u32 *dest = &regs->data[priv->dreg];
        struct tcphdr *tcph;
        u8 *opt;

        tcph = nft_tcp_header_pointer(pkt, sizeof(buff), buff, &tcphdr_len);
        if (!tcph)
                goto err;

        opt = (u8 *)tcph;
        for (i = sizeof(*tcph); i < tcphdr_len - 1; i += optl) {
                optl = optlen(opt, i);

                if (priv->type != opt[i])
                        continue;

                if (i + optl > tcphdr_len || priv->len + priv->offset > optl)
                        goto err;

                offset = i + priv->offset;
                if (priv->flags & NFT_EXTHDR_F_PRESENT) {
                        nft_reg_store8(dest, 1);
                } else {
                        if (priv->len % NFT_REG32_SIZE)
                                dest[priv->len / NFT_REG32_SIZE] = 0;
                        memcpy(dest, opt + offset, priv->len);
                }

                return;
        }

err:
        if (priv->flags & NFT_EXTHDR_F_PRESENT)
                *dest = 0;
        else
                regs->verdict.code = NFT_BREAK;
}

static void nft_exthdr_tcp_set_eval(const struct nft_expr *expr,
                                    struct nft_regs *regs,
                                    const struct nft_pktinfo *pkt)
{
        u8 buff[sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE];
        struct nft_exthdr *priv = nft_expr_priv(expr);
        unsigned int i, optl, tcphdr_len, offset;
        struct tcphdr *tcph;
        u8 *opt;

        tcph = nft_tcp_header_pointer(pkt, sizeof(buff), buff, &tcphdr_len);
        if (!tcph)
                goto err;

        if (skb_ensure_writable(pkt->skb, nft_thoff(pkt) + tcphdr_len))
                goto err;

        tcph = (struct tcphdr *)(pkt->skb->data + nft_thoff(pkt));
        opt = (u8 *)tcph;

        for (i = sizeof(*tcph); i < tcphdr_len - 1; i += optl) {
                union {
                        __be16 v16;
                        __be32 v32;
                } old, new;

                optl = optlen(opt, i);

                if (priv->type != opt[i])
                        continue;

                if (i + optl > tcphdr_len || priv->len + priv->offset > optl)
                        goto err;

                offset = i + priv->offset;

                switch (priv->len) {
                case 2:
                        old.v16 = (__force __be16)get_unaligned((u16 *)(opt + offset));
                        new.v16 = (__force __be16)nft_reg_load16(
                                &regs->data[priv->sreg]);

                        switch (priv->type) {
                        case TCPOPT_MSS:
                                /* increase can cause connection to stall */
                                if (ntohs(old.v16) <= ntohs(new.v16))
                                        return;
                        break;
                        }

                        if (old.v16 == new.v16)
                                return;

                        put_unaligned(new.v16, (__be16*)(opt + offset));
                        inet_proto_csum_replace2(&tcph->check, pkt->skb,
                                                 old.v16, new.v16, false);
                        break;
                case 4:
                        new.v32 = nft_reg_load_be32(&regs->data[priv->sreg]);
                        old.v32 = (__force __be32)get_unaligned((u32 *)(opt + offset));

                        if (old.v32 == new.v32)
                                return;

                        put_unaligned(new.v32, (__be32*)(opt + offset));
                        inet_proto_csum_replace4(&tcph->check, pkt->skb,
                                                 old.v32, new.v32, false);
                        break;
                default:
                        WARN_ON_ONCE(1);
                        break;
                }

                return;
        }
        return;
err:
        regs->verdict.code = NFT_BREAK;
}

static void nft_exthdr_tcp_strip_eval(const struct nft_expr *expr,
                                      struct nft_regs *regs,
                                      const struct nft_pktinfo *pkt)
{
        u8 buff[sizeof(struct tcphdr) + MAX_TCP_OPTION_SPACE];
        struct nft_exthdr *priv = nft_expr_priv(expr);
        unsigned int i, tcphdr_len, optl;
        struct tcphdr *tcph;
        u8 *opt;

        tcph = nft_tcp_header_pointer(pkt, sizeof(buff), buff, &tcphdr_len);
        if (!tcph)
                goto err;

        if (skb_ensure_writable(pkt->skb, nft_thoff(pkt) + tcphdr_len))
                goto drop;

        tcph = (struct tcphdr *)(pkt->skb->data + nft_thoff(pkt));
        opt = (u8 *)tcph;

        for (i = sizeof(*tcph); i < tcphdr_len - 1; i += optl) {
                unsigned int j;

                optl = optlen(opt, i);
                if (priv->type != opt[i])
                        continue;

                if (i + optl > tcphdr_len)
                        goto drop;

                for (j = 0; j < optl; ++j) {
                        u16 n = TCPOPT_NOP;
                        u16 o = opt[i+j];

                        if ((i + j) % 2 == 0) {
                                o <<= 8;
                                n <<= 8;
                        }
                        inet_proto_csum_replace2(&tcph->check, pkt->skb, htons(o),
                                                 htons(n), false);
                }
                memset(opt + i, TCPOPT_NOP, optl);
                return;
        }

        /* option not found, continue. This allows to do multiple
         * option removals per rule.
         */
        return;
err:
        regs->verdict.code = NFT_BREAK;
        return;
drop:
        /* can't remove, no choice but to drop */
        regs->verdict.code = NF_DROP;
}

static void nft_exthdr_sctp_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        unsigned int offset = nft_thoff(pkt) + sizeof(struct sctphdr);
        struct nft_exthdr *priv = nft_expr_priv(expr);
        u32 *dest = &regs->data[priv->dreg];
        const struct sctp_chunkhdr *sch;
        struct sctp_chunkhdr _sch;

        if (pkt->tprot != IPPROTO_SCTP)
                goto err;

        do {
                sch = skb_header_pointer(pkt->skb, offset, sizeof(_sch), &_sch);
                if (!sch || !sch->length)
                        break;

                if (sch->type == priv->type) {
                        if (priv->flags & NFT_EXTHDR_F_PRESENT) {
                                nft_reg_store8(dest, true);
                                return;
                        }
                        if (priv->offset + priv->len > ntohs(sch->length) ||
                            offset + ntohs(sch->length) > pkt->skb->len)
                                break;

                        if (nft_skb_copy_to_reg(pkt->skb, offset + priv->offset,
                                                dest, priv->len) < 0)
                                break;
                        return;
                }
                offset += SCTP_PAD4(ntohs(sch->length));
        } while (offset < pkt->skb->len);
err:
        if (priv->flags & NFT_EXTHDR_F_PRESENT)
                nft_reg_store8(dest, false);
        else
                regs->verdict.code = NFT_BREAK;
}

#ifdef CONFIG_NFT_EXTHDR_DCCP
static void nft_exthdr_dccp_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        unsigned int thoff, dataoff, optoff, optlen, i;
        u32 *dest = &regs->data[priv->dreg];
        const struct dccp_hdr *dh;
        struct dccp_hdr _dh;

        if (pkt->tprot != IPPROTO_DCCP || pkt->fragoff)
                goto err;

        thoff = nft_thoff(pkt);

        dh = skb_header_pointer(pkt->skb, thoff, sizeof(_dh), &_dh);
        if (!dh)
                goto err;

        dataoff = dh->dccph_doff * sizeof(u32);
        optoff = __dccp_hdr_len(dh);
        if (dataoff <= optoff)
                goto err;

        optlen = dataoff - optoff;

        for (i = 0; i < optlen; ) {
                /* Options 0 (DCCPO_PADDING) - 31 (DCCPO_MAX_RESERVED) are 1B in
                 * the length; the remaining options are at least 2B long.  In
                 * all cases, the first byte contains the option type.  In
                 * multi-byte options, the second byte contains the option
                 * length, which must be at least two: 1 for the type plus 1 for
                 * the length plus 0-253 for any following option data.  We
                 * aren't interested in the option data, only the type and the
                 * length, so we don't need to read more than two bytes at a
                 * time.
                 */
                unsigned int buflen = optlen - i;
                u8 buf[2], *bufp;
                u8 type, len;

                if (buflen > sizeof(buf))
                        buflen = sizeof(buf);

                bufp = skb_header_pointer(pkt->skb, thoff + optoff + i, buflen,
                                          &buf);
                if (!bufp)
                        goto err;

                type = bufp[0];

                if (type == priv->type) {
                        nft_reg_store8(dest, 1);
                        return;
                }

                if (type <= DCCPO_MAX_RESERVED) {
                        i++;
                        continue;
                }

                if (buflen < 2)
                        goto err;

                len = bufp[1];

                if (len < 2)
                        goto err;

                i += len;
        }

err:
        *dest = 0;
}
#endif

static const struct nla_policy nft_exthdr_policy[NFTA_EXTHDR_MAX + 1] = {
        [NFTA_EXTHDR_DREG]                = { .type = NLA_U32 },
        [NFTA_EXTHDR_TYPE]                = { .type = NLA_U8 },
        [NFTA_EXTHDR_OFFSET]                = { .type = NLA_U32 },
        [NFTA_EXTHDR_LEN]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_EXTHDR_FLAGS]                = { .type = NLA_U32 },
        [NFTA_EXTHDR_OP]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_EXTHDR_SREG]                = { .type = NLA_U32 },
};

static int nft_exthdr_init(const struct nft_ctx *ctx,
                           const struct nft_expr *expr,
                           const struct nlattr * const tb[])
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        u32 offset, len, flags = 0, op = NFT_EXTHDR_OP_IPV6;
        int err;

        if (!tb[NFTA_EXTHDR_DREG] ||
            !tb[NFTA_EXTHDR_TYPE] ||
            !tb[NFTA_EXTHDR_OFFSET] ||
            !tb[NFTA_EXTHDR_LEN])
                return -EINVAL;

        err = nft_parse_u32_check(tb[NFTA_EXTHDR_OFFSET], U8_MAX, &offset);
        if (err < 0)
                return err;

        err = nft_parse_u32_check(tb[NFTA_EXTHDR_LEN], U8_MAX, &len);
        if (err < 0)
                return err;

        if (tb[NFTA_EXTHDR_FLAGS]) {
                err = nft_parse_u32_check(tb[NFTA_EXTHDR_FLAGS], U8_MAX, &flags);
                if (err < 0)
                        return err;

                if (flags & ~NFT_EXTHDR_F_PRESENT)
                        return -EINVAL;
        }

        if (tb[NFTA_EXTHDR_OP]) {
                err = nft_parse_u32_check(tb[NFTA_EXTHDR_OP], U8_MAX, &op);
                if (err < 0)
                        return err;
        }

        priv->type   = nla_get_u8(tb[NFTA_EXTHDR_TYPE]);
        priv->offset = offset;
        priv->len    = len;
        priv->flags  = flags;
        priv->op     = op;

        return nft_parse_register_store(ctx, tb[NFTA_EXTHDR_DREG],
                                        &priv->dreg, NULL, NFT_DATA_VALUE,
                                        priv->len);
}

static int nft_exthdr_tcp_set_init(const struct nft_ctx *ctx,
                                   const struct nft_expr *expr,
                                   const struct nlattr * const tb[])
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        u32 offset, len, flags = 0, op = NFT_EXTHDR_OP_IPV6;
        int err;

        if (!tb[NFTA_EXTHDR_SREG] ||
            !tb[NFTA_EXTHDR_TYPE] ||
            !tb[NFTA_EXTHDR_OFFSET] ||
            !tb[NFTA_EXTHDR_LEN])
                return -EINVAL;

        if (tb[NFTA_EXTHDR_DREG] || tb[NFTA_EXTHDR_FLAGS])
                return -EINVAL;

        err = nft_parse_u32_check(tb[NFTA_EXTHDR_OFFSET], U8_MAX, &offset);
        if (err < 0)
                return err;

        err = nft_parse_u32_check(tb[NFTA_EXTHDR_LEN], U8_MAX, &len);
        if (err < 0)
                return err;

        if (offset < 2)
                return -EOPNOTSUPP;

        switch (len) {
        case 2: break;
        case 4: break;
        default:
                return -EOPNOTSUPP;
        }

        err = nft_parse_u32_check(tb[NFTA_EXTHDR_OP], U8_MAX, &op);
        if (err < 0)
                return err;

        priv->type   = nla_get_u8(tb[NFTA_EXTHDR_TYPE]);
        priv->offset = offset;
        priv->len    = len;
        priv->flags  = flags;
        priv->op     = op;

        return nft_parse_register_load(ctx, tb[NFTA_EXTHDR_SREG], &priv->sreg,
                                       priv->len);
}

static int nft_exthdr_tcp_strip_init(const struct nft_ctx *ctx,
                                     const struct nft_expr *expr,
                                     const struct nlattr * const tb[])
{
        struct nft_exthdr *priv = nft_expr_priv(expr);

        if (tb[NFTA_EXTHDR_SREG] ||
            tb[NFTA_EXTHDR_DREG] ||
            tb[NFTA_EXTHDR_FLAGS] ||
            tb[NFTA_EXTHDR_OFFSET] ||
            tb[NFTA_EXTHDR_LEN])
                return -EINVAL;

        if (!tb[NFTA_EXTHDR_TYPE])
                return -EINVAL;

        priv->type = nla_get_u8(tb[NFTA_EXTHDR_TYPE]);
        priv->op = NFT_EXTHDR_OP_TCPOPT;

        return 0;
}

static int nft_exthdr_ipv4_init(const struct nft_ctx *ctx,
                                const struct nft_expr *expr,
                                const struct nlattr * const tb[])
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        int err = nft_exthdr_init(ctx, expr, tb);

        if (err < 0)
                return err;

        switch (priv->type) {
        case IPOPT_SSRR:
        case IPOPT_LSRR:
        case IPOPT_RR:
        case IPOPT_RA:
                break;
        default:
                return -EOPNOTSUPP;
        }
        return 0;
}

#ifdef CONFIG_NFT_EXTHDR_DCCP
static int nft_exthdr_dccp_init(const struct nft_ctx *ctx,
                                const struct nft_expr *expr,
                                const struct nlattr * const tb[])
{
        struct nft_exthdr *priv = nft_expr_priv(expr);
        int err = nft_exthdr_init(ctx, expr, tb);

        if (err < 0)
                return err;

        if (!(priv->flags & NFT_EXTHDR_F_PRESENT))
                return -EOPNOTSUPP;

        return 0;
}
#endif

static int nft_exthdr_dump_common(struct sk_buff *skb, const struct nft_exthdr *priv)
{
        if (nla_put_u8(skb, NFTA_EXTHDR_TYPE, priv->type))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_EXTHDR_OFFSET, htonl(priv->offset)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_EXTHDR_LEN, htonl(priv->len)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_EXTHDR_FLAGS, htonl(priv->flags)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_EXTHDR_OP, htonl(priv->op)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static int nft_exthdr_dump(struct sk_buff *skb,
                           const struct nft_expr *expr, bool reset)
{
        const struct nft_exthdr *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_EXTHDR_DREG, priv->dreg))
                return -1;

        return nft_exthdr_dump_common(skb, priv);
}

static int nft_exthdr_dump_set(struct sk_buff *skb,
                               const struct nft_expr *expr, bool reset)
{
        const struct nft_exthdr *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_EXTHDR_SREG, priv->sreg))
                return -1;

        return nft_exthdr_dump_common(skb, priv);
}

static int nft_exthdr_dump_strip(struct sk_buff *skb,
                                 const struct nft_expr *expr, bool reset)
{
        const struct nft_exthdr *priv = nft_expr_priv(expr);

        return nft_exthdr_dump_common(skb, priv);
}

static bool nft_exthdr_reduce(struct nft_regs_track *track,
                               const struct nft_expr *expr)
{
        const struct nft_exthdr *priv = nft_expr_priv(expr);
        const struct nft_exthdr *exthdr;

        if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        exthdr = nft_expr_priv(track->regs[priv->dreg].selector);
        if (priv->type != exthdr->type ||
            priv->op != exthdr->op ||
            priv->flags != exthdr->flags ||
            priv->offset != exthdr->offset ||
            priv->len != exthdr->len) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        if (!track->regs[priv->dreg].bitwise)
                return true;

        return nft_expr_reduce_bitwise(track, expr);
}

static const struct nft_expr_ops nft_exthdr_ipv6_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_ipv6_eval,
        .init                = nft_exthdr_init,
        .dump                = nft_exthdr_dump,
        .reduce                = nft_exthdr_reduce,
};

static const struct nft_expr_ops nft_exthdr_ipv4_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_ipv4_eval,
        .init                = nft_exthdr_ipv4_init,
        .dump                = nft_exthdr_dump,
        .reduce                = nft_exthdr_reduce,
};

static const struct nft_expr_ops nft_exthdr_tcp_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_tcp_eval,
        .init                = nft_exthdr_init,
        .dump                = nft_exthdr_dump,
        .reduce                = nft_exthdr_reduce,
};

static const struct nft_expr_ops nft_exthdr_tcp_set_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_tcp_set_eval,
        .init                = nft_exthdr_tcp_set_init,
        .dump                = nft_exthdr_dump_set,
        .reduce                = NFT_REDUCE_READONLY,
};

static const struct nft_expr_ops nft_exthdr_tcp_strip_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_tcp_strip_eval,
        .init                = nft_exthdr_tcp_strip_init,
        .dump                = nft_exthdr_dump_strip,
        .reduce                = NFT_REDUCE_READONLY,
};

static const struct nft_expr_ops nft_exthdr_sctp_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_sctp_eval,
        .init                = nft_exthdr_init,
        .dump                = nft_exthdr_dump,
        .reduce                = nft_exthdr_reduce,
};

#ifdef CONFIG_NFT_EXTHDR_DCCP
static const struct nft_expr_ops nft_exthdr_dccp_ops = {
        .type                = &nft_exthdr_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)),
        .eval                = nft_exthdr_dccp_eval,
        .init                = nft_exthdr_dccp_init,
        .dump                = nft_exthdr_dump,
        .reduce                = nft_exthdr_reduce,
};
#endif

static const struct nft_expr_ops *
nft_exthdr_select_ops(const struct nft_ctx *ctx,
                      const struct nlattr * const tb[])
{
        u32 op;

        if (!tb[NFTA_EXTHDR_OP])
                return &nft_exthdr_ipv6_ops;

        if (tb[NFTA_EXTHDR_SREG] && tb[NFTA_EXTHDR_DREG])
                return ERR_PTR(-EOPNOTSUPP);

        op = ntohl(nla_get_be32(tb[NFTA_EXTHDR_OP]));
        switch (op) {
        case NFT_EXTHDR_OP_TCPOPT:
                if (tb[NFTA_EXTHDR_SREG])
                        return &nft_exthdr_tcp_set_ops;
                if (tb[NFTA_EXTHDR_DREG])
                        return &nft_exthdr_tcp_ops;
                return &nft_exthdr_tcp_strip_ops;
        case NFT_EXTHDR_OP_IPV6:
                if (tb[NFTA_EXTHDR_DREG])
                        return &nft_exthdr_ipv6_ops;
                break;
        case NFT_EXTHDR_OP_IPV4:
                if (ctx->family != NFPROTO_IPV6) {
                        if (tb[NFTA_EXTHDR_DREG])
                                return &nft_exthdr_ipv4_ops;
                }
                break;
        case NFT_EXTHDR_OP_SCTP:
                if (tb[NFTA_EXTHDR_DREG])
                        return &nft_exthdr_sctp_ops;
                break;
#ifdef CONFIG_NFT_EXTHDR_DCCP
        case NFT_EXTHDR_OP_DCCP:
                if (tb[NFTA_EXTHDR_DREG])
                        return &nft_exthdr_dccp_ops;
                break;
#endif
        }

        return ERR_PTR(-EOPNOTSUPP);
}

struct nft_expr_type nft_exthdr_type __read_mostly = {
        .name                = "exthdr",
        .select_ops        = nft_exthdr_select_ops,
        .policy                = nft_exthdr_policy,
        .maxattr        = NFTA_EXTHDR_MAX,
        .owner                = THIS_MODULE,
};







































































































































































































































































































































































































































































































































































































    6 


























   69 





   28 










   47 

















































































































































































































































































































   24 














































  484 









   56 




   92 









   91 



















   39 



















    4 











    4 









































   26 


























   23 







































































































































   27 
  234 








































   18 














   62 














































































































































































































































































   95 
























































    8 

















   13 















   95 

















    8 

















   12 








































































    2 





    2 




























    2 





    2 






   27 

























    3 




   52 




    3 
   52 



































  381 




  306 




  122 




































































 1792 












  276 
  381 














































  277 
  306 




  122 





































































































































































































































































































































  101 






  102 














   97 




















































































































































































   91 

   91 







 1931 











































































    5 
  111 


















































































































































































































































































































































































































































   65 
 1674 

  222 
















   65 
 1661 

  219 











































































    2 


























    7 










  149 



  117 
































  791 
    2 












    8 
    1 
    1 
    2 























   56 






























































   13 
































  837 
    1 



















































































































































































































































































































   15 






























































   17 




   17 






    9 

   14 





























  326 

































































































































    1 







    1 
   29 


















































































    3 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FS_H
#define _LINUX_FS_H

#include <linux/vfsdebug.h>
#include <linux/linkage.h>
#include <linux/wait_bit.h>
#include <linux/kdev_t.h>
#include <linux/dcache.h>
#include <linux/path.h>
#include <linux/stat.h>
#include <linux/cache.h>
#include <linux/list.h>
#include <linux/list_lru.h>
#include <linux/llist.h>
#include <linux/radix-tree.h>
#include <linux/xarray.h>
#include <linux/rbtree.h>
#include <linux/init.h>
#include <linux/pid.h>
#include <linux/bug.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/mm_types.h>
#include <linux/capability.h>
#include <linux/semaphore.h>
#include <linux/fcntl.h>
#include <linux/rculist_bl.h>
#include <linux/atomic.h>
#include <linux/shrinker.h>
#include <linux/migrate_mode.h>
#include <linux/uidgid.h>
#include <linux/lockdep.h>
#include <linux/percpu-rwsem.h>
#include <linux/workqueue.h>
#include <linux/delayed_call.h>
#include <linux/uuid.h>
#include <linux/errseq.h>
#include <linux/ioprio.h>
#include <linux/fs_types.h>
#include <linux/build_bug.h>
#include <linux/stddef.h>
#include <linux/mount.h>
#include <linux/cred.h>
#include <linux/mnt_idmapping.h>
#include <linux/slab.h>
#include <linux/maple_tree.h>
#include <linux/rw_hint.h>
#include <linux/file_ref.h>
#include <linux/unicode.h>

#include <asm/byteorder.h>
#include <uapi/linux/fs.h>

struct backing_dev_info;
struct bdi_writeback;
struct bio;
struct io_comp_batch;
struct export_operations;
struct fiemap_extent_info;
struct hd_geometry;
struct iovec;
struct kiocb;
struct kobject;
struct pipe_inode_info;
struct poll_table_struct;
struct kstatfs;
struct vm_area_struct;
struct vfsmount;
struct cred;
struct swap_info_struct;
struct seq_file;
struct workqueue_struct;
struct iov_iter;
struct fscrypt_operations;
struct fsverity_operations;
struct fsnotify_mark_connector;
struct fsnotify_sb_info;
struct fs_context;
struct fs_parameter_spec;
struct file_kattr;
struct iomap_ops;

extern void __init inode_init(void);
extern void __init inode_init_early(void);
extern void __init files_init(void);
extern void __init files_maxfiles_init(void);

extern unsigned long get_max_files(void);
extern unsigned int sysctl_nr_open;

typedef __kernel_rwf_t rwf_t;

struct buffer_head;
typedef int (get_block_t)(struct inode *inode, sector_t iblock,
                        struct buffer_head *bh_result, int create);
typedef int (dio_iodone_t)(struct kiocb *iocb, loff_t offset,
                        ssize_t bytes, void *private);

#define MAY_EXEC                0x00000001
#define MAY_WRITE                0x00000002
#define MAY_READ                0x00000004
#define MAY_APPEND                0x00000008
#define MAY_ACCESS                0x00000010
#define MAY_OPEN                0x00000020
#define MAY_CHDIR                0x00000040
/* called from RCU mode, don't block */
#define MAY_NOT_BLOCK                0x00000080

/*
 * flags in file.f_mode.  Note that FMODE_READ and FMODE_WRITE must correspond
 * to O_WRONLY and O_RDWR via the strange trick in do_dentry_open()
 */

/* file is open for reading */
#define FMODE_READ                ((__force fmode_t)(1 << 0))
/* file is open for writing */
#define FMODE_WRITE                ((__force fmode_t)(1 << 1))
/* file is seekable */
#define FMODE_LSEEK                ((__force fmode_t)(1 << 2))
/* file can be accessed using pread */
#define FMODE_PREAD                ((__force fmode_t)(1 << 3))
/* file can be accessed using pwrite */
#define FMODE_PWRITE                ((__force fmode_t)(1 << 4))
/* File is opened for execution with sys_execve / sys_uselib */
#define FMODE_EXEC                ((__force fmode_t)(1 << 5))
/* File writes are restricted (block device specific) */
#define FMODE_WRITE_RESTRICTED        ((__force fmode_t)(1 << 6))
/* File supports atomic writes */
#define FMODE_CAN_ATOMIC_WRITE        ((__force fmode_t)(1 << 7))

/* FMODE_* bit 8 */

/* 32bit hashes as llseek() offset (for directories) */
#define FMODE_32BITHASH         ((__force fmode_t)(1 << 9))
/* 64bit hashes as llseek() offset (for directories) */
#define FMODE_64BITHASH         ((__force fmode_t)(1 << 10))

/*
 * Don't update ctime and mtime.
 *
 * Currently a special hack for the XFS open_by_handle ioctl, but we'll
 * hopefully graduate it to a proper O_CMTIME flag supported by open(2) soon.
 */
#define FMODE_NOCMTIME                ((__force fmode_t)(1 << 11))

/* Expect random access pattern */
#define FMODE_RANDOM                ((__force fmode_t)(1 << 12))

/* Supports IOCB_HAS_METADATA */
#define FMODE_HAS_METADATA        ((__force fmode_t)(1 << 13))

/* File is opened with O_PATH; almost nothing can be done with it */
#define FMODE_PATH                ((__force fmode_t)(1 << 14))

/* File needs atomic accesses to f_pos */
#define FMODE_ATOMIC_POS        ((__force fmode_t)(1 << 15))
/* Write access to underlying fs */
#define FMODE_WRITER                ((__force fmode_t)(1 << 16))
/* Has read method(s) */
#define FMODE_CAN_READ          ((__force fmode_t)(1 << 17))
/* Has write method(s) */
#define FMODE_CAN_WRITE         ((__force fmode_t)(1 << 18))

#define FMODE_OPENED                ((__force fmode_t)(1 << 19))
#define FMODE_CREATED                ((__force fmode_t)(1 << 20))

/* File is stream-like */
#define FMODE_STREAM                ((__force fmode_t)(1 << 21))

/* File supports DIRECT IO */
#define        FMODE_CAN_ODIRECT        ((__force fmode_t)(1 << 22))

#define        FMODE_NOREUSE                ((__force fmode_t)(1 << 23))

/* File is embedded in backing_file object */
#define FMODE_BACKING                ((__force fmode_t)(1 << 24))

/*
 * Together with FMODE_NONOTIFY_PERM defines which fsnotify events shouldn't be
 * generated (see below)
 */
#define FMODE_NONOTIFY                ((__force fmode_t)(1 << 25))

/*
 * Together with FMODE_NONOTIFY defines which fsnotify events shouldn't be
 * generated (see below)
 */
#define FMODE_NONOTIFY_PERM        ((__force fmode_t)(1 << 26))

/* File is capable of returning -EAGAIN if I/O will block */
#define FMODE_NOWAIT                ((__force fmode_t)(1 << 27))

/* File represents mount that needs unmounting */
#define FMODE_NEED_UNMOUNT        ((__force fmode_t)(1 << 28))

/* File does not contribute to nr_files count */
#define FMODE_NOACCOUNT                ((__force fmode_t)(1 << 29))

/*
 * The two FMODE_NONOTIFY* define which fsnotify events should not be generated
 * for an open file. These are the possible values of
 * (f->f_mode & FMODE_FSNOTIFY_MASK) and their meaning:
 *
 * FMODE_NONOTIFY - suppress all (incl. non-permission) events.
 * FMODE_NONOTIFY_PERM - suppress permission (incl. pre-content) events.
 * FMODE_NONOTIFY | FMODE_NONOTIFY_PERM - suppress only FAN_ACCESS_PERM.
 */
#define FMODE_FSNOTIFY_MASK \
        (FMODE_NONOTIFY | FMODE_NONOTIFY_PERM)

#define FMODE_FSNOTIFY_NONE(mode) \
        ((mode & FMODE_FSNOTIFY_MASK) == FMODE_NONOTIFY)
#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
#define FMODE_FSNOTIFY_HSM(mode) \
        ((mode & FMODE_FSNOTIFY_MASK) == 0 || \
         (mode & FMODE_FSNOTIFY_MASK) == (FMODE_NONOTIFY | FMODE_NONOTIFY_PERM))
#define FMODE_FSNOTIFY_ACCESS_PERM(mode) \
        ((mode & FMODE_FSNOTIFY_MASK) == 0)
#else
#define FMODE_FSNOTIFY_ACCESS_PERM(mode) 0
#define FMODE_FSNOTIFY_HSM(mode)        0
#endif

/*
 * Attribute flags.  These should be or-ed together to figure out what
 * has been changed!
 */
#define ATTR_MODE        (1 << 0)
#define ATTR_UID        (1 << 1)
#define ATTR_GID        (1 << 2)
#define ATTR_SIZE        (1 << 3)
#define ATTR_ATIME        (1 << 4)
#define ATTR_MTIME        (1 << 5)
#define ATTR_CTIME        (1 << 6)
#define ATTR_ATIME_SET        (1 << 7)
#define ATTR_MTIME_SET        (1 << 8)
#define ATTR_FORCE        (1 << 9) /* Not a change, but a change it */
#define ATTR_CTIME_SET        (1 << 10)
#define ATTR_KILL_SUID        (1 << 11)
#define ATTR_KILL_SGID        (1 << 12)
#define ATTR_FILE        (1 << 13)
#define ATTR_KILL_PRIV        (1 << 14)
#define ATTR_OPEN        (1 << 15) /* Truncating from open(O_TRUNC) */
#define ATTR_TIMES_SET        (1 << 16)
#define ATTR_TOUCH        (1 << 17)
#define ATTR_DELEG        (1 << 18) /* Delegated attrs. Don't break write delegations */

/*
 * Whiteout is represented by a char device.  The following constants define the
 * mode and device number to use.
 */
#define WHITEOUT_MODE 0
#define WHITEOUT_DEV 0

/*
 * This is the Inode Attributes structure, used for notify_change().  It
 * uses the above definitions as flags, to know which values have changed.
 * Also, in this manner, a Filesystem can look at only the values it cares
 * about.  Basically, these are the attributes that the VFS layer can
 * request to change from the FS layer.
 *
 * Derek Atkins <warlord@MIT.EDU> 94-10-20
 */
struct iattr {
        unsigned int        ia_valid;
        umode_t                ia_mode;
        /*
         * The two anonymous unions wrap structures with the same member.
         *
         * Filesystems raising FS_ALLOW_IDMAP need to use ia_vfs{g,u}id which
         * are a dedicated type requiring the filesystem to use the dedicated
         * helpers. Other filesystem can continue to use ia_{g,u}id until they
         * have been ported.
         *
         * They always contain the same value. In other words FS_ALLOW_IDMAP
         * pass down the same value on idmapped mounts as they would on regular
         * mounts.
         */
        union {
                kuid_t                ia_uid;
                vfsuid_t        ia_vfsuid;
        };
        union {
                kgid_t                ia_gid;
                vfsgid_t        ia_vfsgid;
        };
        loff_t                ia_size;
        struct timespec64 ia_atime;
        struct timespec64 ia_mtime;
        struct timespec64 ia_ctime;

        /*
         * Not an attribute, but an auxiliary info for filesystems wanting to
         * implement an ftruncate() like method.  NOTE: filesystem should
         * check for (ia_valid & ATTR_FILE), and not for (ia_file != NULL).
         */
        struct file        *ia_file;
};

/*
 * Includes for diskquotas.
 */
#include <linux/quota.h>

/*
 * Maximum number of layers of fs stack.  Needs to be limited to
 * prevent kernel stack overflow
 */
#define FILESYSTEM_MAX_STACK_DEPTH 2

/** 
 * enum positive_aop_returns - aop return codes with specific semantics
 *
 * @AOP_WRITEPAGE_ACTIVATE: Informs the caller that page writeback has
 *                             completed, that the page is still locked, and
 *                             should be considered active.  The VM uses this hint
 *                             to return the page to the active list -- it won't
 *                             be a candidate for writeback again in the near
 *                             future.  Other callers must be careful to unlock
 *                             the page if they get this return.  Returned by
 *                             writepage(); 
 *
 * @AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has
 *                          unlocked it and the page might have been truncated.
 *                          The caller should back up to acquiring a new page and
 *                          trying again.  The aop will be taking reasonable
 *                          precautions not to livelock.  If the caller held a page
 *                          reference, it should drop it before retrying.  Returned
 *                          by read_folio().
 *
 * address_space_operation functions return these large constants to indicate
 * special semantics to the caller.  These are much larger than the bytes in a
 * page to allow for functions that return the number of bytes operated on in a
 * given page.
 */

enum positive_aop_returns {
        AOP_WRITEPAGE_ACTIVATE        = 0x80000,
        AOP_TRUNCATED_PAGE        = 0x80001,
};

/*
 * oh the beauties of C type declarations.
 */
struct page;
struct address_space;
struct writeback_control;
struct readahead_control;

/* Match RWF_* bits to IOCB bits */
#define IOCB_HIPRI                (__force int) RWF_HIPRI
#define IOCB_DSYNC                (__force int) RWF_DSYNC
#define IOCB_SYNC                (__force int) RWF_SYNC
#define IOCB_NOWAIT                (__force int) RWF_NOWAIT
#define IOCB_APPEND                (__force int) RWF_APPEND
#define IOCB_ATOMIC                (__force int) RWF_ATOMIC
#define IOCB_DONTCACHE                (__force int) RWF_DONTCACHE
#define IOCB_NOSIGNAL                (__force int) RWF_NOSIGNAL

/* non-RWF related bits - start at 16 */
#define IOCB_EVENTFD                (1 << 16)
#define IOCB_DIRECT                (1 << 17)
#define IOCB_WRITE                (1 << 18)
/* iocb->ki_waitq is valid */
#define IOCB_WAITQ                (1 << 19)
#define IOCB_NOIO                (1 << 20)
/* can use bio alloc cache */
#define IOCB_ALLOC_CACHE        (1 << 21)
/*
 * IOCB_DIO_CALLER_COMP can be set by the iocb owner, to indicate that the
 * iocb completion can be passed back to the owner for execution from a safe
 * context rather than needing to be punted through a workqueue. If this
 * flag is set, the bio completion handling may set iocb->dio_complete to a
 * handler function and iocb->private to context information for that handler.
 * The issuer should call the handler with that context information from task
 * context to complete the processing of the iocb. Note that while this
 * provides a task context for the dio_complete() callback, it should only be
 * used on the completion side for non-IO generating completions. It's fine to
 * call blocking functions from this callback, but they should not wait for
 * unrelated IO (like cache flushing, new IO generation, etc).
 */
#define IOCB_DIO_CALLER_COMP        (1 << 22)
/* kiocb is a read or write operation submitted by fs/aio.c. */
#define IOCB_AIO_RW                (1 << 23)
#define IOCB_HAS_METADATA        (1 << 24)

/* for use in trace events */
#define TRACE_IOCB_STRINGS \
        { IOCB_HIPRI,                "HIPRI" }, \
        { IOCB_DSYNC,                "DSYNC" }, \
        { IOCB_SYNC,                "SYNC" }, \
        { IOCB_NOWAIT,                "NOWAIT" }, \
        { IOCB_APPEND,                "APPEND" }, \
        { IOCB_ATOMIC,                "ATOMIC" }, \
        { IOCB_DONTCACHE,        "DONTCACHE" }, \
        { IOCB_EVENTFD,                "EVENTFD"}, \
        { IOCB_DIRECT,                "DIRECT" }, \
        { IOCB_WRITE,                "WRITE" }, \
        { IOCB_WAITQ,                "WAITQ" }, \
        { IOCB_NOIO,                "NOIO" }, \
        { IOCB_ALLOC_CACHE,        "ALLOC_CACHE" }, \
        { IOCB_DIO_CALLER_COMP,        "CALLER_COMP" }, \
        { IOCB_AIO_RW,                "AIO_RW" }, \
        { IOCB_HAS_METADATA,        "AIO_HAS_METADATA" }

struct kiocb {
        struct file                *ki_filp;
        loff_t                        ki_pos;
        void (*ki_complete)(struct kiocb *iocb, long ret);
        void                        *private;
        int                        ki_flags;
        u16                        ki_ioprio; /* See linux/ioprio.h */
        u8                        ki_write_stream;
        union {
                /*
                 * Only used for async buffered reads, where it denotes the
                 * page waitqueue associated with completing the read. Valid
                 * IFF IOCB_WAITQ is set.
                 */
                struct wait_page_queue        *ki_waitq;
                /*
                 * Can be used for O_DIRECT IO, where the completion handling
                 * is punted back to the issuer of the IO. May only be set
                 * if IOCB_DIO_CALLER_COMP is set by the issuer, and the issuer
                 * must then check for presence of this handler when ki_complete
                 * is invoked. The data passed in to this handler must be
                 * assigned to ->private when dio_complete is assigned.
                 */
                ssize_t (*dio_complete)(void *data);
        };
};

static inline bool is_sync_kiocb(struct kiocb *kiocb)
{
        return kiocb->ki_complete == NULL;
}

struct address_space_operations {
        int (*read_folio)(struct file *, struct folio *);

        /* Write back some dirty pages from this mapping. */
        int (*writepages)(struct address_space *, struct writeback_control *);

        /* Mark a folio dirty.  Return true if this dirtied it */
        bool (*dirty_folio)(struct address_space *, struct folio *);

        void (*readahead)(struct readahead_control *);

        int (*write_begin)(const struct kiocb *, struct address_space *mapping,
                                loff_t pos, unsigned len,
                                struct folio **foliop, void **fsdata);
        int (*write_end)(const struct kiocb *, struct address_space *mapping,
                                loff_t pos, unsigned len, unsigned copied,
                                struct folio *folio, void *fsdata);

        /* Unfortunately this kludge is needed for FIBMAP. Don't use it */
        sector_t (*bmap)(struct address_space *, sector_t);
        void (*invalidate_folio) (struct folio *, size_t offset, size_t len);
        bool (*release_folio)(struct folio *, gfp_t);
        void (*free_folio)(struct folio *folio);
        ssize_t (*direct_IO)(struct kiocb *, struct iov_iter *iter);
        /*
         * migrate the contents of a folio to the specified target. If
         * migrate_mode is MIGRATE_ASYNC, it must not block.
         */
        int (*migrate_folio)(struct address_space *, struct folio *dst,
                        struct folio *src, enum migrate_mode);
        int (*launder_folio)(struct folio *);
        bool (*is_partially_uptodate) (struct folio *, size_t from,
                        size_t count);
        void (*is_dirty_writeback) (struct folio *, bool *dirty, bool *wb);
        int (*error_remove_folio)(struct address_space *, struct folio *);

        /* swapfile support */
        int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
                                sector_t *span);
        void (*swap_deactivate)(struct file *file);
        int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
};

extern const struct address_space_operations empty_aops;

/**
 * struct address_space - Contents of a cacheable, mappable object.
 * @host: Owner, either the inode or the block_device.
 * @i_pages: Cached pages.
 * @invalidate_lock: Guards coherency between page cache contents and
 *   file offset->disk block mappings in the filesystem during invalidates.
 *   It is also used to block modification of page cache contents through
 *   memory mappings.
 * @gfp_mask: Memory allocation flags to use for allocating pages.
 * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings.
 * @nr_thps: Number of THPs in the pagecache (non-shmem only).
 * @i_mmap: Tree of private and shared mappings.
 * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
 * @nrpages: Number of page entries, protected by the i_pages lock.
 * @writeback_index: Writeback starts here.
 * @a_ops: Methods.
 * @flags: Error bits and flags (AS_*).
 * @wb_err: The most recent error which has occurred.
 * @i_private_lock: For use by the owner of the address_space.
 * @i_private_list: For use by the owner of the address_space.
 * @i_private_data: For use by the owner of the address_space.
 */
struct address_space {
        struct inode                *host;
        struct xarray                i_pages;
        struct rw_semaphore        invalidate_lock;
        gfp_t                        gfp_mask;
        atomic_t                i_mmap_writable;
#ifdef CONFIG_READ_ONLY_THP_FOR_FS
        /* number of thp, only for non-shmem files */
        atomic_t                nr_thps;
#endif
        struct rb_root_cached        i_mmap;
        unsigned long                nrpages;
        pgoff_t                        writeback_index;
        const struct address_space_operations *a_ops;
        unsigned long                flags;
        errseq_t                wb_err;
        spinlock_t                i_private_lock;
        struct list_head        i_private_list;
        struct rw_semaphore        i_mmap_rwsem;
        void *                        i_private_data;
} __attribute__((aligned(sizeof(long)))) __randomize_layout;
        /*
         * On most architectures that alignment is already the case; but
         * must be enforced here for CRIS, to let the least significant bit
         * of struct folio's "mapping" pointer be used for FOLIO_MAPPING_ANON.
         */

/* XArray tags, for tagging dirty and writeback pages in the pagecache. */
#define PAGECACHE_TAG_DIRTY        XA_MARK_0
#define PAGECACHE_TAG_WRITEBACK        XA_MARK_1
#define PAGECACHE_TAG_TOWRITE        XA_MARK_2

/*
 * Returns true if any of the pages in the mapping are marked with the tag.
 */
static inline bool mapping_tagged(const struct address_space *mapping, xa_mark_t tag)
{
        return xa_marked(&mapping->i_pages, tag);
}

static inline void i_mmap_lock_write(struct address_space *mapping)
{
        down_write(&mapping->i_mmap_rwsem);
}

static inline int i_mmap_trylock_write(struct address_space *mapping)
{
        return down_write_trylock(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_unlock_write(struct address_space *mapping)
{
        up_write(&mapping->i_mmap_rwsem);
}

static inline int i_mmap_trylock_read(struct address_space *mapping)
{
        return down_read_trylock(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_lock_read(struct address_space *mapping)
{
        down_read(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_unlock_read(struct address_space *mapping)
{
        up_read(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_assert_locked(struct address_space *mapping)
{
        lockdep_assert_held(&mapping->i_mmap_rwsem);
}

static inline void i_mmap_assert_write_locked(struct address_space *mapping)
{
        lockdep_assert_held_write(&mapping->i_mmap_rwsem);
}

/*
 * Might pages of this file be mapped into userspace?
 */
static inline int mapping_mapped(const struct address_space *mapping)
{
        return        !RB_EMPTY_ROOT(&mapping->i_mmap.rb_root);
}

/*
 * Might pages of this file have been modified in userspace?
 * Note that i_mmap_writable counts all VM_SHARED, VM_MAYWRITE vmas: do_mmap
 * marks vma as VM_SHARED if it is shared, and the file was opened for
 * writing i.e. vma may be mprotected writable even if now readonly.
 *
 * If i_mmap_writable is negative, no new writable mappings are allowed. You
 * can only deny writable mappings, if none exists right now.
 */
static inline int mapping_writably_mapped(const struct address_space *mapping)
{
        return atomic_read(&mapping->i_mmap_writable) > 0;
}

static inline int mapping_map_writable(struct address_space *mapping)
{
        return atomic_inc_unless_negative(&mapping->i_mmap_writable) ?
                0 : -EPERM;
}

static inline void mapping_unmap_writable(struct address_space *mapping)
{
        atomic_dec(&mapping->i_mmap_writable);
}

static inline int mapping_deny_writable(struct address_space *mapping)
{
        return atomic_dec_unless_positive(&mapping->i_mmap_writable) ?
                0 : -EBUSY;
}

static inline void mapping_allow_writable(struct address_space *mapping)
{
        atomic_inc(&mapping->i_mmap_writable);
}

/*
 * Use sequence counter to get consistent i_size on 32-bit processors.
 */
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
#include <linux/seqlock.h>
#define __NEED_I_SIZE_ORDERED
#define i_size_ordered_init(inode) seqcount_init(&inode->i_size_seqcount)
#else
#define i_size_ordered_init(inode) do { } while (0)
#endif

struct posix_acl;
#define ACL_NOT_CACHED ((void *)(-1))
/*
 * ACL_DONT_CACHE is for stacked filesystems, that rely on underlying fs to
 * cache the ACL.  This also means that ->get_inode_acl() can be called in RCU
 * mode with the LOOKUP_RCU flag.
 */
#define ACL_DONT_CACHE ((void *)(-3))

static inline struct posix_acl *
uncached_acl_sentinel(struct task_struct *task)
{
        return (void *)task + 1;
}

static inline bool
is_uncached_acl(struct posix_acl *acl)
{
        return (long)acl & 1;
}

#define IOP_FASTPERM        0x0001
#define IOP_LOOKUP        0x0002
#define IOP_NOFOLLOW        0x0004
#define IOP_XATTR        0x0008
#define IOP_DEFAULT_READLINK        0x0010
#define IOP_MGTIME        0x0020
#define IOP_CACHED_LINK        0x0040

/*
 * Inode state bits.  Protected by inode->i_lock
 *
 * Four bits determine the dirty state of the inode: I_DIRTY_SYNC,
 * I_DIRTY_DATASYNC, I_DIRTY_PAGES, and I_DIRTY_TIME.
 *
 * Four bits define the lifetime of an inode.  Initially, inodes are I_NEW,
 * until that flag is cleared.  I_WILL_FREE, I_FREEING and I_CLEAR are set at
 * various stages of removing an inode.
 *
 * Two bits are used for locking and completion notification, I_NEW and I_SYNC.
 *
 * I_DIRTY_SYNC                Inode is dirty, but doesn't have to be written on
 *                        fdatasync() (unless I_DIRTY_DATASYNC is also set).
 *                        Timestamp updates are the usual cause.
 * I_DIRTY_DATASYNC        Data-related inode changes pending.  We keep track of
 *                        these changes separately from I_DIRTY_SYNC so that we
 *                        don't have to write inode on fdatasync() when only
 *                        e.g. the timestamps have changed.
 * I_DIRTY_PAGES        Inode has dirty pages.  Inode itself may be clean.
 * I_DIRTY_TIME                The inode itself has dirty timestamps, and the
 *                        lazytime mount option is enabled.  We keep track of this
 *                        separately from I_DIRTY_SYNC in order to implement
 *                        lazytime.  This gets cleared if I_DIRTY_INODE
 *                        (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But
 *                        I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already
 *                        in place because writeback might already be in progress
 *                        and we don't want to lose the time update
 * I_NEW                Serves as both a mutex and completion notification.
 *                        New inodes set I_NEW.  If two processes both create
 *                        the same inode, one of them will release its inode and
 *                        wait for I_NEW to be released before returning.
 *                        Inodes in I_WILL_FREE, I_FREEING or I_CLEAR state can
 *                        also cause waiting on I_NEW, without I_NEW actually
 *                        being set.  find_inode() uses this to prevent returning
 *                        nearly-dead inodes.
 * I_WILL_FREE                Must be set when calling write_inode_now() if i_count
 *                        is zero.  I_FREEING must be set when I_WILL_FREE is
 *                        cleared.
 * I_FREEING                Set when inode is about to be freed but still has dirty
 *                        pages or buffers attached or the inode itself is still
 *                        dirty.
 * I_CLEAR                Added by clear_inode().  In this state the inode is
 *                        clean and can be destroyed.  Inode keeps I_FREEING.
 *
 *                        Inodes that are I_WILL_FREE, I_FREEING or I_CLEAR are
 *                        prohibited for many purposes.  iget() must wait for
 *                        the inode to be completely released, then create it
 *                        anew.  Other functions will just ignore such inodes,
 *                        if appropriate.  I_NEW is used for waiting.
 *
 * I_SYNC                Writeback of inode is running. The bit is set during
 *                        data writeback, and cleared with a wakeup on the bit
 *                        address once it is done. The bit is also used to pin
 *                        the inode in memory for flusher thread.
 *
 * I_REFERENCED                Marks the inode as recently references on the LRU list.
 *
 * I_WB_SWITCH                Cgroup bdi_writeback switching in progress.  Used to
 *                        synchronize competing switching instances and to tell
 *                        wb stat updates to grab the i_pages lock.  See
 *                        inode_switch_wbs_work_fn() for details.
 *
 * I_OVL_INUSE                Used by overlayfs to get exclusive ownership on upper
 *                        and work dirs among overlayfs mounts.
 *
 * I_CREATING                New object's inode in the middle of setting up.
 *
 * I_DONTCACHE                Evict inode as soon as it is not used anymore.
 *
 * I_SYNC_QUEUED        Inode is queued in b_io or b_more_io writeback lists.
 *                        Used to detect that mark_inode_dirty() should not move
 *                        inode between dirty lists.
 *
 * I_PINNING_FSCACHE_WB        Inode is pinning an fscache object for writeback.
 *
 * I_LRU_ISOLATING        Inode is pinned being isolated from LRU without holding
 *                        i_count.
 *
 * Q: What is the difference between I_WILL_FREE and I_FREEING?
 *
 * __I_{SYNC,NEW,LRU_ISOLATING} are used to derive unique addresses to wait
 * upon. There's one free address left.
 */

enum inode_state_bits {
        __I_NEW                        = 0U,
        __I_SYNC                = 1U,
        __I_LRU_ISOLATING        = 2U
        /* reserved wait address bit 3 */
};

enum inode_state_flags_t {
        I_NEW                        = (1U << __I_NEW),
        I_SYNC                        = (1U << __I_SYNC),
        I_LRU_ISOLATING         = (1U << __I_LRU_ISOLATING),
        /* reserved flag bit 3 */
        I_DIRTY_SYNC                = (1U << 4),
        I_DIRTY_DATASYNC        = (1U << 5),
        I_DIRTY_PAGES                = (1U << 6),
        I_WILL_FREE                = (1U << 7),
        I_FREEING                = (1U << 8),
        I_CLEAR                        = (1U << 9),
        I_REFERENCED                = (1U << 10),
        I_LINKABLE                = (1U << 11),
        I_DIRTY_TIME                = (1U << 12),
        I_WB_SWITCH                = (1U << 13),
        I_OVL_INUSE                = (1U << 14),
        I_CREATING                = (1U << 15),
        I_DONTCACHE                = (1U << 16),
        I_SYNC_QUEUED                = (1U << 17),
        I_PINNING_NETFS_WB        = (1U << 18)
};

#define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
#define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES)
#define I_DIRTY_ALL (I_DIRTY | I_DIRTY_TIME)

/*
 * Keep mostly read-only and often accessed (especially for
 * the RCU path lookup and 'stat' data) fields at the beginning
 * of the 'struct inode'
 */
struct inode {
        umode_t                        i_mode;
        unsigned short                i_opflags;
        kuid_t                        i_uid;
        kgid_t                        i_gid;
        unsigned int                i_flags;

#ifdef CONFIG_FS_POSIX_ACL
        struct posix_acl        *i_acl;
        struct posix_acl        *i_default_acl;
#endif

        const struct inode_operations        *i_op;
        struct super_block        *i_sb;
        struct address_space        *i_mapping;

#ifdef CONFIG_SECURITY
        void                        *i_security;
#endif

        /* Stat data, not accessed from path walking */
        unsigned long                i_ino;
        /*
         * Filesystems may only read i_nlink directly.  They shall use the
         * following functions for modification:
         *
         *    (set|clear|inc|drop)_nlink
         *    inode_(inc|dec)_link_count
         */
        union {
                const unsigned int i_nlink;
                unsigned int __i_nlink;
        };
        dev_t                        i_rdev;
        loff_t                        i_size;
        time64_t                i_atime_sec;
        time64_t                i_mtime_sec;
        time64_t                i_ctime_sec;
        u32                        i_atime_nsec;
        u32                        i_mtime_nsec;
        u32                        i_ctime_nsec;
        u32                        i_generation;
        spinlock_t                i_lock;        /* i_blocks, i_bytes, maybe i_size */
        unsigned short          i_bytes;
        u8                        i_blkbits;
        enum rw_hint                i_write_hint;
        blkcnt_t                i_blocks;

#ifdef __NEED_I_SIZE_ORDERED
        seqcount_t                i_size_seqcount;
#endif

        /* Misc */
        enum inode_state_flags_t        i_state;
        /* 32-bit hole */
        struct rw_semaphore        i_rwsem;

        unsigned long                dirtied_when;        /* jiffies of first dirtying */
        unsigned long                dirtied_time_when;

        struct hlist_node        i_hash;
        struct list_head        i_io_list;        /* backing dev IO list */
#ifdef CONFIG_CGROUP_WRITEBACK
        struct bdi_writeback        *i_wb;                /* the associated cgroup wb */

        /* foreign inode detection, see wbc_detach_inode() */
        int                        i_wb_frn_winner;
        u16                        i_wb_frn_avg_time;
        u16                        i_wb_frn_history;
#endif
        struct list_head        i_lru;                /* inode LRU list */
        struct list_head        i_sb_list;
        struct list_head        i_wb_list;        /* backing dev writeback list */
        union {
                struct hlist_head        i_dentry;
                struct rcu_head                i_rcu;
        };
        atomic64_t                i_version;
        atomic64_t                i_sequence; /* see futex */
        atomic_t                i_count;
        atomic_t                i_dio_count;
        atomic_t                i_writecount;
#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
        atomic_t                i_readcount; /* struct files open RO */
#endif
        union {
                const struct file_operations        *i_fop;        /* former ->i_op->default_file_ops */
                void (*free_inode)(struct inode *);
        };
        struct file_lock_context        *i_flctx;
        struct address_space        i_data;
        union {
                struct list_head        i_devices;
                int                        i_linklen;
        };
        union {
                struct pipe_inode_info        *i_pipe;
                struct cdev                *i_cdev;
                char                        *i_link;
                unsigned                i_dir_seq;
        };


#ifdef CONFIG_FSNOTIFY
        __u32                        i_fsnotify_mask; /* all events this inode cares about */
        /* 32-bit hole reserved for expanding i_fsnotify_mask */
        struct fsnotify_mark_connector __rcu        *i_fsnotify_marks;
#endif

        void                        *i_private; /* fs or device private pointer */
} __randomize_layout;

static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
{
        VFS_WARN_ON_INODE(strlen(link) != linklen, inode);
        VFS_WARN_ON_INODE(inode->i_opflags & IOP_CACHED_LINK, inode);
        inode->i_link = link;
        inode->i_linklen = linklen;
        inode->i_opflags |= IOP_CACHED_LINK;
}

/*
 * Get bit address from inode->i_state to use with wait_var_event()
 * infrastructre.
 */
#define inode_state_wait_address(inode, bit) ((char *)&(inode)->i_state + (bit))

struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe,
                                            struct inode *inode, u32 bit);

static inline void inode_wake_up_bit(struct inode *inode, u32 bit)
{
        /* Caller is responsible for correct memory barriers. */
        wake_up_var(inode_state_wait_address(inode, bit));
}

struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode);

static inline unsigned int i_blocksize(const struct inode *node)
{
        return (1 << node->i_blkbits);
}

static inline int inode_unhashed(struct inode *inode)
{
        return hlist_unhashed(&inode->i_hash);
}

/*
 * __mark_inode_dirty expects inodes to be hashed.  Since we don't
 * want special inodes in the fileset inode space, we make them
 * appear hashed, but do not put on any lists.  hlist_del()
 * will work fine and require no locking.
 */
static inline void inode_fake_hash(struct inode *inode)
{
        hlist_add_fake(&inode->i_hash);
}

/*
 * inode->i_rwsem nesting subclasses for the lock validator:
 *
 * 0: the object of the current VFS operation
 * 1: parent
 * 2: child/target
 * 3: xattr
 * 4: second non-directory
 * 5: second parent (when locking independent directories in rename)
 *
 * I_MUTEX_NONDIR2 is for certain operations (such as rename) which lock two
 * non-directories at once.
 *
 * The locking order between these classes is
 * parent[2] -> child -> grandchild -> normal -> xattr -> second non-directory
 */
enum inode_i_mutex_lock_class
{
        I_MUTEX_NORMAL,
        I_MUTEX_PARENT,
        I_MUTEX_CHILD,
        I_MUTEX_XATTR,
        I_MUTEX_NONDIR2,
        I_MUTEX_PARENT2,
};

static inline void inode_lock(struct inode *inode)
{
        down_write(&inode->i_rwsem);
}

static inline __must_check int inode_lock_killable(struct inode *inode)
{
        return down_write_killable(&inode->i_rwsem);
}

static inline void inode_unlock(struct inode *inode)
{
        up_write(&inode->i_rwsem);
}

static inline void inode_lock_shared(struct inode *inode)
{
        down_read(&inode->i_rwsem);
}

static inline __must_check int inode_lock_shared_killable(struct inode *inode)
{
        return down_read_killable(&inode->i_rwsem);
}

static inline void inode_unlock_shared(struct inode *inode)
{
        up_read(&inode->i_rwsem);
}

static inline int inode_trylock(struct inode *inode)
{
        return down_write_trylock(&inode->i_rwsem);
}

static inline int inode_trylock_shared(struct inode *inode)
{
        return down_read_trylock(&inode->i_rwsem);
}

static inline int inode_is_locked(struct inode *inode)
{
        return rwsem_is_locked(&inode->i_rwsem);
}

static inline void inode_lock_nested(struct inode *inode, unsigned subclass)
{
        down_write_nested(&inode->i_rwsem, subclass);
}

static inline void inode_lock_shared_nested(struct inode *inode, unsigned subclass)
{
        down_read_nested(&inode->i_rwsem, subclass);
}

static inline void filemap_invalidate_lock(struct address_space *mapping)
{
        down_write(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_unlock(struct address_space *mapping)
{
        up_write(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_lock_shared(struct address_space *mapping)
{
        down_read(&mapping->invalidate_lock);
}

static inline int filemap_invalidate_trylock_shared(
                                        struct address_space *mapping)
{
        return down_read_trylock(&mapping->invalidate_lock);
}

static inline void filemap_invalidate_unlock_shared(
                                        struct address_space *mapping)
{
        up_read(&mapping->invalidate_lock);
}

void lock_two_nondirectories(struct inode *, struct inode*);
void unlock_two_nondirectories(struct inode *, struct inode*);

void filemap_invalidate_lock_two(struct address_space *mapping1,
                                 struct address_space *mapping2);
void filemap_invalidate_unlock_two(struct address_space *mapping1,
                                   struct address_space *mapping2);


/*
 * NOTE: in a 32bit arch with a preemptable kernel and
 * an UP compile the i_size_read/write must be atomic
 * with respect to the local cpu (unlike with preempt disabled),
 * but they don't need to be atomic with respect to other cpus like in
 * true SMP (so they need either to either locally disable irq around
 * the read or for example on x86 they can be still implemented as a
 * cmpxchg8b without the need of the lock prefix). For SMP compiles
 * and 64bit archs it makes no difference if preempt is enabled or not.
 */
static inline loff_t i_size_read(const struct inode *inode)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        loff_t i_size;
        unsigned int seq;

        do {
                seq = read_seqcount_begin(&inode->i_size_seqcount);
                i_size = inode->i_size;
        } while (read_seqcount_retry(&inode->i_size_seqcount, seq));
        return i_size;
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
        loff_t i_size;

        preempt_disable();
        i_size = inode->i_size;
        preempt_enable();
        return i_size;
#else
        /* Pairs with smp_store_release() in i_size_write() */
        return smp_load_acquire(&inode->i_size);
#endif
}

/*
 * NOTE: unlike i_size_read(), i_size_write() does need locking around it
 * (normally i_rwsem), otherwise on 32bit/SMP an update of i_size_seqcount
 * can be lost, resulting in subsequent i_size_read() calls spinning forever.
 */
static inline void i_size_write(struct inode *inode, loff_t i_size)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        preempt_disable();
        write_seqcount_begin(&inode->i_size_seqcount);
        inode->i_size = i_size;
        write_seqcount_end(&inode->i_size_seqcount);
        preempt_enable();
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
        preempt_disable();
        inode->i_size = i_size;
        preempt_enable();
#else
        /*
         * Pairs with smp_load_acquire() in i_size_read() to ensure
         * changes related to inode size (such as page contents) are
         * visible before we see the changed inode size.
         */
        smp_store_release(&inode->i_size, i_size);
#endif
}

static inline unsigned iminor(const struct inode *inode)
{
        return MINOR(inode->i_rdev);
}

static inline unsigned imajor(const struct inode *inode)
{
        return MAJOR(inode->i_rdev);
}

struct fown_struct {
        struct file *file;        /* backpointer for security modules */
        rwlock_t lock;          /* protects pid, uid, euid fields */
        struct pid *pid;        /* pid or -pgrp where SIGIO should be sent */
        enum pid_type pid_type;        /* Kind of process group SIGIO should be sent to */
        kuid_t uid, euid;        /* uid/euid of process setting the owner */
        int signum;                /* posix.1b rt signal to be delivered on IO */
};

/**
 * struct file_ra_state - Track a file's readahead state.
 * @start: Where the most recent readahead started.
 * @size: Number of pages read in the most recent readahead.
 * @async_size: Numer of pages that were/are not needed immediately
 *      and so were/are genuinely "ahead".  Start next readahead when
 *      the first of these pages is accessed.
 * @ra_pages: Maximum size of a readahead request, copied from the bdi.
 * @order: Preferred folio order used for most recent readahead.
 * @mmap_miss: How many mmap accesses missed in the page cache.
 * @prev_pos: The last byte in the most recent read request.
 *
 * When this structure is passed to ->readahead(), the "most recent"
 * readahead means the current readahead.
 */
struct file_ra_state {
        pgoff_t start;
        unsigned int size;
        unsigned int async_size;
        unsigned int ra_pages;
        unsigned short order;
        unsigned short mmap_miss;
        loff_t prev_pos;
};

/*
 * Check if @index falls in the readahead windows.
 */
static inline int ra_has_index(struct file_ra_state *ra, pgoff_t index)
{
        return (index >= ra->start &&
                index <  ra->start + ra->size);
}

/**
 * struct file - Represents a file
 * @f_lock: Protects f_ep, f_flags. Must not be taken from IRQ context.
 * @f_mode: FMODE_* flags often used in hotpaths
 * @f_op: file operations
 * @f_mapping: Contents of a cacheable, mappable object.
 * @private_data: filesystem or driver specific data
 * @f_inode: cached inode
 * @f_flags: file flags
 * @f_iocb_flags: iocb flags
 * @f_cred: stashed credentials of creator/opener
 * @f_owner: file owner
 * @f_path: path of the file
 * @__f_path: writable alias for @f_path; *ONLY* for core VFS and only before
 *   the file gets open
 * @f_pos_lock: lock protecting file position
 * @f_pipe: specific to pipes
 * @f_pos: file position
 * @f_security: LSM security context of this file
 * @f_wb_err: writeback error
 * @f_sb_err: per sb writeback errors
 * @f_ep: link of all epoll hooks for this file
 * @f_task_work: task work entry point
 * @f_llist: work queue entrypoint
 * @f_ra: file's readahead state
 * @f_freeptr: Pointer used by SLAB_TYPESAFE_BY_RCU file cache (don't touch.)
 * @f_ref: reference count
 */
struct file {
        spinlock_t                        f_lock;
        fmode_t                                f_mode;
        const struct file_operations        *f_op;
        struct address_space                *f_mapping;
        void                                *private_data;
        struct inode                        *f_inode;
        unsigned int                        f_flags;
        unsigned int                        f_iocb_flags;
        const struct cred                *f_cred;
        struct fown_struct                *f_owner;
        /* --- cacheline 1 boundary (64 bytes) --- */
        union {
                const struct path        f_path;
                struct path                __f_path;
        };
        union {
                /* regular files (with FMODE_ATOMIC_POS) and directories */
                struct mutex                f_pos_lock;
                /* pipes */
                u64                        f_pipe;
        };
        loff_t                                f_pos;
#ifdef CONFIG_SECURITY
        void                                *f_security;
#endif
        /* --- cacheline 2 boundary (128 bytes) --- */
        errseq_t                        f_wb_err;
        errseq_t                        f_sb_err;
#ifdef CONFIG_EPOLL
        struct hlist_head                *f_ep;
#endif
        union {
                struct callback_head        f_task_work;
                struct llist_node        f_llist;
                struct file_ra_state        f_ra;
                freeptr_t                f_freeptr;
        };
        file_ref_t                        f_ref;
        /* --- cacheline 3 boundary (192 bytes) --- */
} __randomize_layout
  __attribute__((aligned(4)));        /* lest something weird decides that 2 is OK */

struct file_handle {
        __u32 handle_bytes;
        int handle_type;
        /* file identifier */
        unsigned char f_handle[] __counted_by(handle_bytes);
};

static inline struct file *get_file(struct file *f)
{
        file_ref_inc(&f->f_ref);
        return f;
}

struct file *get_file_rcu(struct file __rcu **f);
struct file *get_file_active(struct file **f);

#define file_count(f)        file_ref_read(&(f)->f_ref)

#define        MAX_NON_LFS        ((1UL<<31) - 1)

/* Page cache limit. The filesystems should put that into their s_maxbytes 
   limits, otherwise bad things can happen in VM. */ 
#if BITS_PER_LONG==32
#define MAX_LFS_FILESIZE        ((loff_t)ULONG_MAX << PAGE_SHIFT)
#elif BITS_PER_LONG==64
#define MAX_LFS_FILESIZE         ((loff_t)LLONG_MAX)
#endif

/* legacy typedef, should eventually be removed */
typedef void *fl_owner_t;

struct file_lock;
struct file_lease;

/* The following constant reflects the upper bound of the file/locking space */
#ifndef OFFSET_MAX
#define OFFSET_MAX        type_max(loff_t)
#define OFFT_OFFSET_MAX        type_max(off_t)
#endif

int file_f_owner_allocate(struct file *file);
static inline struct fown_struct *file_f_owner(const struct file *file)
{
        return READ_ONCE(file->f_owner);
}

extern void send_sigio(struct fown_struct *fown, int fd, int band);

static inline struct inode *file_inode(const struct file *f)
{
        return f->f_inode;
}

/*
 * file_dentry() is a relic from the days that overlayfs was using files with a
 * "fake" path, meaning, f_path on overlayfs and f_inode on underlying fs.
 * In those days, file_dentry() was needed to get the underlying fs dentry that
 * matches f_inode.
 * Files with "fake" path should not exist nowadays, so use an assertion to make
 * sure that file_dentry() was not papering over filesystem bugs.
 */
static inline struct dentry *file_dentry(const struct file *file)
{
        struct dentry *dentry = file->f_path.dentry;

        WARN_ON_ONCE(d_inode(dentry) != file_inode(file));
        return dentry;
}

struct fasync_struct {
        rwlock_t                fa_lock;
        int                        magic;
        int                        fa_fd;
        struct fasync_struct        *fa_next; /* singly linked list */
        struct file                *fa_file;
        struct rcu_head                fa_rcu;
};

#define FASYNC_MAGIC 0x4601

/* SMP safe fasync helpers: */
extern int fasync_helper(int, struct file *, int, struct fasync_struct **);
extern struct fasync_struct *fasync_insert_entry(int, struct file *, struct fasync_struct **, struct fasync_struct *);
extern int fasync_remove_entry(struct file *, struct fasync_struct **);
extern struct fasync_struct *fasync_alloc(void);
extern void fasync_free(struct fasync_struct *);

/* can be called from interrupts */
extern void kill_fasync(struct fasync_struct **, int, int);

extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force);
extern int f_setown(struct file *filp, int who, int force);
extern void f_delown(struct file *filp);
extern pid_t f_getown(struct file *filp);
extern int send_sigurg(struct file *file);

/*
 * sb->s_flags.  Note that these mirror the equivalent MS_* flags where
 * represented in both.
 */
#define SB_RDONLY       BIT(0)        /* Mount read-only */
#define SB_NOSUID       BIT(1)        /* Ignore suid and sgid bits */
#define SB_NODEV        BIT(2)        /* Disallow access to device special files */
#define SB_NOEXEC       BIT(3)        /* Disallow program execution */
#define SB_SYNCHRONOUS  BIT(4)        /* Writes are synced at once */
#define SB_MANDLOCK     BIT(6)        /* Allow mandatory locks on an FS */
#define SB_DIRSYNC      BIT(7)        /* Directory modifications are synchronous */
#define SB_NOATIME      BIT(10)        /* Do not update access times. */
#define SB_NODIRATIME   BIT(11)        /* Do not update directory access times */
#define SB_SILENT       BIT(15)
#define SB_POSIXACL     BIT(16)        /* Supports POSIX ACLs */
#define SB_INLINECRYPT  BIT(17)        /* Use blk-crypto for encrypted files */
#define SB_KERNMOUNT    BIT(22)        /* this is a kern_mount call */
#define SB_I_VERSION    BIT(23)        /* Update inode I_version field */
#define SB_LAZYTIME     BIT(25)        /* Update the on-disk [acm]times lazily */

/* These sb flags are internal to the kernel */
#define SB_DEAD         BIT(21)
#define SB_DYING        BIT(24)
#define SB_FORCE        BIT(27)
#define SB_NOSEC        BIT(28)
#define SB_BORN         BIT(29)
#define SB_ACTIVE       BIT(30)
#define SB_NOUSER       BIT(31)

/* These flags relate to encoding and casefolding */
#define SB_ENC_STRICT_MODE_FL                (1 << 0)
#define SB_ENC_NO_COMPAT_FALLBACK_FL        (1 << 1)

#define sb_has_strict_encoding(sb) \
        (sb->s_encoding_flags & SB_ENC_STRICT_MODE_FL)

#if IS_ENABLED(CONFIG_UNICODE)
#define sb_no_casefold_compat_fallback(sb) \
        (sb->s_encoding_flags & SB_ENC_NO_COMPAT_FALLBACK_FL)
#else
#define sb_no_casefold_compat_fallback(sb) (1)
#endif

/*
 *        Umount options
 */

#define MNT_FORCE        0x00000001        /* Attempt to forcibily umount */
#define MNT_DETACH        0x00000002        /* Just detach from the tree */
#define MNT_EXPIRE        0x00000004        /* Mark for expiry */
#define UMOUNT_NOFOLLOW        0x00000008        /* Don't follow symlink on umount */
#define UMOUNT_UNUSED        0x80000000        /* Flag guaranteed to be unused */

/* sb->s_iflags */
#define SB_I_CGROUPWB        0x00000001        /* cgroup-aware writeback enabled */
#define SB_I_NOEXEC        0x00000002        /* Ignore executables on this fs */
#define SB_I_NODEV        0x00000004        /* Ignore devices on this fs */
#define SB_I_STABLE_WRITES 0x00000008        /* don't modify blks until WB is done */

/* sb->s_iflags to limit user namespace mounts */
#define SB_I_USERNS_VISIBLE                0x00000010 /* fstype already mounted */
#define SB_I_IMA_UNVERIFIABLE_SIGNATURE        0x00000020
#define SB_I_UNTRUSTED_MOUNTER                0x00000040
#define SB_I_EVM_HMAC_UNSUPPORTED        0x00000080

#define SB_I_SKIP_SYNC        0x00000100        /* Skip superblock at global sync */
#define SB_I_PERSB_BDI        0x00000200        /* has a per-sb bdi */
#define SB_I_TS_EXPIRY_WARNED 0x00000400 /* warned about timestamp range expiry */
#define SB_I_RETIRED        0x00000800        /* superblock shouldn't be reused */
#define SB_I_NOUMASK        0x00001000        /* VFS does not apply umask */
#define SB_I_NOIDMAP        0x00002000        /* No idmapped mounts on this superblock */
#define SB_I_ALLOW_HSM        0x00004000        /* Allow HSM events on this superblock */

/* Possible states of 'frozen' field */
enum {
        SB_UNFROZEN = 0,                /* FS is unfrozen */
        SB_FREEZE_WRITE        = 1,                /* Writes, dir ops, ioctls frozen */
        SB_FREEZE_PAGEFAULT = 2,        /* Page faults stopped as well */
        SB_FREEZE_FS = 3,                /* For internal FS use (e.g. to stop
                                         * internal threads if needed) */
        SB_FREEZE_COMPLETE = 4,                /* ->freeze_fs finished successfully */
};

#define SB_FREEZE_LEVELS (SB_FREEZE_COMPLETE - 1)

struct sb_writers {
        unsigned short                        frozen;                /* Is sb frozen? */
        int                                freeze_kcount;        /* How many kernel freeze requests? */
        int                                freeze_ucount;        /* How many userspace freeze requests? */
        const void                        *freeze_owner;        /* Owner of the freeze */
        struct percpu_rw_semaphore        rw_sem[SB_FREEZE_LEVELS];
};

struct mount;

struct super_block {
        struct list_head        s_list;                /* Keep this first */
        dev_t                        s_dev;                /* search index; _not_ kdev_t */
        unsigned char                s_blocksize_bits;
        unsigned long                s_blocksize;
        loff_t                        s_maxbytes;        /* Max file size */
        struct file_system_type        *s_type;
        const struct super_operations        *s_op;
        const struct dquot_operations        *dq_op;
        const struct quotactl_ops        *s_qcop;
        const struct export_operations *s_export_op;
        unsigned long                s_flags;
        unsigned long                s_iflags;        /* internal SB_I_* flags */
        unsigned long                s_magic;
        struct dentry                *s_root;
        struct rw_semaphore        s_umount;
        int                        s_count;
        atomic_t                s_active;
#ifdef CONFIG_SECURITY
        void                    *s_security;
#endif
        const struct xattr_handler * const *s_xattr;
#ifdef CONFIG_FS_ENCRYPTION
        const struct fscrypt_operations        *s_cop;
        struct fscrypt_keyring        *s_master_keys; /* master crypto keys in use */
#endif
#ifdef CONFIG_FS_VERITY
        const struct fsverity_operations *s_vop;
#endif
#if IS_ENABLED(CONFIG_UNICODE)
        struct unicode_map *s_encoding;
        __u16 s_encoding_flags;
#endif
        struct hlist_bl_head        s_roots;        /* alternate root dentries for NFS */
        struct mount                *s_mounts;        /* list of mounts; _not_ for fs use */
        struct block_device        *s_bdev;        /* can go away once we use an accessor for @s_bdev_file */
        struct file                *s_bdev_file;
        struct backing_dev_info *s_bdi;
        struct mtd_info                *s_mtd;
        struct hlist_node        s_instances;
        unsigned int                s_quota_types;        /* Bitmask of supported quota types */
        struct quota_info        s_dquot;        /* Diskquota specific options */

        struct sb_writers        s_writers;

        /*
         * Keep s_fs_info, s_time_gran, s_fsnotify_mask, and
         * s_fsnotify_info together for cache efficiency. They are frequently
         * accessed and rarely modified.
         */
        void                        *s_fs_info;        /* Filesystem private info */

        /* Granularity of c/m/atime in ns (cannot be worse than a second) */
        u32                        s_time_gran;
        /* Time limits for c/m/atime in seconds */
        time64_t                   s_time_min;
        time64_t                   s_time_max;
#ifdef CONFIG_FSNOTIFY
        u32                        s_fsnotify_mask;
        struct fsnotify_sb_info        *s_fsnotify_info;
#endif

        /*
         * q: why are s_id and s_sysfs_name not the same? both are human
         * readable strings that identify the filesystem
         * a: s_id is allowed to change at runtime; it's used in log messages,
         * and we want to when a device starts out as single device (s_id is dev
         * name) but then a device is hot added and we have to switch to
         * identifying it by UUID
         * but s_sysfs_name is a handle for programmatic access, and can't
         * change at runtime
         */
        char                        s_id[32];        /* Informational name */
        uuid_t                        s_uuid;                /* UUID */
        u8                        s_uuid_len;        /* Default 16, possibly smaller for weird filesystems */

        /* if set, fs shows up under sysfs at /sys/fs/$FSTYP/s_sysfs_name */
        char                        s_sysfs_name[UUID_STRING_LEN + 1];

        unsigned int                s_max_links;
        unsigned int                s_d_flags;        /* default d_flags for dentries */

        /*
         * The next field is for VFS *only*. No filesystems have any business
         * even looking at it. You had been warned.
         */
        struct mutex s_vfs_rename_mutex;        /* Kludge */

        /*
         * Filesystem subtype.  If non-empty the filesystem type field
         * in /proc/mounts will be "type.subtype"
         */
        const char *s_subtype;

        const struct dentry_operations *__s_d_op; /* default d_op for dentries */

        struct shrinker *s_shrink;        /* per-sb shrinker handle */

        /* Number of inodes with nlink == 0 but still referenced */
        atomic_long_t s_remove_count;

        /* Read-only state of the superblock is being changed */
        int s_readonly_remount;

        /* per-sb errseq_t for reporting writeback errors via syncfs */
        errseq_t s_wb_err;

        /* AIO completions deferred from interrupt context */
        struct workqueue_struct *s_dio_done_wq;
        struct hlist_head s_pins;

        /*
         * Owning user namespace and default context in which to
         * interpret filesystem uids, gids, quotas, device nodes,
         * xattrs and security labels.
         */
        struct user_namespace *s_user_ns;

        /*
         * The list_lru structure is essentially just a pointer to a table
         * of per-node lru lists, each of which has its own spinlock.
         * There is no need to put them into separate cachelines.
         */
        struct list_lru                s_dentry_lru;
        struct list_lru                s_inode_lru;
        struct rcu_head                rcu;
        struct work_struct        destroy_work;

        struct mutex                s_sync_lock;        /* sync serialisation lock */

        /*
         * Indicates how deep in a filesystem stack this SB is
         */
        int s_stack_depth;

        /* s_inode_list_lock protects s_inodes */
        spinlock_t                s_inode_list_lock ____cacheline_aligned_in_smp;
        struct list_head        s_inodes;        /* all inodes */

        spinlock_t                s_inode_wblist_lock;
        struct list_head        s_inodes_wb;        /* writeback inodes */
} __randomize_layout;

static inline struct user_namespace *i_user_ns(const struct inode *inode)
{
        return inode->i_sb->s_user_ns;
}

/* Helper functions so that in most cases filesystems will
 * not need to deal directly with kuid_t and kgid_t and can
 * instead deal with the raw numeric values that are stored
 * in the filesystem.
 */
static inline uid_t i_uid_read(const struct inode *inode)
{
        return from_kuid(i_user_ns(inode), inode->i_uid);
}

static inline gid_t i_gid_read(const struct inode *inode)
{
        return from_kgid(i_user_ns(inode), inode->i_gid);
}

static inline void i_uid_write(struct inode *inode, uid_t uid)
{
        inode->i_uid = make_kuid(i_user_ns(inode), uid);
}

static inline void i_gid_write(struct inode *inode, gid_t gid)
{
        inode->i_gid = make_kgid(i_user_ns(inode), gid);
}

/**
 * i_uid_into_vfsuid - map an inode's i_uid down according to an idmapping
 * @idmap: idmap of the mount the inode was found from
 * @inode: inode to map
 *
 * Return: whe inode's i_uid mapped down according to @idmap.
 * If the inode's i_uid has no mapping INVALID_VFSUID is returned.
 */
static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
                                         const struct inode *inode)
{
        return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid);
}

/**
 * i_uid_needs_update - check whether inode's i_uid needs to be updated
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Check whether the $inode's i_uid field needs to be updated taking idmapped
 * mounts into account if the filesystem supports it.
 *
 * Return: true if @inode's i_uid field needs to be updated, false if not.
 */
static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
                                      const struct iattr *attr,
                                      const struct inode *inode)
{
        return ((attr->ia_valid & ATTR_UID) &&
                !vfsuid_eq(attr->ia_vfsuid,
                           i_uid_into_vfsuid(idmap, inode)));
}

/**
 * i_uid_update - update @inode's i_uid field
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Safely update @inode's i_uid field translating the vfsuid of any idmapped
 * mount into the filesystem kuid.
 */
static inline void i_uid_update(struct mnt_idmap *idmap,
                                const struct iattr *attr,
                                struct inode *inode)
{
        if (attr->ia_valid & ATTR_UID)
                inode->i_uid = from_vfsuid(idmap, i_user_ns(inode),
                                           attr->ia_vfsuid);
}

/**
 * i_gid_into_vfsgid - map an inode's i_gid down according to an idmapping
 * @idmap: idmap of the mount the inode was found from
 * @inode: inode to map
 *
 * Return: the inode's i_gid mapped down according to @idmap.
 * If the inode's i_gid has no mapping INVALID_VFSGID is returned.
 */
static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
                                         const struct inode *inode)
{
        return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid);
}

/**
 * i_gid_needs_update - check whether inode's i_gid needs to be updated
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Check whether the $inode's i_gid field needs to be updated taking idmapped
 * mounts into account if the filesystem supports it.
 *
 * Return: true if @inode's i_gid field needs to be updated, false if not.
 */
static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
                                      const struct iattr *attr,
                                      const struct inode *inode)
{
        return ((attr->ia_valid & ATTR_GID) &&
                !vfsgid_eq(attr->ia_vfsgid,
                           i_gid_into_vfsgid(idmap, inode)));
}

/**
 * i_gid_update - update @inode's i_gid field
 * @idmap: idmap of the mount the inode was found from
 * @attr: the new attributes of @inode
 * @inode: the inode to update
 *
 * Safely update @inode's i_gid field translating the vfsgid of any idmapped
 * mount into the filesystem kgid.
 */
static inline void i_gid_update(struct mnt_idmap *idmap,
                                const struct iattr *attr,
                                struct inode *inode)
{
        if (attr->ia_valid & ATTR_GID)
                inode->i_gid = from_vfsgid(idmap, i_user_ns(inode),
                                           attr->ia_vfsgid);
}

/**
 * inode_fsuid_set - initialize inode's i_uid field with callers fsuid
 * @inode: inode to initialize
 * @idmap: idmap of the mount the inode was found from
 *
 * Initialize the i_uid field of @inode. If the inode was found/created via
 * an idmapped mount map the caller's fsuid according to @idmap.
 */
static inline void inode_fsuid_set(struct inode *inode,
                                   struct mnt_idmap *idmap)
{
        inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode));
}

/**
 * inode_fsgid_set - initialize inode's i_gid field with callers fsgid
 * @inode: inode to initialize
 * @idmap: idmap of the mount the inode was found from
 *
 * Initialize the i_gid field of @inode. If the inode was found/created via
 * an idmapped mount map the caller's fsgid according to @idmap.
 */
static inline void inode_fsgid_set(struct inode *inode,
                                   struct mnt_idmap *idmap)
{
        inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode));
}

/**
 * fsuidgid_has_mapping() - check whether caller's fsuid/fsgid is mapped
 * @sb: the superblock we want a mapping in
 * @idmap: idmap of the relevant mount
 *
 * Check whether the caller's fsuid and fsgid have a valid mapping in the
 * s_user_ns of the superblock @sb. If the caller is on an idmapped mount map
 * the caller's fsuid and fsgid according to the @idmap first.
 *
 * Return: true if fsuid and fsgid is mapped, false if not.
 */
static inline bool fsuidgid_has_mapping(struct super_block *sb,
                                        struct mnt_idmap *idmap)
{
        struct user_namespace *fs_userns = sb->s_user_ns;
        kuid_t kuid;
        kgid_t kgid;

        kuid = mapped_fsuid(idmap, fs_userns);
        if (!uid_valid(kuid))
                return false;
        kgid = mapped_fsgid(idmap, fs_userns);
        if (!gid_valid(kgid))
                return false;
        return kuid_has_mapping(fs_userns, kuid) &&
               kgid_has_mapping(fs_userns, kgid);
}

struct timespec64 current_time(struct inode *inode);
struct timespec64 inode_set_ctime_current(struct inode *inode);
struct timespec64 inode_set_ctime_deleg(struct inode *inode,
                                        struct timespec64 update);

static inline time64_t inode_get_atime_sec(const struct inode *inode)
{
        return inode->i_atime_sec;
}

static inline long inode_get_atime_nsec(const struct inode *inode)
{
        return inode->i_atime_nsec;
}

static inline struct timespec64 inode_get_atime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_atime_sec(inode),
                                 .tv_nsec = inode_get_atime_nsec(inode) };

        return ts;
}

static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode,
                                                      struct timespec64 ts)
{
        inode->i_atime_sec = ts.tv_sec;
        inode->i_atime_nsec = ts.tv_nsec;
        return ts;
}

static inline struct timespec64 inode_set_atime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };

        return inode_set_atime_to_ts(inode, ts);
}

static inline time64_t inode_get_mtime_sec(const struct inode *inode)
{
        return inode->i_mtime_sec;
}

static inline long inode_get_mtime_nsec(const struct inode *inode)
{
        return inode->i_mtime_nsec;
}

static inline struct timespec64 inode_get_mtime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_mtime_sec(inode),
                                 .tv_nsec = inode_get_mtime_nsec(inode) };
        return ts;
}

static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode,
                                                      struct timespec64 ts)
{
        inode->i_mtime_sec = ts.tv_sec;
        inode->i_mtime_nsec = ts.tv_nsec;
        return ts;
}

static inline struct timespec64 inode_set_mtime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };
        return inode_set_mtime_to_ts(inode, ts);
}

/*
 * Multigrain timestamps
 *
 * Conditionally use fine-grained ctime and mtime timestamps when there
 * are users actively observing them via getattr. The primary use-case
 * for this is NFS clients that use the ctime to distinguish between
 * different states of the file, and that are often fooled by multiple
 * operations that occur in the same coarse-grained timer tick.
 */
#define I_CTIME_QUERIED                ((u32)BIT(31))

static inline time64_t inode_get_ctime_sec(const struct inode *inode)
{
        return inode->i_ctime_sec;
}

static inline long inode_get_ctime_nsec(const struct inode *inode)
{
        return inode->i_ctime_nsec & ~I_CTIME_QUERIED;
}

static inline struct timespec64 inode_get_ctime(const struct inode *inode)
{
        struct timespec64 ts = { .tv_sec  = inode_get_ctime_sec(inode),
                                 .tv_nsec = inode_get_ctime_nsec(inode) };

        return ts;
}

struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts);

/**
 * inode_set_ctime - set the ctime in the inode
 * @inode: inode in which to set the ctime
 * @sec: tv_sec value to set
 * @nsec: tv_nsec value to set
 *
 * Set the ctime in @inode to { @sec, @nsec }
 */
static inline struct timespec64 inode_set_ctime(struct inode *inode,
                                                time64_t sec, long nsec)
{
        struct timespec64 ts = { .tv_sec  = sec,
                                 .tv_nsec = nsec };

        return inode_set_ctime_to_ts(inode, ts);
}

struct timespec64 simple_inode_init_ts(struct inode *inode);

/*
 * Snapshotting support.
 */

/*
 * These are internal functions, please use sb_start_{write,pagefault,intwrite}
 * instead.
 */
static inline void __sb_end_write(struct super_block *sb, int level)
{
        percpu_up_read(sb->s_writers.rw_sem + level-1);
}

static inline void __sb_start_write(struct super_block *sb, int level)
{
        percpu_down_read_freezable(sb->s_writers.rw_sem + level - 1, true);
}

static inline bool __sb_start_write_trylock(struct super_block *sb, int level)
{
        return percpu_down_read_trylock(sb->s_writers.rw_sem + level - 1);
}

#define __sb_writers_acquired(sb, lev)        \
        percpu_rwsem_acquire(&(sb)->s_writers.rw_sem[(lev)-1], 1, _THIS_IP_)
#define __sb_writers_release(sb, lev)        \
        percpu_rwsem_release(&(sb)->s_writers.rw_sem[(lev)-1], _THIS_IP_)

/**
 * __sb_write_started - check if sb freeze level is held
 * @sb: the super we write to
 * @level: the freeze level
 *
 * * > 0 - sb freeze level is held
 * *   0 - sb freeze level is not held
 * * < 0 - !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN
 */
static inline int __sb_write_started(const struct super_block *sb, int level)
{
        return lockdep_is_held_type(sb->s_writers.rw_sem + level - 1, 1);
}

/**
 * sb_write_started - check if SB_FREEZE_WRITE is held
 * @sb: the super we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 */
static inline bool sb_write_started(const struct super_block *sb)
{
        return __sb_write_started(sb, SB_FREEZE_WRITE);
}

/**
 * sb_write_not_started - check if SB_FREEZE_WRITE is not held
 * @sb: the super we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 */
static inline bool sb_write_not_started(const struct super_block *sb)
{
        return __sb_write_started(sb, SB_FREEZE_WRITE) <= 0;
}

/**
 * file_write_started - check if SB_FREEZE_WRITE is held
 * @file: the file we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 * May be false positive with !S_ISREG, because file_start_write() has
 * no effect on !S_ISREG.
 */
static inline bool file_write_started(const struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_write_started(file_inode(file)->i_sb);
}

/**
 * file_write_not_started - check if SB_FREEZE_WRITE is not held
 * @file: the file we write to
 *
 * May be false positive with !CONFIG_LOCKDEP/LOCK_STATE_UNKNOWN.
 * May be false positive with !S_ISREG, because file_start_write() has
 * no effect on !S_ISREG.
 */
static inline bool file_write_not_started(const struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_write_not_started(file_inode(file)->i_sb);
}

/**
 * sb_end_write - drop write access to a superblock
 * @sb: the super we wrote to
 *
 * Decrement number of writers to the filesystem. Wake up possible waiters
 * wanting to freeze the filesystem.
 */
static inline void sb_end_write(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_WRITE);
}

/**
 * sb_end_pagefault - drop write access to a superblock from a page fault
 * @sb: the super we wrote to
 *
 * Decrement number of processes handling write page fault to the filesystem.
 * Wake up possible waiters wanting to freeze the filesystem.
 */
static inline void sb_end_pagefault(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_PAGEFAULT);
}

/**
 * sb_end_intwrite - drop write access to a superblock for internal fs purposes
 * @sb: the super we wrote to
 *
 * Decrement fs-internal number of writers to the filesystem.  Wake up possible
 * waiters wanting to freeze the filesystem.
 */
static inline void sb_end_intwrite(struct super_block *sb)
{
        __sb_end_write(sb, SB_FREEZE_FS);
}

/**
 * sb_start_write - get write access to a superblock
 * @sb: the super we write to
 *
 * When a process wants to write data or metadata to a file system (i.e. dirty
 * a page or an inode), it should embed the operation in a sb_start_write() -
 * sb_end_write() pair to get exclusion against file system freezing. This
 * function increments number of writers preventing freezing. If the file
 * system is already frozen, the function waits until the file system is
 * thawed.
 *
 * Since freeze protection behaves as a lock, users have to preserve
 * ordering of freeze protection and other filesystem locks. Generally,
 * freeze protection should be the outermost lock. In particular, we have:
 *
 * sb_start_write
 *   -> i_rwsem                        (write path, truncate, directory ops, ...)
 *   -> s_umount                (freeze_super, thaw_super)
 */
static inline void sb_start_write(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_WRITE);
}

static inline bool sb_start_write_trylock(struct super_block *sb)
{
        return __sb_start_write_trylock(sb, SB_FREEZE_WRITE);
}

/**
 * sb_start_pagefault - get write access to a superblock from a page fault
 * @sb: the super we write to
 *
 * When a process starts handling write page fault, it should embed the
 * operation into sb_start_pagefault() - sb_end_pagefault() pair to get
 * exclusion against file system freezing. This is needed since the page fault
 * is going to dirty a page. This function increments number of running page
 * faults preventing freezing. If the file system is already frozen, the
 * function waits until the file system is thawed.
 *
 * Since page fault freeze protection behaves as a lock, users have to preserve
 * ordering of freeze protection and other filesystem locks. It is advised to
 * put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault
 * handling code implies lock dependency:
 *
 * mmap_lock
 *   -> sb_start_pagefault
 */
static inline void sb_start_pagefault(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_PAGEFAULT);
}

/**
 * sb_start_intwrite - get write access to a superblock for internal fs purposes
 * @sb: the super we write to
 *
 * This is the third level of protection against filesystem freezing. It is
 * free for use by a filesystem. The only requirement is that it must rank
 * below sb_start_pagefault.
 *
 * For example filesystem can call sb_start_intwrite() when starting a
 * transaction which somewhat eases handling of freezing for internal sources
 * of filesystem changes (internal fs threads, discarding preallocation on file
 * close, etc.).
 */
static inline void sb_start_intwrite(struct super_block *sb)
{
        __sb_start_write(sb, SB_FREEZE_FS);
}

static inline bool sb_start_intwrite_trylock(struct super_block *sb)
{
        return __sb_start_write_trylock(sb, SB_FREEZE_FS);
}

bool inode_owner_or_capable(struct mnt_idmap *idmap,
                            const struct inode *inode);

/*
 * VFS helper functions..
 */
int vfs_create(struct mnt_idmap *, struct inode *,
               struct dentry *, umode_t, bool);
struct dentry *vfs_mkdir(struct mnt_idmap *, struct inode *,
                         struct dentry *, umode_t);
int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
              umode_t, dev_t);
int vfs_symlink(struct mnt_idmap *, struct inode *,
                struct dentry *, const char *);
int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *,
             struct dentry *, struct inode **);
int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *);
int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
               struct inode **);

/**
 * struct renamedata - contains all information required for renaming
 * @mnt_idmap:     idmap of the mount in which the rename is happening.
 * @old_parent:        parent of source
 * @old_dentry:                source
 * @new_parent:        parent of destination
 * @new_dentry:                destination
 * @delegated_inode:   returns an inode needing a delegation break
 * @flags:             rename flags
 */
struct renamedata {
        struct mnt_idmap *mnt_idmap;
        struct dentry *old_parent;
        struct dentry *old_dentry;
        struct dentry *new_parent;
        struct dentry *new_dentry;
        struct inode **delegated_inode;
        unsigned int flags;
} __randomize_layout;

int vfs_rename(struct renamedata *);

static inline int vfs_whiteout(struct mnt_idmap *idmap,
                               struct inode *dir, struct dentry *dentry)
{
        return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE,
                         WHITEOUT_DEV);
}

struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
                                 const struct path *parentpath,
                                 umode_t mode, int open_flag,
                                 const struct cred *cred);
struct file *kernel_file_open(const struct path *path, int flags,
                              const struct cred *cred);

int vfs_mkobj(struct dentry *, umode_t,
                int (*f)(struct dentry *, umode_t, void *),
                void *);

int vfs_fchown(struct file *file, uid_t user, gid_t group);
int vfs_fchmod(struct file *file, umode_t mode);
int vfs_utimes(const struct path *path, struct timespec64 *times);

#ifdef CONFIG_COMPAT
extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
                                        unsigned long arg);
#else
#define compat_ptr_ioctl NULL
#endif

/*
 * VFS file helper functions.
 */
void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
                      const struct inode *dir, umode_t mode);
extern bool may_open_dev(const struct path *path);
umode_t mode_strip_sgid(struct mnt_idmap *idmap,
                        const struct inode *dir, umode_t mode);
bool in_group_or_capable(struct mnt_idmap *idmap,
                         const struct inode *inode, vfsgid_t vfsgid);

/*
 * This is the "filldir" function type, used by readdir() to let
 * the kernel specify what kind of dirent layout it wants to have.
 * This allows the kernel to read directories into kernel space or
 * to have different dirent layouts depending on the binary type.
 * Return 'true' to keep going and 'false' if there are no more entries.
 */
struct dir_context;
typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
                         unsigned);

struct dir_context {
        filldir_t actor;
        loff_t pos;
        /*
         * Filesystems MUST NOT MODIFY count, but may use as a hint:
         * 0            unknown
         * > 0      space in buffer (assume at least one entry)
         * INT_MAX  unlimited
         */
        int count;
};

/* If OR-ed with d_type, pending signals are not checked */
#define FILLDIR_FLAG_NOINTR        0x1000

/*
 * These flags let !MMU mmap() govern direct device mapping vs immediate
 * copying more easily for MAP_PRIVATE, especially for ROM filesystems.
 *
 * NOMMU_MAP_COPY:        Copy can be mapped (MAP_PRIVATE)
 * NOMMU_MAP_DIRECT:        Can be mapped directly (MAP_SHARED)
 * NOMMU_MAP_READ:        Can be mapped for reading
 * NOMMU_MAP_WRITE:        Can be mapped for writing
 * NOMMU_MAP_EXEC:        Can be mapped for execution
 */
#define NOMMU_MAP_COPY                0x00000001
#define NOMMU_MAP_DIRECT        0x00000008
#define NOMMU_MAP_READ                VM_MAYREAD
#define NOMMU_MAP_WRITE                VM_MAYWRITE
#define NOMMU_MAP_EXEC                VM_MAYEXEC

#define NOMMU_VMFLAGS \
        (NOMMU_MAP_READ | NOMMU_MAP_WRITE | NOMMU_MAP_EXEC)

/*
 * These flags control the behavior of the remap_file_range function pointer.
 * If it is called with len == 0 that means "remap to end of source file".
 * See Documentation/filesystems/vfs.rst for more details about this call.
 *
 * REMAP_FILE_DEDUP: only remap if contents identical (i.e. deduplicate)
 * REMAP_FILE_CAN_SHORTEN: caller can handle a shortened request
 */
#define REMAP_FILE_DEDUP                (1 << 0)
#define REMAP_FILE_CAN_SHORTEN                (1 << 1)

/*
 * These flags signal that the caller is ok with altering various aspects of
 * the behavior of the remap operation.  The changes must be made by the
 * implementation; the vfs remap helper functions can take advantage of them.
 * Flags in this category exist to preserve the quirky behavior of the hoisted
 * btrfs clone/dedupe ioctls.
 */
#define REMAP_FILE_ADVISORY                (REMAP_FILE_CAN_SHORTEN)

/*
 * These flags control the behavior of vfs_copy_file_range().
 * They are not available to the user via syscall.
 *
 * COPY_FILE_SPLICE: call splice direct instead of fs clone/copy ops
 */
#define COPY_FILE_SPLICE                (1 << 0)

struct iov_iter;
struct io_uring_cmd;
struct offset_ctx;

typedef unsigned int __bitwise fop_flags_t;

struct file_operations {
        struct module *owner;
        fop_flags_t fop_flags;
        loff_t (*llseek) (struct file *, loff_t, int);
        ssize_t (*read) (struct file *, char __user *, size_t, loff_t *);
        ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
        ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
        ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
        int (*iopoll)(struct kiocb *kiocb, struct io_comp_batch *,
                        unsigned int flags);
        int (*iterate_shared) (struct file *, struct dir_context *);
        __poll_t (*poll) (struct file *, struct poll_table_struct *);
        long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long);
        long (*compat_ioctl) (struct file *, unsigned int, unsigned long);
        int (*mmap) (struct file *, struct vm_area_struct *);
        int (*open) (struct inode *, struct file *);
        int (*flush) (struct file *, fl_owner_t id);
        int (*release) (struct inode *, struct file *);
        int (*fsync) (struct file *, loff_t, loff_t, int datasync);
        int (*fasync) (int, struct file *, int);
        int (*lock) (struct file *, int, struct file_lock *);
        unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
        int (*check_flags)(int);
        int (*flock) (struct file *, int, struct file_lock *);
        ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int);
        ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int);
        void (*splice_eof)(struct file *file);
        int (*setlease)(struct file *, int, struct file_lease **, void **);
        long (*fallocate)(struct file *file, int mode, loff_t offset,
                          loff_t len);
        void (*show_fdinfo)(struct seq_file *m, struct file *f);
#ifndef CONFIG_MMU
        unsigned (*mmap_capabilities)(struct file *);
#endif
        ssize_t (*copy_file_range)(struct file *, loff_t, struct file *,
                        loff_t, size_t, unsigned int);
        loff_t (*remap_file_range)(struct file *file_in, loff_t pos_in,
                                   struct file *file_out, loff_t pos_out,
                                   loff_t len, unsigned int remap_flags);
        int (*fadvise)(struct file *, loff_t, loff_t, int);
        int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
        int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
                                unsigned int poll_flags);
        int (*mmap_prepare)(struct vm_area_desc *);
} __randomize_layout;

/* Supports async buffered reads */
#define FOP_BUFFER_RASYNC        ((__force fop_flags_t)(1 << 0))
/* Supports async buffered writes */
#define FOP_BUFFER_WASYNC        ((__force fop_flags_t)(1 << 1))
/* Supports synchronous page faults for mappings */
#define FOP_MMAP_SYNC                ((__force fop_flags_t)(1 << 2))
/* Supports non-exclusive O_DIRECT writes from multiple threads */
#define FOP_DIO_PARALLEL_WRITE        ((__force fop_flags_t)(1 << 3))
/* Contains huge pages */
#define FOP_HUGE_PAGES                ((__force fop_flags_t)(1 << 4))
/* Treat loff_t as unsigned (e.g., /dev/mem) */
#define FOP_UNSIGNED_OFFSET        ((__force fop_flags_t)(1 << 5))
/* Supports asynchronous lock callbacks */
#define FOP_ASYNC_LOCK                ((__force fop_flags_t)(1 << 6))
/* File system supports uncached read/write buffered IO */
#define FOP_DONTCACHE                ((__force fop_flags_t)(1 << 7))

/* Wrap a directory iterator that needs exclusive inode access */
int wrap_directory_iterator(struct file *, struct dir_context *,
                            int (*) (struct file *, struct dir_context *));
#define WRAP_DIR_ITER(x) \
        static int shared_##x(struct file *file , struct dir_context *ctx) \
        { return wrap_directory_iterator(file, ctx, x); }

struct inode_operations {
        struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
        const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *);
        int (*permission) (struct mnt_idmap *, struct inode *, int);
        struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);

        int (*readlink) (struct dentry *, char __user *,int);

        int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
                       umode_t, bool);
        int (*link) (struct dentry *,struct inode *,struct dentry *);
        int (*unlink) (struct inode *,struct dentry *);
        int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
                        const char *);
        struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *,
                                 struct dentry *, umode_t);
        int (*rmdir) (struct inode *,struct dentry *);
        int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,
                      umode_t,dev_t);
        int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
                        struct inode *, struct dentry *, unsigned int);
        int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
        int (*getattr) (struct mnt_idmap *, const struct path *,
                        struct kstat *, u32, unsigned int);
        ssize_t (*listxattr) (struct dentry *, char *, size_t);
        int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
                      u64 len);
        int (*update_time)(struct inode *, int);
        int (*atomic_open)(struct inode *, struct dentry *,
                           struct file *, unsigned open_flag,
                           umode_t create_mode);
        int (*tmpfile) (struct mnt_idmap *, struct inode *,
                        struct file *, umode_t);
        struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *,
                                     int);
        int (*set_acl)(struct mnt_idmap *, struct dentry *,
                       struct posix_acl *, int);
        int (*fileattr_set)(struct mnt_idmap *idmap,
                            struct dentry *dentry, struct file_kattr *fa);
        int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa);
        struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
} ____cacheline_aligned;

/* Did the driver provide valid mmap hook configuration? */
static inline bool can_mmap_file(struct file *file)
{
        bool has_mmap = file->f_op->mmap;
        bool has_mmap_prepare = file->f_op->mmap_prepare;

        /* Hooks are mutually exclusive. */
        if (WARN_ON_ONCE(has_mmap && has_mmap_prepare))
                return false;
        if (!has_mmap && !has_mmap_prepare)
                return false;

        return true;
}

int __compat_vma_mmap_prepare(const struct file_operations *f_op,
                struct file *file, struct vm_area_struct *vma);
int compat_vma_mmap_prepare(struct file *file, struct vm_area_struct *vma);

static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma)
{
        if (file->f_op->mmap_prepare)
                return compat_vma_mmap_prepare(file, vma);

        return file->f_op->mmap(file, vma);
}

static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc)
{
        return file->f_op->mmap_prepare(desc);
}

extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
extern ssize_t vfs_copy_file_range(struct file *, loff_t , struct file *,
                                   loff_t, size_t, unsigned int);
int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write);
int __generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                    struct file *file_out, loff_t pos_out,
                                    loff_t *len, unsigned int remap_flags,
                                    const struct iomap_ops *dax_read_ops);
int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                  struct file *file_out, loff_t pos_out,
                                  loff_t *count, unsigned int remap_flags);
extern loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
                                   struct file *file_out, loff_t pos_out,
                                   loff_t len, unsigned int remap_flags);
extern int vfs_dedupe_file_range(struct file *file,
                                 struct file_dedupe_range *same);
extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
                                        struct file *dst_file, loff_t dst_pos,
                                        loff_t len, unsigned int remap_flags);

/**
 * enum freeze_holder - holder of the freeze
 * @FREEZE_HOLDER_KERNEL: kernel wants to freeze or thaw filesystem
 * @FREEZE_HOLDER_USERSPACE: userspace wants to freeze or thaw filesystem
 * @FREEZE_MAY_NEST: whether nesting freeze and thaw requests is allowed
 * @FREEZE_EXCL: a freeze that can only be undone by the owner
 *
 * Indicate who the owner of the freeze or thaw request is and whether
 * the freeze needs to be exclusive or can nest.
 * Without @FREEZE_MAY_NEST, multiple freeze and thaw requests from the
 * same holder aren't allowed. It is however allowed to hold a single
 * @FREEZE_HOLDER_USERSPACE and a single @FREEZE_HOLDER_KERNEL freeze at
 * the same time. This is relied upon by some filesystems during online
 * repair or similar.
 */
enum freeze_holder {
        FREEZE_HOLDER_KERNEL        = (1U << 0),
        FREEZE_HOLDER_USERSPACE        = (1U << 1),
        FREEZE_MAY_NEST                = (1U << 2),
        FREEZE_EXCL                = (1U << 3),
};

struct super_operations {
           struct inode *(*alloc_inode)(struct super_block *sb);
        void (*destroy_inode)(struct inode *);
        void (*free_inode)(struct inode *);

           void (*dirty_inode) (struct inode *, int flags);
        int (*write_inode) (struct inode *, struct writeback_control *wbc);
        int (*drop_inode) (struct inode *);
        void (*evict_inode) (struct inode *);
        void (*put_super) (struct super_block *);
        int (*sync_fs)(struct super_block *sb, int wait);
        int (*freeze_super) (struct super_block *, enum freeze_holder who, const void *owner);
        int (*freeze_fs) (struct super_block *);
        int (*thaw_super) (struct super_block *, enum freeze_holder who, const void *owner);
        int (*unfreeze_fs) (struct super_block *);
        int (*statfs) (struct dentry *, struct kstatfs *);
        int (*remount_fs) (struct super_block *, int *, char *);
        void (*umount_begin) (struct super_block *);

        int (*show_options)(struct seq_file *, struct dentry *);
        int (*show_devname)(struct seq_file *, struct dentry *);
        int (*show_path)(struct seq_file *, struct dentry *);
        int (*show_stats)(struct seq_file *, struct dentry *);
#ifdef CONFIG_QUOTA
        ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t);
        ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t);
        struct dquot __rcu **(*get_dquots)(struct inode *);
#endif
        long (*nr_cached_objects)(struct super_block *,
                                  struct shrink_control *);
        long (*free_cached_objects)(struct super_block *,
                                    struct shrink_control *);
        /*
         * If a filesystem can support graceful removal of a device and
         * continue read-write operations, implement this callback.
         *
         * Return 0 if the filesystem can continue read-write.
         * Non-zero return value or no such callback means the fs will be shutdown
         * as usual.
         */
        int (*remove_bdev)(struct super_block *sb, struct block_device *bdev);
        void (*shutdown)(struct super_block *sb);
};

/*
 * Inode flags - they have no relation to superblock flags now
 */
#define S_SYNC                (1 << 0)  /* Writes are synced at once */
#define S_NOATIME        (1 << 1)  /* Do not update access times */
#define S_APPEND        (1 << 2)  /* Append-only file */
#define S_IMMUTABLE        (1 << 3)  /* Immutable file */
#define S_DEAD                (1 << 4)  /* removed, but still open directory */
#define S_NOQUOTA        (1 << 5)  /* Inode is not counted to quota */
#define S_DIRSYNC        (1 << 6)  /* Directory modifications are synchronous */
#define S_NOCMTIME        (1 << 7)  /* Do not update file c/mtime */
#define S_SWAPFILE        (1 << 8)  /* Do not truncate: swapon got its bmaps */
#define S_PRIVATE        (1 << 9)  /* Inode is fs-internal */
#define S_IMA                (1 << 10) /* Inode has an associated IMA struct */
#define S_AUTOMOUNT        (1 << 11) /* Automount/referral quasi-directory */
#define S_NOSEC                (1 << 12) /* no suid or xattr security attributes */
#ifdef CONFIG_FS_DAX
#define S_DAX                (1 << 13) /* Direct Access, avoiding the page cache */
#else
#define S_DAX                0          /* Make all the DAX code disappear */
#endif
#define S_ENCRYPTED        (1 << 14) /* Encrypted file (using fs/crypto/) */
#define S_CASEFOLD        (1 << 15) /* Casefolded file */
#define S_VERITY        (1 << 16) /* Verity file (using fs/verity/) */
#define S_KERNEL_FILE        (1 << 17) /* File is in use by the kernel (eg. fs/cachefiles) */
#define S_ANON_INODE        (1 << 19) /* Inode is an anonymous inode */

/*
 * Note that nosuid etc flags are inode-specific: setting some file-system
 * flags just means all the inodes inherit those flags by default. It might be
 * possible to override it selectively if you really wanted to with some
 * ioctl() that is not currently implemented.
 *
 * Exception: SB_RDONLY is always applied to the entire file system.
 *
 * Unfortunately, it is possible to change a filesystems flags with it mounted
 * with files in use.  This means that all of the inodes will not have their
 * i_flags updated.  Hence, i_flags no longer inherit the superblock mount
 * flags, so these have to be checked separately. -- rmk@arm.uk.linux.org
 */
#define __IS_FLG(inode, flg)        ((inode)->i_sb->s_flags & (flg))

static inline bool sb_rdonly(const struct super_block *sb) { return sb->s_flags & SB_RDONLY; }
#define IS_RDONLY(inode)        sb_rdonly((inode)->i_sb)
#define IS_SYNC(inode)                (__IS_FLG(inode, SB_SYNCHRONOUS) || \
                                        ((inode)->i_flags & S_SYNC))
#define IS_DIRSYNC(inode)        (__IS_FLG(inode, SB_SYNCHRONOUS|SB_DIRSYNC) || \
                                        ((inode)->i_flags & (S_SYNC|S_DIRSYNC)))
#define IS_MANDLOCK(inode)        __IS_FLG(inode, SB_MANDLOCK)
#define IS_NOATIME(inode)        __IS_FLG(inode, SB_RDONLY|SB_NOATIME)
#define IS_I_VERSION(inode)        __IS_FLG(inode, SB_I_VERSION)

#define IS_NOQUOTA(inode)        ((inode)->i_flags & S_NOQUOTA)
#define IS_APPEND(inode)        ((inode)->i_flags & S_APPEND)
#define IS_IMMUTABLE(inode)        ((inode)->i_flags & S_IMMUTABLE)

#ifdef CONFIG_FS_POSIX_ACL
#define IS_POSIXACL(inode)        __IS_FLG(inode, SB_POSIXACL)
#else
#define IS_POSIXACL(inode)        0
#endif

#define IS_DEADDIR(inode)        ((inode)->i_flags & S_DEAD)
#define IS_NOCMTIME(inode)        ((inode)->i_flags & S_NOCMTIME)

#ifdef CONFIG_SWAP
#define IS_SWAPFILE(inode)        ((inode)->i_flags & S_SWAPFILE)
#else
#define IS_SWAPFILE(inode)        ((void)(inode), 0U)
#endif

#define IS_PRIVATE(inode)        ((inode)->i_flags & S_PRIVATE)
#define IS_IMA(inode)                ((inode)->i_flags & S_IMA)
#define IS_AUTOMOUNT(inode)        ((inode)->i_flags & S_AUTOMOUNT)
#define IS_NOSEC(inode)                ((inode)->i_flags & S_NOSEC)
#define IS_DAX(inode)                ((inode)->i_flags & S_DAX)
#define IS_ENCRYPTED(inode)        ((inode)->i_flags & S_ENCRYPTED)
#define IS_CASEFOLDED(inode)        ((inode)->i_flags & S_CASEFOLD)
#define IS_VERITY(inode)        ((inode)->i_flags & S_VERITY)

#define IS_WHITEOUT(inode)        (S_ISCHR(inode->i_mode) && \
                                 (inode)->i_rdev == WHITEOUT_DEV)
#define IS_ANON_FILE(inode)        ((inode)->i_flags & S_ANON_INODE)

static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap,
                                   struct inode *inode)
{
        return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
               !vfsgid_valid(i_gid_into_vfsgid(idmap, inode));
}

static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
{
        *kiocb = (struct kiocb) {
                .ki_filp = filp,
                .ki_flags = filp->f_iocb_flags,
                .ki_ioprio = get_current_ioprio(),
        };
}

static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src,
                               struct file *filp)
{
        *kiocb = (struct kiocb) {
                .ki_filp = filp,
                .ki_flags = kiocb_src->ki_flags,
                .ki_ioprio = kiocb_src->ki_ioprio,
                .ki_pos = kiocb_src->ki_pos,
        };
}

extern void __mark_inode_dirty(struct inode *, int);
static inline void mark_inode_dirty(struct inode *inode)
{
        __mark_inode_dirty(inode, I_DIRTY);
}

static inline void mark_inode_dirty_sync(struct inode *inode)
{
        __mark_inode_dirty(inode, I_DIRTY_SYNC);
}

static inline int icount_read(const struct inode *inode)
{
        return atomic_read(&inode->i_count);
}

/*
 * Returns true if the given inode itself only has dirty timestamps (its pages
 * may still be dirty) and isn't currently being allocated or freed.
 * Filesystems should call this if when writing an inode when lazytime is
 * enabled, they want to opportunistically write the timestamps of other inodes
 * located very nearby on-disk, e.g. in the same inode block.  This returns true
 * if the given inode is in need of such an opportunistic update.  Requires
 * i_lock, or at least later re-checking under i_lock.
 */
static inline bool inode_is_dirtytime_only(struct inode *inode)
{
        return (inode->i_state & (I_DIRTY_TIME | I_NEW |
                                  I_FREEING | I_WILL_FREE)) == I_DIRTY_TIME;
}

extern void inc_nlink(struct inode *inode);
extern void drop_nlink(struct inode *inode);
extern void clear_nlink(struct inode *inode);
extern void set_nlink(struct inode *inode, unsigned int nlink);

static inline void inode_inc_link_count(struct inode *inode)
{
        inc_nlink(inode);
        mark_inode_dirty(inode);
}

static inline void inode_dec_link_count(struct inode *inode)
{
        drop_nlink(inode);
        mark_inode_dirty(inode);
}

enum file_time_flags {
        S_ATIME = 1,
        S_MTIME = 2,
        S_CTIME = 4,
        S_VERSION = 8,
};

extern bool atime_needs_update(const struct path *, struct inode *);
extern void touch_atime(const struct path *);
int inode_update_time(struct inode *inode, int flags);

static inline void file_accessed(struct file *file)
{
        if (!(file->f_flags & O_NOATIME))
                touch_atime(&file->f_path);
}

extern int file_modified(struct file *file);
int kiocb_modified(struct kiocb *iocb);

int sync_inode_metadata(struct inode *inode, int wait);

struct file_system_type {
        const char *name;
        int fs_flags;
#define FS_REQUIRES_DEV                1 
#define FS_BINARY_MOUNTDATA        2
#define FS_HAS_SUBTYPE                4
#define FS_USERNS_MOUNT                8        /* Can be mounted by userns root */
#define FS_DISALLOW_NOTIFY_PERM        16        /* Disable fanotify permission events */
#define FS_ALLOW_IDMAP         32      /* FS has been updated to handle vfs idmappings. */
#define FS_MGTIME                64        /* FS uses multigrain timestamps */
#define FS_LBS                        128        /* FS supports LBS */
#define FS_RENAME_DOES_D_MOVE        32768        /* FS will handle d_move() during rename() internally. */
        int (*init_fs_context)(struct fs_context *);
        const struct fs_parameter_spec *parameters;
        struct dentry *(*mount) (struct file_system_type *, int,
                       const char *, void *);
        void (*kill_sb) (struct super_block *);
        struct module *owner;
        struct file_system_type * next;
        struct hlist_head fs_supers;

        struct lock_class_key s_lock_key;
        struct lock_class_key s_umount_key;
        struct lock_class_key s_vfs_rename_key;
        struct lock_class_key s_writers_key[SB_FREEZE_LEVELS];

        struct lock_class_key i_lock_key;
        struct lock_class_key i_mutex_key;
        struct lock_class_key invalidate_lock_key;
        struct lock_class_key i_mutex_dir_key;
};

#define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME)

/**
 * is_mgtime: is this inode using multigrain timestamps
 * @inode: inode to test for multigrain timestamps
 *
 * Return true if the inode uses multigrain timestamps, false otherwise.
 */
static inline bool is_mgtime(const struct inode *inode)
{
        return inode->i_opflags & IOP_MGTIME;
}

extern struct dentry *mount_subtree(struct vfsmount *mnt, const char *path);
void retire_super(struct super_block *sb);
void generic_shutdown_super(struct super_block *sb);
void kill_block_super(struct super_block *sb);
void kill_anon_super(struct super_block *sb);
void kill_litter_super(struct super_block *sb);
void deactivate_super(struct super_block *sb);
void deactivate_locked_super(struct super_block *sb);
int set_anon_super(struct super_block *s, void *data);
int set_anon_super_fc(struct super_block *s, struct fs_context *fc);
int get_anon_bdev(dev_t *);
void free_anon_bdev(dev_t);
struct super_block *sget_fc(struct fs_context *fc,
                            int (*test)(struct super_block *, struct fs_context *),
                            int (*set)(struct super_block *, struct fs_context *));
struct super_block *sget(struct file_system_type *type,
                        int (*test)(struct super_block *,void *),
                        int (*set)(struct super_block *,void *),
                        int flags, void *data);
struct super_block *sget_dev(struct fs_context *fc, dev_t dev);

/* Alas, no aliases. Too much hassle with bringing module.h everywhere */
#define fops_get(fops) ({                                                \
        const struct file_operations *_fops = (fops);                        \
        (((_fops) && try_module_get((_fops)->owner) ? (_fops) : NULL));        \
})

#define fops_put(fops) ({                                                \
        const struct file_operations *_fops = (fops);                        \
        if (_fops)                                                        \
                module_put((_fops)->owner);                                \
})

/*
 * This one is to be used *ONLY* from ->open() instances.
 * fops must be non-NULL, pinned down *and* module dependencies
 * should be sufficient to pin the caller down as well.
 */
#define replace_fops(f, fops) \
        do {        \
                struct file *__file = (f); \
                fops_put(__file->f_op); \
                BUG_ON(!(__file->f_op = (fops))); \
        } while(0)

extern int register_filesystem(struct file_system_type *);
extern int unregister_filesystem(struct file_system_type *);
extern int vfs_statfs(const struct path *, struct kstatfs *);
extern int user_statfs(const char __user *, struct kstatfs *);
extern int fd_statfs(int, struct kstatfs *);
int freeze_super(struct super_block *super, enum freeze_holder who,
                 const void *freeze_owner);
int thaw_super(struct super_block *super, enum freeze_holder who,
               const void *freeze_owner);
extern __printf(2, 3)
int super_setup_bdi_name(struct super_block *sb, char *fmt, ...);
extern int super_setup_bdi(struct super_block *sb);

static inline void super_set_uuid(struct super_block *sb, const u8 *uuid, unsigned len)
{
        if (WARN_ON(len > sizeof(sb->s_uuid)))
                len = sizeof(sb->s_uuid);
        sb->s_uuid_len = len;
        memcpy(&sb->s_uuid, uuid, len);
}

/* set sb sysfs name based on sb->s_bdev */
static inline void super_set_sysfs_name_bdev(struct super_block *sb)
{
        snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pg", sb->s_bdev);
}

/* set sb sysfs name based on sb->s_uuid */
static inline void super_set_sysfs_name_uuid(struct super_block *sb)
{
        WARN_ON(sb->s_uuid_len != sizeof(sb->s_uuid));
        snprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), "%pU", sb->s_uuid.b);
}

/* set sb sysfs name based on sb->s_id */
static inline void super_set_sysfs_name_id(struct super_block *sb)
{
        strscpy(sb->s_sysfs_name, sb->s_id, sizeof(sb->s_sysfs_name));
}

/* try to use something standard before you use this */
__printf(2, 3)
static inline void super_set_sysfs_name_generic(struct super_block *sb, const char *fmt, ...)
{
        va_list args;

        va_start(args, fmt);
        vsnprintf(sb->s_sysfs_name, sizeof(sb->s_sysfs_name), fmt, args);
        va_end(args);
}

extern int current_umask(void);

extern void ihold(struct inode * inode);
extern void iput(struct inode *);
int inode_update_timestamps(struct inode *inode, int flags);
int generic_update_time(struct inode *, int);

/* /sys/fs */
extern struct kobject *fs_kobj;

#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)

/* fs/open.c */
struct audit_names;
struct filename {
        const char                *name;        /* pointer to actual string */
        const __user char        *uptr;        /* original userland pointer */
        atomic_t                refcnt;
        struct audit_names        *aname;
        const char                iname[];
};
static_assert(offsetof(struct filename, iname) % sizeof(long) == 0);

static inline struct mnt_idmap *file_mnt_idmap(const struct file *file)
{
        return mnt_idmap(file->f_path.mnt);
}

/**
 * is_idmapped_mnt - check whether a mount is mapped
 * @mnt: the mount to check
 *
 * If @mnt has an non @nop_mnt_idmap attached to it then @mnt is mapped.
 *
 * Return: true if mount is mapped, false if not.
 */
static inline bool is_idmapped_mnt(const struct vfsmount *mnt)
{
        return mnt_idmap(mnt) != &nop_mnt_idmap;
}

int vfs_truncate(const struct path *, loff_t);
int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start,
                unsigned int time_attrs, struct file *filp);
extern int vfs_fallocate(struct file *file, int mode, loff_t offset,
                        loff_t len);
int do_sys_open(int dfd, const char __user *filename, int flags,
                umode_t mode);
extern struct file *file_open_name(struct filename *, int, umode_t);
extern struct file *filp_open(const char *, int, umode_t);
extern struct file *file_open_root(const struct path *,
                                   const char *, int, umode_t);
static inline struct file *file_open_root_mnt(struct vfsmount *mnt,
                                   const char *name, int flags, umode_t mode)
{
        return file_open_root(&(struct path){.mnt = mnt, .dentry = mnt->mnt_root},
                              name, flags, mode);
}
struct file *dentry_open(const struct path *path, int flags,
                         const struct cred *creds);
struct file *dentry_open_nonotify(const struct path *path, int flags,
                                  const struct cred *cred);
struct file *dentry_create(const struct path *path, int flags, umode_t mode,
                           const struct cred *cred);
const struct path *backing_file_user_path(const struct file *f);

/*
 * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file
 * stored in ->vm_file is a backing file whose f_inode is on the underlying
 * filesystem.  When the mapped file path and inode number are displayed to
 * user (e.g. via /proc/<pid>/maps), these helpers should be used to get the
 * path and inode number to display to the user, which is the path of the fd
 * that user has requested to map and the inode number that would be returned
 * by fstat() on that same fd.
 */
/* Get the path to display in /proc/<pid>/maps */
static inline const struct path *file_user_path(const struct file *f)
{
        if (unlikely(f->f_mode & FMODE_BACKING))
                return backing_file_user_path(f);
        return &f->f_path;
}
/* Get the inode whose inode number to display in /proc/<pid>/maps */
static inline const struct inode *file_user_inode(const struct file *f)
{
        if (unlikely(f->f_mode & FMODE_BACKING))
                return d_inode(backing_file_user_path(f)->dentry);
        return file_inode(f);
}

static inline struct file *file_clone_open(struct file *file)
{
        return dentry_open(&file->f_path, file->f_flags, file->f_cred);
}
extern int filp_close(struct file *, fl_owner_t id);

extern struct filename *getname_flags(const char __user *, int);
extern struct filename *getname_uflags(const char __user *, int);
static inline struct filename *getname(const char __user *name)
{
        return getname_flags(name, 0);
}
extern struct filename *getname_kernel(const char *);
extern struct filename *__getname_maybe_null(const char __user *);
static inline struct filename *getname_maybe_null(const char __user *name, int flags)
{
        if (!(flags & AT_EMPTY_PATH))
                return getname(name);

        if (!name)
                return NULL;
        return __getname_maybe_null(name);
}
extern void putname(struct filename *name);
DEFINE_FREE(putname, struct filename *, if (!IS_ERR_OR_NULL(_T)) putname(_T))

static inline struct filename *refname(struct filename *name)
{
        atomic_inc(&name->refcnt);
        return name;
}

extern int finish_open(struct file *file, struct dentry *dentry,
                        int (*open)(struct inode *, struct file *));
extern int finish_no_open(struct file *file, struct dentry *dentry);

/* Helper for the simple case when original dentry is used */
static inline int finish_open_simple(struct file *file, int error)
{
        if (error)
                return error;

        return finish_open(file, file->f_path.dentry, NULL);
}

/* fs/dcache.c */
extern void __init vfs_caches_init_early(void);
extern void __init vfs_caches_init(void);

extern struct kmem_cache *names_cachep;

#define __getname()                kmem_cache_alloc(names_cachep, GFP_KERNEL)
#define __putname(name)                kmem_cache_free(names_cachep, (void *)(name))

extern struct super_block *blockdev_superblock;
static inline bool sb_is_blkdev_sb(struct super_block *sb)
{
        return IS_ENABLED(CONFIG_BLOCK) && sb == blockdev_superblock;
}

void emergency_thaw_all(void);
extern int sync_filesystem(struct super_block *);
extern const struct file_operations def_blk_fops;
extern const struct file_operations def_chr_fops;

/* fs/char_dev.c */
#define CHRDEV_MAJOR_MAX 512
/* Marks the bottom of the first segment of free char majors */
#define CHRDEV_MAJOR_DYN_END 234
/* Marks the top and bottom of the second segment of free char majors */
#define CHRDEV_MAJOR_DYN_EXT_START 511
#define CHRDEV_MAJOR_DYN_EXT_END 384

extern int alloc_chrdev_region(dev_t *, unsigned, unsigned, const char *);
extern int register_chrdev_region(dev_t, unsigned, const char *);
extern int __register_chrdev(unsigned int major, unsigned int baseminor,
                             unsigned int count, const char *name,
                             const struct file_operations *fops);
extern void __unregister_chrdev(unsigned int major, unsigned int baseminor,
                                unsigned int count, const char *name);
extern void unregister_chrdev_region(dev_t, unsigned);
extern void chrdev_show(struct seq_file *,off_t);

static inline int register_chrdev(unsigned int major, const char *name,
                                  const struct file_operations *fops)
{
        return __register_chrdev(major, 0, 256, name, fops);
}

static inline void unregister_chrdev(unsigned int major, const char *name)
{
        __unregister_chrdev(major, 0, 256, name);
}

extern void init_special_inode(struct inode *, umode_t, dev_t);

/* Invalid inode operations -- fs/bad_inode.c */
extern void make_bad_inode(struct inode *);
extern bool is_bad_inode(struct inode *);

extern int __must_check file_fdatawait_range(struct file *file, loff_t lstart,
                                                loff_t lend);
extern int __must_check file_check_and_advance_wb_err(struct file *file);
extern int __must_check file_write_and_wait_range(struct file *file,
                                                loff_t start, loff_t end);
int filemap_fdatawrite_range_kick(struct address_space *mapping, loff_t start,
                loff_t end);

static inline int file_write_and_wait(struct file *file)
{
        return file_write_and_wait_range(file, 0, LLONG_MAX);
}

extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end,
                           int datasync);
extern int vfs_fsync(struct file *file, int datasync);

extern int sync_file_range(struct file *file, loff_t offset, loff_t nbytes,
                                unsigned int flags);

static inline bool iocb_is_dsync(const struct kiocb *iocb)
{
        return (iocb->ki_flags & IOCB_DSYNC) ||
                IS_SYNC(iocb->ki_filp->f_mapping->host);
}

/*
 * Sync the bytes written if this was a synchronous write.  Expect ki_pos
 * to already be updated for the write, and will return either the amount
 * of bytes passed in, or an error if syncing the file failed.
 */
static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count)
{
        if (iocb_is_dsync(iocb)) {
                int ret = vfs_fsync_range(iocb->ki_filp,
                                iocb->ki_pos - count, iocb->ki_pos - 1,
                                (iocb->ki_flags & IOCB_SYNC) ? 0 : 1);
                if (ret)
                        return ret;
        } else if (iocb->ki_flags & IOCB_DONTCACHE) {
                struct address_space *mapping = iocb->ki_filp->f_mapping;

                filemap_fdatawrite_range_kick(mapping, iocb->ki_pos - count,
                                              iocb->ki_pos - 1);
        }

        return count;
}

extern void emergency_sync(void);
extern void emergency_remount(void);

#ifdef CONFIG_BLOCK
extern int bmap(struct inode *inode, sector_t *block);
#else
static inline int bmap(struct inode *inode,  sector_t *block)
{
        return -EINVAL;
}
#endif

int notify_change(struct mnt_idmap *, struct dentry *,
                  struct iattr *, struct inode **);
int inode_permission(struct mnt_idmap *, struct inode *, int);
int generic_permission(struct mnt_idmap *, struct inode *, int);
static inline int file_permission(struct file *file, int mask)
{
        return inode_permission(file_mnt_idmap(file),
                                file_inode(file), mask);
}
static inline int path_permission(const struct path *path, int mask)
{
        return inode_permission(mnt_idmap(path->mnt),
                                d_inode(path->dentry), mask);
}
int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
                   struct inode *inode);

static inline bool execute_ok(struct inode *inode)
{
        return (inode->i_mode & S_IXUGO) || S_ISDIR(inode->i_mode);
}

static inline bool inode_wrong_type(const struct inode *inode, umode_t mode)
{
        return (inode->i_mode ^ mode) & S_IFMT;
}

/**
 * file_start_write - get write access to a superblock for regular file io
 * @file: the file we want to write to
 *
 * This is a variant of sb_start_write() which is a noop on non-regualr file.
 * Should be matched with a call to file_end_write().
 */
static inline void file_start_write(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return;
        sb_start_write(file_inode(file)->i_sb);
}

static inline bool file_start_write_trylock(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return true;
        return sb_start_write_trylock(file_inode(file)->i_sb);
}

/**
 * file_end_write - drop write access to a superblock of a regular file
 * @file: the file we wrote to
 *
 * Should be matched with a call to file_start_write().
 */
static inline void file_end_write(struct file *file)
{
        if (!S_ISREG(file_inode(file)->i_mode))
                return;
        sb_end_write(file_inode(file)->i_sb);
}

/**
 * kiocb_start_write - get write access to a superblock for async file io
 * @iocb: the io context we want to submit the write with
 *
 * This is a variant of sb_start_write() for async io submission.
 * Should be matched with a call to kiocb_end_write().
 */
static inline void kiocb_start_write(struct kiocb *iocb)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        sb_start_write(inode->i_sb);
        /*
         * Fool lockdep by telling it the lock got released so that it
         * doesn't complain about the held lock when we return to userspace.
         */
        __sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
}

/**
 * kiocb_end_write - drop write access to a superblock after async file io
 * @iocb: the io context we sumbitted the write with
 *
 * Should be matched with a call to kiocb_start_write().
 */
static inline void kiocb_end_write(struct kiocb *iocb)
{
        struct inode *inode = file_inode(iocb->ki_filp);

        /*
         * Tell lockdep we inherited freeze protection from submission thread.
         */
        __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
        sb_end_write(inode->i_sb);
}

/*
 * This is used for regular files where some users -- especially the
 * currently executed binary in a process, previously handled via
 * VM_DENYWRITE -- cannot handle concurrent write (and maybe mmap
 * read-write shared) accesses.
 *
 * get_write_access() gets write permission for a file.
 * put_write_access() releases this write permission.
 * deny_write_access() denies write access to a file.
 * allow_write_access() re-enables write access to a file.
 *
 * The i_writecount field of an inode can have the following values:
 * 0: no write access, no denied write access
 * < 0: (-i_writecount) users that denied write access to the file.
 * > 0: (i_writecount) users that have write access to the file.
 *
 * Normally we operate on that counter with atomic_{inc,dec} and it's safe
 * except for the cases where we don't hold i_writecount yet. Then we need to
 * use {get,deny}_write_access() - these functions check the sign and refuse
 * to do the change if sign is wrong.
 */
static inline int get_write_access(struct inode *inode)
{
        return atomic_inc_unless_negative(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline int deny_write_access(struct file *file)
{
        struct inode *inode = file_inode(file);
        return atomic_dec_unless_positive(&inode->i_writecount) ? 0 : -ETXTBSY;
}
static inline void put_write_access(struct inode * inode)
{
        atomic_dec(&inode->i_writecount);
}
static inline void allow_write_access(struct file *file)
{
        if (file)
                atomic_inc(&file_inode(file)->i_writecount);
}

/*
 * Do not prevent write to executable file when watched by pre-content events.
 *
 * Note that FMODE_FSNOTIFY_HSM mode is set depending on pre-content watches at
 * the time of file open and remains constant for entire lifetime of the file,
 * so if pre-content watches are added post execution or removed before the end
 * of the execution, it will not cause i_writecount reference leak.
 */
static inline int exe_file_deny_write_access(struct file *exe_file)
{
        if (unlikely(FMODE_FSNOTIFY_HSM(exe_file->f_mode)))
                return 0;
        return deny_write_access(exe_file);
}
static inline void exe_file_allow_write_access(struct file *exe_file)
{
        if (unlikely(!exe_file || FMODE_FSNOTIFY_HSM(exe_file->f_mode)))
                return;
        allow_write_access(exe_file);
}

static inline void file_set_fsnotify_mode(struct file *file, fmode_t mode)
{
        file->f_mode &= ~FMODE_FSNOTIFY_MASK;
        file->f_mode |= mode;
}

static inline bool inode_is_open_for_write(const struct inode *inode)
{
        return atomic_read(&inode->i_writecount) > 0;
}

#if defined(CONFIG_IMA) || defined(CONFIG_FILE_LOCKING)
static inline void i_readcount_dec(struct inode *inode)
{
        BUG_ON(atomic_dec_return(&inode->i_readcount) < 0);
}
static inline void i_readcount_inc(struct inode *inode)
{
        atomic_inc(&inode->i_readcount);
}
#else
static inline void i_readcount_dec(struct inode *inode)
{
        return;
}
static inline void i_readcount_inc(struct inode *inode)
{
        return;
}
#endif
extern int do_pipe_flags(int *, int);

extern ssize_t kernel_read(struct file *, void *, size_t, loff_t *);
ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos);
extern ssize_t kernel_write(struct file *, const void *, size_t, loff_t *);
extern ssize_t __kernel_write(struct file *, const void *, size_t, loff_t *);
extern struct file * open_exec(const char *);
 
/* fs/dcache.c -- generic fs support functions */
extern bool is_subdir(struct dentry *, struct dentry *);
extern bool path_is_under(const struct path *, const struct path *);

extern char *file_path(struct file *, char *, int);

/**
 * is_dot_dotdot - returns true only if @name is "." or ".."
 * @name: file name to check
 * @len: length of file name, in bytes
 */
static inline bool is_dot_dotdot(const char *name, size_t len)
{
        return len && unlikely(name[0] == '.') &&
                (len == 1 || (len == 2 && name[1] == '.'));
}

/**
 * name_contains_dotdot - check if a file name contains ".." path components
 * @name: File path string to check
 * Search for ".." surrounded by either '/' or start/end of string.
 */
static inline bool name_contains_dotdot(const char *name)
{
        size_t name_len;

        name_len = strlen(name);
        return strcmp(name, "..") == 0 ||
               strncmp(name, "../", 3) == 0 ||
               strstr(name, "/../") != NULL ||
               (name_len >= 3 && strcmp(name + name_len - 3, "/..") == 0);
}

#include <linux/err.h>

/* needed for stackable file system support */
extern loff_t default_llseek(struct file *file, loff_t offset, int whence);

extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence);

extern int inode_init_always_gfp(struct super_block *, struct inode *, gfp_t);
static inline int inode_init_always(struct super_block *sb, struct inode *inode)
{
        return inode_init_always_gfp(sb, inode, GFP_NOFS);
}

extern void inode_init_once(struct inode *);
extern void address_space_init_once(struct address_space *mapping);
extern struct inode * igrab(struct inode *);
extern ino_t iunique(struct super_block *, ino_t);
extern int inode_needs_sync(struct inode *inode);
extern int inode_just_drop(struct inode *inode);
static inline int inode_generic_drop(struct inode *inode)
{
        return !inode->i_nlink || inode_unhashed(inode);
}
extern void d_mark_dontcache(struct inode *inode);

extern struct inode *ilookup5_nowait(struct super_block *sb,
                unsigned long hashval, int (*test)(struct inode *, void *),
                void *data);
extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
                int (*test)(struct inode *, void *), void *data);
extern struct inode *ilookup(struct super_block *sb, unsigned long ino);

extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
                int (*test)(struct inode *, void *),
                int (*set)(struct inode *, void *),
                void *data);
struct inode *iget5_locked(struct super_block *, unsigned long,
                           int (*test)(struct inode *, void *),
                           int (*set)(struct inode *, void *), void *);
struct inode *iget5_locked_rcu(struct super_block *, unsigned long,
                               int (*test)(struct inode *, void *),
                               int (*set)(struct inode *, void *), void *);
extern struct inode * iget_locked(struct super_block *, unsigned long);
extern struct inode *find_inode_nowait(struct super_block *,
                                       unsigned long,
                                       int (*match)(struct inode *,
                                                    unsigned long, void *),
                                       void *data);
extern struct inode *find_inode_rcu(struct super_block *, unsigned long,
                                    int (*)(struct inode *, void *), void *);
extern struct inode *find_inode_by_ino_rcu(struct super_block *, unsigned long);
extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *);
extern int insert_inode_locked(struct inode *);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void lockdep_annotate_inode_mutex_key(struct inode *inode);
#else
static inline void lockdep_annotate_inode_mutex_key(struct inode *inode) { };
#endif
extern void unlock_new_inode(struct inode *);
extern void discard_new_inode(struct inode *);
extern unsigned int get_next_ino(void);
extern void evict_inodes(struct super_block *sb);
void dump_mapping(const struct address_space *);

/*
 * Userspace may rely on the inode number being non-zero. For example, glibc
 * simply ignores files with zero i_ino in unlink() and other places.
 *
 * As an additional complication, if userspace was compiled with
 * _FILE_OFFSET_BITS=32 on a 64-bit kernel we'll only end up reading out the
 * lower 32 bits, so we need to check that those aren't zero explicitly. With
 * _FILE_OFFSET_BITS=64, this may cause some harmless false-negatives, but
 * better safe than sorry.
 */
static inline bool is_zero_ino(ino_t ino)
{
        return (u32)ino == 0;
}

/*
 * inode->i_lock must be held
 */
static inline void __iget(struct inode *inode)
{
        atomic_inc(&inode->i_count);
}

extern void iget_failed(struct inode *);
extern void clear_inode(struct inode *);
extern void __destroy_inode(struct inode *);
struct inode *alloc_inode(struct super_block *sb);
static inline struct inode *new_inode_pseudo(struct super_block *sb)
{
        return alloc_inode(sb);
}
extern struct inode *new_inode(struct super_block *sb);
extern void free_inode_nonrcu(struct inode *inode);
extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *);
extern int file_remove_privs(struct file *);
int setattr_should_drop_sgid(struct mnt_idmap *idmap,
                             const struct inode *inode);

/*
 * This must be used for allocating filesystems specific inodes to set
 * up the inode reclaim context correctly.
 */
#define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp)

extern void __insert_inode_hash(struct inode *, unsigned long hashval);
static inline void insert_inode_hash(struct inode *inode)
{
        __insert_inode_hash(inode, inode->i_ino);
}

extern void __remove_inode_hash(struct inode *);
static inline void remove_inode_hash(struct inode *inode)
{
        if (!inode_unhashed(inode) && !hlist_fake(&inode->i_hash))
                __remove_inode_hash(inode);
}

extern void inode_sb_list_add(struct inode *inode);
extern void inode_add_lru(struct inode *inode);

extern int sb_set_blocksize(struct super_block *, int);
extern int sb_min_blocksize(struct super_block *, int);

int generic_file_mmap(struct file *, struct vm_area_struct *);
int generic_file_mmap_prepare(struct vm_area_desc *desc);
int generic_file_readonly_mmap(struct file *, struct vm_area_struct *);
int generic_file_readonly_mmap_prepare(struct vm_area_desc *desc);
extern ssize_t generic_write_checks(struct kiocb *, struct iov_iter *);
int generic_write_checks_count(struct kiocb *iocb, loff_t *count);
extern int generic_write_check_limits(struct file *file, loff_t pos,
                loff_t *count);
extern int generic_file_rw_checks(struct file *file_in, struct file *file_out);
ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *to,
                ssize_t already_read);
extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *);
extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *);
extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *);
ssize_t generic_perform_write(struct kiocb *, struct iov_iter *);
ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter,
                ssize_t direct_written, ssize_t buffered_written);

ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
                rwf_t flags);
ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
                rwf_t flags);
ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
                           struct iov_iter *iter);
ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
                            struct iov_iter *iter);

/* fs/splice.c */
ssize_t filemap_splice_read(struct file *in, loff_t *ppos,
                            struct pipe_inode_info *pipe,
                            size_t len, unsigned int flags);
ssize_t copy_splice_read(struct file *in, loff_t *ppos,
                         struct pipe_inode_info *pipe,
                         size_t len, unsigned int flags);
extern ssize_t iter_file_splice_write(struct pipe_inode_info *,
                struct file *, loff_t *, size_t, unsigned int);


extern void
file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
extern loff_t noop_llseek(struct file *file, loff_t offset, int whence);
extern loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize);
extern loff_t generic_file_llseek(struct file *file, loff_t offset, int whence);
extern loff_t generic_file_llseek_size(struct file *file, loff_t offset,
                int whence, loff_t maxsize, loff_t eof);
loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence,
                             u64 *cookie);
extern loff_t fixed_size_llseek(struct file *file, loff_t offset,
                int whence, loff_t size);
extern loff_t no_seek_end_llseek_size(struct file *, loff_t, int, loff_t);
extern loff_t no_seek_end_llseek(struct file *, loff_t, int);
int rw_verify_area(int, struct file *, const loff_t *, size_t);
extern int generic_file_open(struct inode * inode, struct file * filp);
extern int nonseekable_open(struct inode * inode, struct file * filp);
extern int stream_open(struct inode * inode, struct file * filp);

#ifdef CONFIG_BLOCK
typedef void (dio_submit_t)(struct bio *bio, struct inode *inode,
                            loff_t file_offset);

enum {
        /* need locking between buffered and direct access */
        DIO_LOCKING        = 0x01,

        /* filesystem does not support filling holes */
        DIO_SKIP_HOLES        = 0x02,
};

ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
                             struct block_device *bdev, struct iov_iter *iter,
                             get_block_t get_block,
                             dio_iodone_t end_io,
                             int flags);

static inline ssize_t blockdev_direct_IO(struct kiocb *iocb,
                                         struct inode *inode,
                                         struct iov_iter *iter,
                                         get_block_t get_block)
{
        return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
                        get_block, NULL, DIO_LOCKING | DIO_SKIP_HOLES);
}
#endif

bool inode_dio_finished(const struct inode *inode);
void inode_dio_wait(struct inode *inode);
void inode_dio_wait_interruptible(struct inode *inode);

/**
 * inode_dio_begin - signal start of a direct I/O requests
 * @inode: inode the direct I/O happens on
 *
 * This is called once we've finished processing a direct I/O request,
 * and is used to wake up callers waiting for direct I/O to be quiesced.
 */
static inline void inode_dio_begin(struct inode *inode)
{
        atomic_inc(&inode->i_dio_count);
}

/**
 * inode_dio_end - signal finish of a direct I/O requests
 * @inode: inode the direct I/O happens on
 *
 * This is called once we've finished processing a direct I/O request,
 * and is used to wake up callers waiting for direct I/O to be quiesced.
 */
static inline void inode_dio_end(struct inode *inode)
{
        if (atomic_dec_and_test(&inode->i_dio_count))
                wake_up_var(&inode->i_dio_count);
}

extern void inode_set_flags(struct inode *inode, unsigned int flags,
                            unsigned int mask);

extern const struct file_operations generic_ro_fops;

#define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))

extern int readlink_copy(char __user *, int, const char *, int);
extern int page_readlink(struct dentry *, char __user *, int);
extern const char *page_get_link_raw(struct dentry *, struct inode *,
                                     struct delayed_call *);
extern const char *page_get_link(struct dentry *, struct inode *,
                                 struct delayed_call *);
extern void page_put_link(void *);
extern int page_symlink(struct inode *inode, const char *symname, int len);
extern const struct inode_operations page_symlink_inode_operations;
extern void kfree_link(void *);
void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode);
void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *);
void generic_fill_statx_attr(struct inode *inode, struct kstat *stat);
void generic_fill_statx_atomic_writes(struct kstat *stat,
                                      unsigned int unit_min,
                                      unsigned int unit_max,
                                      unsigned int unit_max_opt);
extern int vfs_getattr_nosec(const struct path *, struct kstat *, u32, unsigned int);
extern int vfs_getattr(const struct path *, struct kstat *, u32, unsigned int);
void __inode_add_bytes(struct inode *inode, loff_t bytes);
void inode_add_bytes(struct inode *inode, loff_t bytes);
void __inode_sub_bytes(struct inode *inode, loff_t bytes);
void inode_sub_bytes(struct inode *inode, loff_t bytes);
static inline loff_t __inode_get_bytes(struct inode *inode)
{
        return (((loff_t)inode->i_blocks) << 9) + inode->i_bytes;
}
loff_t inode_get_bytes(struct inode *inode);
void inode_set_bytes(struct inode *inode, loff_t bytes);
const char *simple_get_link(struct dentry *, struct inode *,
                            struct delayed_call *);
extern const struct inode_operations simple_symlink_inode_operations;

extern int iterate_dir(struct file *, struct dir_context *);

int vfs_fstatat(int dfd, const char __user *filename, struct kstat *stat,
                int flags);
int vfs_fstat(int fd, struct kstat *stat);

static inline int vfs_stat(const char __user *filename, struct kstat *stat)
{
        return vfs_fstatat(AT_FDCWD, filename, stat, 0);
}
static inline int vfs_lstat(const char __user *name, struct kstat *stat)
{
        return vfs_fstatat(AT_FDCWD, name, stat, AT_SYMLINK_NOFOLLOW);
}

extern const char *vfs_get_link(struct dentry *, struct delayed_call *);
extern int vfs_readlink(struct dentry *, char __user *, int);

extern struct file_system_type *get_filesystem(struct file_system_type *fs);
extern void put_filesystem(struct file_system_type *fs);
extern struct file_system_type *get_fs_type(const char *name);
extern void drop_super(struct super_block *sb);
extern void drop_super_exclusive(struct super_block *sb);
extern void iterate_supers(void (*f)(struct super_block *, void *), void *arg);
extern void iterate_supers_type(struct file_system_type *,
                                void (*)(struct super_block *, void *), void *);
void filesystems_freeze(void);
void filesystems_thaw(void);

extern int dcache_dir_open(struct inode *, struct file *);
extern int dcache_dir_close(struct inode *, struct file *);
extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
extern int dcache_readdir(struct file *, struct dir_context *);
extern int simple_setattr(struct mnt_idmap *, struct dentry *,
                          struct iattr *);
extern int simple_getattr(struct mnt_idmap *, const struct path *,
                          struct kstat *, u32, unsigned int);
extern int simple_statfs(struct dentry *, struct kstatfs *);
extern int simple_open(struct inode *inode, struct file *file);
extern int simple_link(struct dentry *, struct inode *, struct dentry *);
extern int simple_unlink(struct inode *, struct dentry *);
extern int simple_rmdir(struct inode *, struct dentry *);
void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
                             struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
                                  struct inode *new_dir, struct dentry *new_dentry);
extern int simple_rename(struct mnt_idmap *, struct inode *,
                         struct dentry *, struct inode *, struct dentry *,
                         unsigned int);
extern void simple_recursive_removal(struct dentry *,
                              void (*callback)(struct dentry *));
extern void locked_recursive_removal(struct dentry *,
                              void (*callback)(struct dentry *));
extern int noop_fsync(struct file *, loff_t, loff_t, int);
extern ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
extern int simple_empty(struct dentry *);
extern int simple_write_begin(const struct kiocb *iocb,
                              struct address_space *mapping,
                              loff_t pos, unsigned len,
                              struct folio **foliop, void **fsdata);
extern const struct address_space_operations ram_aops;
extern int always_delete_dentry(const struct dentry *);
extern struct inode *alloc_anon_inode(struct super_block *);
struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name,
                                           const struct inode *context_inode);
extern int simple_nosetlease(struct file *, int, struct file_lease **, void **);

extern struct dentry *simple_lookup(struct inode *, struct dentry *, unsigned int flags);
extern ssize_t generic_read_dir(struct file *, char __user *, size_t, loff_t *);
extern const struct file_operations simple_dir_operations;
extern const struct inode_operations simple_dir_inode_operations;
extern void make_empty_dir_inode(struct inode *inode);
extern bool is_empty_dir_inode(struct inode *inode);
struct tree_descr { const char *name; const struct file_operations *ops; int mode; };
struct dentry *d_alloc_name(struct dentry *, const char *);
extern int simple_fill_super(struct super_block *, unsigned long,
                             const struct tree_descr *);
extern int simple_pin_fs(struct file_system_type *, struct vfsmount **mount, int *count);
extern void simple_release_fs(struct vfsmount **mount, int *count);
struct dentry *simple_start_creating(struct dentry *, const char *);

extern ssize_t simple_read_from_buffer(void __user *to, size_t count,
                        loff_t *ppos, const void *from, size_t available);
extern ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos,
                const void __user *from, size_t count);

struct offset_ctx {
        struct maple_tree        mt;
        unsigned long                next_offset;
};

void simple_offset_init(struct offset_ctx *octx);
int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry);
void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry);
int simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry,
                         struct inode *new_dir, struct dentry *new_dentry);
int simple_offset_rename_exchange(struct inode *old_dir,
                                  struct dentry *old_dentry,
                                  struct inode *new_dir,
                                  struct dentry *new_dentry);
void simple_offset_destroy(struct offset_ctx *octx);

extern const struct file_operations simple_offset_dir_operations;

extern int __generic_file_fsync(struct file *, loff_t, loff_t, int);
extern int generic_file_fsync(struct file *, loff_t, loff_t, int);

extern int generic_check_addressable(unsigned, u64);

extern void generic_set_sb_d_ops(struct super_block *sb);
extern int generic_ci_match(const struct inode *parent,
                            const struct qstr *name,
                            const struct qstr *folded_name,
                            const u8 *de_name, u32 de_name_len);

#if IS_ENABLED(CONFIG_UNICODE)
int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str);
int generic_ci_d_compare(const struct dentry *dentry, unsigned int len,
                         const char *str, const struct qstr *name);

/**
 * generic_ci_validate_strict_name - Check if a given name is suitable
 * for a directory
 *
 * This functions checks if the proposed filename is valid for the
 * parent directory. That means that only valid UTF-8 filenames will be
 * accepted for casefold directories from filesystems created with the
 * strict encoding flag.  That also means that any name will be
 * accepted for directories that doesn't have casefold enabled, or
 * aren't being strict with the encoding.
 *
 * @dir: inode of the directory where the new file will be created
 * @name: name of the new file
 *
 * Return:
 * * True: if the filename is suitable for this directory. It can be
 *   true if a given name is not suitable for a strict encoding
 *   directory, but the directory being used isn't strict
 * * False if the filename isn't suitable for this directory. This only
 *   happens when a directory is casefolded and the filesystem is strict
 *   about its encoding.
 */
static inline bool generic_ci_validate_strict_name(struct inode *dir,
                                                   const struct qstr *name)
{
        if (!IS_CASEFOLDED(dir) || !sb_has_strict_encoding(dir->i_sb))
                return true;

        /*
         * A casefold dir must have a encoding set, unless the filesystem
         * is corrupted
         */
        if (WARN_ON_ONCE(!dir->i_sb->s_encoding))
                return true;

        return !utf8_validate(dir->i_sb->s_encoding, name);
}
#else
static inline bool generic_ci_validate_strict_name(struct inode *dir,
                                                   const struct qstr *name)
{
        return true;
}
#endif

static inline struct unicode_map *sb_encoding(const struct super_block *sb)
{
#if IS_ENABLED(CONFIG_UNICODE)
        return sb->s_encoding;
#else
        return NULL;
#endif
}

static inline bool sb_has_encoding(const struct super_block *sb)
{
        return !!sb_encoding(sb);
}

/*
 * Compare if two super blocks have the same encoding and flags
 */
static inline bool sb_same_encoding(const struct super_block *sb1,
                                    const struct super_block *sb2)
{
#if IS_ENABLED(CONFIG_UNICODE)
        if (sb1->s_encoding == sb2->s_encoding)
                return true;

        return (sb1->s_encoding && sb2->s_encoding &&
               (sb1->s_encoding->version == sb2->s_encoding->version) &&
               (sb1->s_encoding_flags == sb2->s_encoding_flags));
#else
        return true;
#endif
}

int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
                unsigned int ia_valid);
int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *);
extern int inode_newsize_ok(const struct inode *, loff_t offset);
void setattr_copy(struct mnt_idmap *, struct inode *inode,
                  const struct iattr *attr);

extern int file_update_time(struct file *file);

static inline bool file_is_dax(const struct file *file)
{
        return file && IS_DAX(file->f_mapping->host);
}

static inline bool vma_is_dax(const struct vm_area_struct *vma)
{
        return file_is_dax(vma->vm_file);
}

static inline bool vma_is_fsdax(struct vm_area_struct *vma)
{
        struct inode *inode;

        if (!IS_ENABLED(CONFIG_FS_DAX) || !vma->vm_file)
                return false;
        if (!vma_is_dax(vma))
                return false;
        inode = file_inode(vma->vm_file);
        if (S_ISCHR(inode->i_mode))
                return false; /* device-dax */
        return true;
}

static inline int iocb_flags(struct file *file)
{
        int res = 0;
        if (file->f_flags & O_APPEND)
                res |= IOCB_APPEND;
        if (file->f_flags & O_DIRECT)
                res |= IOCB_DIRECT;
        if (file->f_flags & O_DSYNC)
                res |= IOCB_DSYNC;
        if (file->f_flags & __O_SYNC)
                res |= IOCB_SYNC;
        return res;
}

static inline int kiocb_set_rw_flags(struct kiocb *ki, rwf_t flags,
                                     int rw_type)
{
        int kiocb_flags = 0;

        /* make sure there's no overlap between RWF and private IOCB flags */
        BUILD_BUG_ON((__force int) RWF_SUPPORTED & IOCB_EVENTFD);

        if (!flags)
                return 0;
        if (unlikely(flags & ~RWF_SUPPORTED))
                return -EOPNOTSUPP;
        if (unlikely((flags & RWF_APPEND) && (flags & RWF_NOAPPEND)))
                return -EINVAL;

        if (flags & RWF_NOWAIT) {
                if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
                        return -EOPNOTSUPP;
        }
        if (flags & RWF_ATOMIC) {
                if (rw_type != WRITE)
                        return -EOPNOTSUPP;
                if (!(ki->ki_filp->f_mode & FMODE_CAN_ATOMIC_WRITE))
                        return -EOPNOTSUPP;
        }
        if (flags & RWF_DONTCACHE) {
                /* file system must support it */
                if (!(ki->ki_filp->f_op->fop_flags & FOP_DONTCACHE))
                        return -EOPNOTSUPP;
                /* DAX mappings not supported */
                if (IS_DAX(ki->ki_filp->f_mapping->host))
                        return -EOPNOTSUPP;
        }
        kiocb_flags |= (__force int) (flags & RWF_SUPPORTED);
        if (flags & RWF_SYNC)
                kiocb_flags |= IOCB_DSYNC;

        if ((flags & RWF_NOAPPEND) && (ki->ki_flags & IOCB_APPEND)) {
                if (IS_APPEND(file_inode(ki->ki_filp)))
                        return -EPERM;
                ki->ki_flags &= ~IOCB_APPEND;
        }

        ki->ki_flags |= kiocb_flags;
        return 0;
}

/* Transaction based IO helpers */

/*
 * An argresp is stored in an allocated page and holds the
 * size of the argument or response, along with its content
 */
struct simple_transaction_argresp {
        ssize_t size;
        char data[];
};

#define SIMPLE_TRANSACTION_LIMIT (PAGE_SIZE - sizeof(struct simple_transaction_argresp))

char *simple_transaction_get(struct file *file, const char __user *buf,
                                size_t size);
ssize_t simple_transaction_read(struct file *file, char __user *buf,
                                size_t size, loff_t *pos);
int simple_transaction_release(struct inode *inode, struct file *file);

void simple_transaction_set(struct file *file, size_t n);

/*
 * simple attribute files
 *
 * These attributes behave similar to those in sysfs:
 *
 * Writing to an attribute immediately sets a value, an open file can be
 * written to multiple times.
 *
 * Reading from an attribute creates a buffer from the value that might get
 * read with multiple read calls. When the attribute has been read
 * completely, no further read calls are possible until the file is opened
 * again.
 *
 * All attributes contain a text representation of a numeric value
 * that are accessed with the get() and set() functions.
 */
#define DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, __is_signed)        \
static int __fops ## _open(struct inode *inode, struct file *file)        \
{                                                                        \
        __simple_attr_check_format(__fmt, 0ull);                        \
        return simple_attr_open(inode, file, __get, __set, __fmt);        \
}                                                                        \
static const struct file_operations __fops = {                                \
        .owner         = THIS_MODULE,                                                \
        .open         = __fops ## _open,                                        \
        .release = simple_attr_release,                                        \
        .read         = simple_attr_read,                                        \
        .write         = (__is_signed) ? simple_attr_write_signed : simple_attr_write,        \
        .llseek         = generic_file_llseek,                                        \
}

#define DEFINE_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt)                \
        DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, false)

#define DEFINE_SIMPLE_ATTRIBUTE_SIGNED(__fops, __get, __set, __fmt)        \
        DEFINE_SIMPLE_ATTRIBUTE_XSIGNED(__fops, __get, __set, __fmt, true)

static inline __printf(1, 2)
void __simple_attr_check_format(const char *fmt, ...)
{
        /* don't do anything, just let the compiler check the arguments; */
}

int simple_attr_open(struct inode *inode, struct file *file,
                     int (*get)(void *, u64 *), int (*set)(void *, u64),
                     const char *fmt);
int simple_attr_release(struct inode *inode, struct file *file);
ssize_t simple_attr_read(struct file *file, char __user *buf,
                         size_t len, loff_t *ppos);
ssize_t simple_attr_write(struct file *file, const char __user *buf,
                          size_t len, loff_t *ppos);
ssize_t simple_attr_write_signed(struct file *file, const char __user *buf,
                                 size_t len, loff_t *ppos);

struct ctl_table;
int __init list_bdev_fs_names(char *buf, size_t size);

#define __FMODE_EXEC                ((__force int) FMODE_EXEC)

#define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])
#define OPEN_FMODE(flag) ((__force fmode_t)((flag + 1) & O_ACCMODE))

static inline bool is_sxid(umode_t mode)
{
        return mode & (S_ISUID | S_ISGID);
}

static inline int check_sticky(struct mnt_idmap *idmap,
                               struct inode *dir, struct inode *inode)
{
        if (!(dir->i_mode & S_ISVTX))
                return 0;

        return __check_sticky(idmap, dir, inode);
}

static inline void inode_has_no_xattr(struct inode *inode)
{
        if (!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & SB_NOSEC))
                inode->i_flags |= S_NOSEC;
}

static inline bool is_root_inode(struct inode *inode)
{
        return inode == inode->i_sb->s_root->d_inode;
}

static inline bool dir_emit(struct dir_context *ctx,
                            const char *name, int namelen,
                            u64 ino, unsigned type)
{
        return ctx->actor(ctx, name, namelen, ctx->pos, ino, type);
}
static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx)
{
        return ctx->actor(ctx, ".", 1, ctx->pos,
                          file->f_path.dentry->d_inode->i_ino, DT_DIR);
}
static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx)
{
        return ctx->actor(ctx, "..", 2, ctx->pos,
                          d_parent_ino(file->f_path.dentry), DT_DIR);
}
static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx)
{
        if (ctx->pos == 0) {
                if (!dir_emit_dot(file, ctx))
                        return false;
                ctx->pos = 1;
        }
        if (ctx->pos == 1) {
                if (!dir_emit_dotdot(file, ctx))
                        return false;
                ctx->pos = 2;
        }
        return true;
}
static inline bool dir_relax(struct inode *inode)
{
        inode_unlock(inode);
        inode_lock(inode);
        return !IS_DEADDIR(inode);
}

static inline bool dir_relax_shared(struct inode *inode)
{
        inode_unlock_shared(inode);
        inode_lock_shared(inode);
        return !IS_DEADDIR(inode);
}

extern bool path_noexec(const struct path *path);
extern void inode_nohighmem(struct inode *inode);

/* mm/fadvise.c */
extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len,
                       int advice);
extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
                           int advice);

static inline bool vfs_empty_path(int dfd, const char __user *path)
{
        char c;

        if (dfd < 0)
                return false;

        /* We now allow NULL to be used for empty path. */
        if (!path)
                return true;

        if (unlikely(get_user(c, path)))
                return false;

        return !c;
}

int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter);

static inline bool extensible_ioctl_valid(unsigned int cmd_a,
                                          unsigned int cmd_b, size_t min_size)
{
        if (_IOC_DIR(cmd_a) != _IOC_DIR(cmd_b))
                return false;
        if (_IOC_TYPE(cmd_a) != _IOC_TYPE(cmd_b))
                return false;
        if (_IOC_NR(cmd_a) != _IOC_NR(cmd_b))
                return false;
        if (_IOC_SIZE(cmd_a) < min_size)
                return false;
        return true;
}

#endif /* _LINUX_FS_H */





















  436 














    7 






  416 





    1 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the IP protocol.
 *
 * Version:        @(#)ip.h        1.0.2        04/28/93
 *
 * Authors:        Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 */
#ifndef _LINUX_IP_H
#define _LINUX_IP_H

#include <linux/skbuff.h>
#include <uapi/linux/ip.h>

static inline struct iphdr *ip_hdr(const struct sk_buff *skb)
{
        return (struct iphdr *)skb_network_header(skb);
}

static inline struct iphdr *inner_ip_hdr(const struct sk_buff *skb)
{
        return (struct iphdr *)skb_inner_network_header(skb);
}

static inline struct iphdr *ipip_hdr(const struct sk_buff *skb)
{
        return (struct iphdr *)skb_transport_header(skb);
}

static inline unsigned int ip_transport_len(const struct sk_buff *skb)
{
        return ntohs(ip_hdr(skb)->tot_len) - skb_network_header_len(skb);
}

static inline unsigned int iph_totlen(const struct sk_buff *skb, const struct iphdr *iph)
{
        u32 len = ntohs(iph->tot_len);

        return (len || !skb_is_gso(skb) || !skb_is_gso_tcp(skb)) ?
               len : skb->len - skb_network_offset(skb);
}

static inline unsigned int skb_ip_totlen(const struct sk_buff *skb)
{
        return iph_totlen(skb, ip_hdr(skb));
}

/* IPv4 datagram length is stored into 16bit field (tot_len) */
#define IP_MAX_MTU        0xFFFFU

static inline void iph_set_totlen(struct iphdr *iph, unsigned int len)
{
        iph->tot_len = len <= IP_MAX_MTU ? htons(len) : 0;
}
#endif        /* _LINUX_IP_H */







































   66 








   68 

   68 
   40 
   39 
   62 



   69 

   69 






























   69 


   55 




   18 







   69 




    2 









   69 
   69 



   68 

















   43 





    6 

    3 
    3 

   27 
    4 

    8 





















   41 
















    1 
































   39 




    4 
    4 

















   40 
   42 













    1 
    1 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/skbuff.h>
#include <linux/sctp.h>
#include <net/gso.h>
#include <net/gro.h>

/**
 *        skb_eth_gso_segment - segmentation handler for ethernet protocols.
 *        @skb: buffer to segment
 *        @features: features for the output path (see dev->features)
 *        @type: Ethernet Protocol ID
 */
struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,
                                    netdev_features_t features, __be16 type)
{
        struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
        struct packet_offload *ptype;

        rcu_read_lock();
        list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
                if (ptype->type == type && ptype->callbacks.gso_segment) {
                        segs = ptype->callbacks.gso_segment(skb, features);
                        break;
                }
        }
        rcu_read_unlock();

        return segs;
}
EXPORT_SYMBOL(skb_eth_gso_segment);

/**
 *        skb_mac_gso_segment - mac layer segmentation handler.
 *        @skb: buffer to segment
 *        @features: features for the output path (see dev->features)
 */
struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
                                    netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
        struct packet_offload *ptype;
        int vlan_depth = skb->mac_len;
        __be16 type = skb_network_protocol(skb, &vlan_depth);

        if (unlikely(!type))
                return ERR_PTR(-EINVAL);

        __skb_pull(skb, vlan_depth);

        rcu_read_lock();
        list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) {
                if (ptype->type == type && ptype->callbacks.gso_segment) {
                        segs = ptype->callbacks.gso_segment(skb, features);
                        break;
                }
        }
        rcu_read_unlock();

        __skb_push(skb, skb->data - skb_mac_header(skb));

        return segs;
}
EXPORT_SYMBOL(skb_mac_gso_segment);
/* openvswitch calls this on rx path, so we need a different check.
 */
static bool skb_needs_check(const struct sk_buff *skb, bool tx_path)
{
        if (tx_path)
                return skb->ip_summed != CHECKSUM_PARTIAL &&
                       skb->ip_summed != CHECKSUM_UNNECESSARY;

        return skb->ip_summed == CHECKSUM_NONE;
}

/**
 *        __skb_gso_segment - Perform segmentation on skb.
 *        @skb: buffer to segment
 *        @features: features for the output path (see dev->features)
 *        @tx_path: whether it is called in TX path
 *
 *        This function segments the given skb and returns a list of segments.
 *
 *        It may return NULL if the skb requires no segmentation.  This is
 *        only possible when GSO is used for verifying header integrity.
 *
 *        Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
 */
struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
                                  netdev_features_t features, bool tx_path)
{
        struct sk_buff *segs;

        if (unlikely(skb_needs_check(skb, tx_path))) {
                int err;

                /* We're going to init ->check field in TCP or UDP header */
                err = skb_cow_head(skb, 0);
                if (err < 0)
                        return ERR_PTR(err);
        }

        /* Only report GSO partial support if it will enable us to
         * support segmentation on this frame without needing additional
         * work.
         */
        if (features & NETIF_F_GSO_PARTIAL) {
                netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
                struct net_device *dev = skb->dev;

                partial_features |= dev->features & dev->gso_partial_features;
                if (!skb_gso_ok(skb, features | partial_features))
                        features &= ~NETIF_F_GSO_PARTIAL;
        }

        BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
                     sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));

        SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
        SKB_GSO_CB(skb)->encap_level = 0;

        skb_reset_mac_header(skb);
        skb_reset_mac_len(skb);

        segs = skb_mac_gso_segment(skb, features);

        if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
                skb_warn_bad_offload(skb);

        return segs;
}
EXPORT_SYMBOL(__skb_gso_segment);

/**
 * skb_gso_transport_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_transport_seglen is used to determine the real size of the
 * individual segments, including Layer4 headers (TCP/UDP).
 *
 * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
 */
static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
{
        const struct skb_shared_info *shinfo = skb_shinfo(skb);
        unsigned int thlen = 0;

        if (skb->encapsulation) {
                thlen = skb_inner_transport_header(skb) -
                        skb_transport_header(skb);

                if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
                        thlen += inner_tcp_hdrlen(skb);
        } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
                thlen = tcp_hdrlen(skb);
        } else if (unlikely(skb_is_gso_sctp(skb))) {
                thlen = sizeof(struct sctphdr);
        } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
                thlen = sizeof(struct udphdr);
        }
        /* UFO sets gso_size to the size of the fragmentation
         * payload, i.e. the size of the L4 (UDP) header is already
         * accounted for.
         */
        return thlen + shinfo->gso_size;
}

/**
 * skb_gso_network_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_network_seglen is used to determine the real size of the
 * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
 *
 * The MAC/L2 header is not accounted for.
 */
static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
{
        unsigned int hdr_len = skb_transport_header(skb) -
                               skb_network_header(skb);

        return hdr_len + skb_gso_transport_seglen(skb);
}

/**
 * skb_gso_mac_seglen - Return length of individual segments of a gso packet
 *
 * @skb: GSO skb
 *
 * skb_gso_mac_seglen is used to determine the real size of the
 * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
 * headers (TCP/UDP).
 */
static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
{
        unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);

        return hdr_len + skb_gso_transport_seglen(skb);
}

/**
 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
 *
 * There are a couple of instances where we have a GSO skb, and we
 * want to determine what size it would be after it is segmented.
 *
 * We might want to check:
 * -    L3+L4+payload size (e.g. IP forwarding)
 * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
 *
 * This is a helper to do that correctly considering GSO_BY_FRAGS.
 *
 * @skb: GSO skb
 *
 * @seg_len: The segmented length (from skb_gso_*_seglen). In the
 *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
 *
 * @max_len: The maximum permissible length.
 *
 * Returns true if the segmented length <= max length.
 */
static inline bool skb_gso_size_check(const struct sk_buff *skb,
                                      unsigned int seg_len,
                                      unsigned int max_len) {
        const struct skb_shared_info *shinfo = skb_shinfo(skb);
        const struct sk_buff *iter;

        if (shinfo->gso_size != GSO_BY_FRAGS)
                return seg_len <= max_len;

        /* Undo this so we can re-use header sizes */
        seg_len -= GSO_BY_FRAGS;

        skb_walk_frags(skb, iter) {
                if (seg_len + skb_headlen(iter) > max_len)
                        return false;
        }

        return true;
}

/**
 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
 *
 * @skb: GSO skb
 * @mtu: MTU to validate against
 *
 * skb_gso_validate_network_len validates if a given skb will fit a
 * wanted MTU once split. It considers L3 headers, L4 headers, and the
 * payload.
 */
bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
{
        return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
}
EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);

/**
 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
 *
 * @skb: GSO skb
 * @len: length to validate against
 *
 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
 * length once split, including L2, L3 and L4 headers and the payload.
 */
bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
{
        return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
}
EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);




























































































   32 

    7 
    8 

































































    9 


















    9 
    9 
    9 



    9 










    9 



    9 



    9 




    9 





    9 




    9 


    9 


    9 




    9 












    9 






































    9 




























    9 


















    9 







    9 



    2 





    9 
































    9 




    2 
    7 



    9 





















    9 


    9 

    9 

    9 











    9 














































































































































    9 


    9 


















    9 







































































































































































































































































    9 




    9 



    9 







    9 



    9 





    9 















































    2 


    2 







    2 




    2 
    2 

    2 



























































































































    9 



    9 





    9 

    9 

















    9 

















    9 


















































































































































































































    1 





    1 
    1 


































    1 










































































































   11 
   11 

   10 




    5 









   11 
   11 

   11 








    1 

   10 

    8 
    4 




    9 
    3 






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   10 

   10 

   10 








   30 


   31 




   38 







   38 

    1 











   38 













    8 




    7 
    1 








   32 



   32 










   32 



































    4 


    2 


    2 


    2 


    2 








   11 









































   11 
   11 


   11 
   11 
    4 






    2 













   11 




















































































    4 




















   35 



















































































































    7 





    7 

    7 
    7 








    2 

    1 


    7 



   55 






































   40 






















































   63 
   62 

    1 











   59 


    2 
    1 






    1 




   54 


    4 
    1 





   56 



    2 
    1 





   55 


    1 
    1 


















































    6 




   29 








    8 

   31 













   32 








   32 

    8 







   31 


























   32 





   31 

   29 
    3 



   32 
   32 


    5 



   32 

   32 
   32 



    8 


    6 




   29 



   31 
    1 

   31 














   26 





    6 

   29 






   45 



   42 
    6 




   35 


    7 

   17 



   22 













   52 


















   32 


    6 
    8 
   12 









    1 


























    1 



    1 




    4 





    1 




   43 
    1 









    1 





    3 













    1 






   42 
    1 








    8 



   42 


    7 
   43 

   40 
   22 












   40 






    6 



   40 

   38 
    2 








   37 













    1 
    2 


   40 

























   29 




   52 







   29 






    1 


   36 










    3 







   39 











    9 



   39 
















   13 





    2 
   10 



   12 
    2 
    3 
    8 

    2 









   57 

















    1 





   11 
    4 






    3 














    3 
    4 






   52 















   53 
    4 

   56 
    1 

   56 
    1 

   57 








   56 
    1 

   56 
    1 


    1 




   21 

   54 


   55 
    2 


    1 






   56 
    1 







    1 






   56 
    1 






   55 

    1 



    2 


   54 



    3 










    1 





   56 
    1 
    1 






   54 

    1 



    1 
    1 



    1 




   13 

   54 


   55 



    2 



   55 



    2 



   56 



    1 



   54 



    1 

    2 



   55 


    1 

    1 


   56 













    1 







    2 






    2 







    1 




   52 





   52 



    1 



   52 
    1 






   51 





    2 


   52 
    1 

   51 






    1 

    1 











   55 
   55 






    3 


   52 





   22 









   16 




    1 










    6 





    6 

    1 








    1 






    1 












    1 










    6 


    6 
    1 


    6 



    6 







    8 





    8 

    7 
    1 



   48 





































   73 







    2 


   63 

   17 




    8 






   73 


   68 

   10 




    3 






    2 

    1 



    2 





    1 

    2 

    3 








    1 

    1 



    1 




    2 


   72 



   73 



   73 



   72 




   73 











   73 




























































  250 








  155 








 1101 



 1099 
  250 

    5 

    2 

























































































  148 




  147 





































    2 
    2 







    2 




   11 



   10 






    2 





    5 








































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
// SPDX-License-Identifier: GPL-2.0-only
/*
 * VXLAN: Virtual eXtensible Local Area Network
 *
 * Copyright (c) 2012-2013 Vyatta Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/udp.h>
#include <linux/igmp.h>
#include <linux/if_ether.h>
#include <linux/ethtool.h>
#include <linux/rhashtable.h>
#include <net/arp.h>
#include <net/ndisc.h>
#include <net/gro.h>
#include <net/ipv6_stubs.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/rtnetlink.h>
#include <net/inet_ecn.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/netdev_lock.h>
#include <net/tun_proto.h>
#include <net/vxlan.h>
#include <net/nexthop.h>

#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_tunnel.h>
#include <net/ip6_checksum.h>
#endif

#include "vxlan_private.h"

#define VXLAN_VERSION        "0.1"

#define FDB_AGE_DEFAULT 300 /* 5 min */
#define FDB_AGE_INTERVAL (10 * HZ)        /* rescan interval */

/* UDP port for VXLAN traffic.
 * The IANA assigned port is 4789, but the Linux default is 8472
 * for compatibility with early adopters.
 */
static unsigned short vxlan_port __read_mostly = 8472;
module_param_named(udp_port, vxlan_port, ushort, 0444);
MODULE_PARM_DESC(udp_port, "Destination UDP port");

static bool log_ecn_error = true;
module_param(log_ecn_error, bool, 0644);
MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");

unsigned int vxlan_net_id;

const u8 all_zeros_mac[ETH_ALEN + 2];
static struct rtnl_link_ops vxlan_link_ops;

static int vxlan_sock_add(struct vxlan_dev *vxlan);

static void vxlan_vs_del_dev(struct vxlan_dev *vxlan);

static const struct rhashtable_params vxlan_fdb_rht_params = {
        .head_offset = offsetof(struct vxlan_fdb, rhnode),
        .key_offset = offsetof(struct vxlan_fdb, key),
        .key_len = sizeof(struct vxlan_fdb_key),
        .automatic_shrinking = true,
};

static inline bool vxlan_collect_metadata(struct vxlan_sock *vs)
{
        return vs->flags & VXLAN_F_COLLECT_METADATA ||
               ip_tunnel_collect_metadata();
}

/* Find VXLAN socket based on network namespace, address family, UDP port,
 * enabled unshareable flags and socket device binding (see l3mdev with
 * non-default VRF).
 */
static struct vxlan_sock *vxlan_find_sock(struct net *net, sa_family_t family,
                                          __be16 port, u32 flags, int ifindex)
{
        struct vxlan_sock *vs;

        flags &= VXLAN_F_RCV_FLAGS;

        hlist_for_each_entry_rcu(vs, vs_head(net, port), hlist) {
                if (inet_sk(vs->sock->sk)->inet_sport == port &&
                    vxlan_get_sk_family(vs) == family &&
                    vs->flags == flags &&
                    vs->sock->sk->sk_bound_dev_if == ifindex)
                        return vs;
        }
        return NULL;
}

static struct vxlan_dev *vxlan_vs_find_vni(struct vxlan_sock *vs,
                                           int ifindex, __be32 vni,
                                           struct vxlan_vni_node **vninode)
{
        struct vxlan_vni_node *vnode;
        struct vxlan_dev_node *node;

        /* For flow based devices, map all packets to VNI 0 */
        if (vs->flags & VXLAN_F_COLLECT_METADATA &&
            !(vs->flags & VXLAN_F_VNIFILTER))
                vni = 0;

        hlist_for_each_entry_rcu(node, vni_head(vs, vni), hlist) {
                if (!node->vxlan)
                        continue;
                vnode = NULL;
                if (node->vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
                        vnode = vxlan_vnifilter_lookup(node->vxlan, vni);
                        if (!vnode)
                                continue;
                } else if (node->vxlan->default_dst.remote_vni != vni) {
                        continue;
                }

                if (IS_ENABLED(CONFIG_IPV6)) {
                        const struct vxlan_config *cfg = &node->vxlan->cfg;

                        if ((cfg->flags & VXLAN_F_IPV6_LINKLOCAL) &&
                            cfg->remote_ifindex != ifindex)
                                continue;
                }

                if (vninode)
                        *vninode = vnode;
                return node->vxlan;
        }

        return NULL;
}

/* Look up VNI in a per net namespace table */
static struct vxlan_dev *vxlan_find_vni(struct net *net, int ifindex,
                                        __be32 vni, sa_family_t family,
                                        __be16 port, u32 flags)
{
        struct vxlan_sock *vs;

        vs = vxlan_find_sock(net, family, port, flags, ifindex);
        if (!vs)
                return NULL;

        return vxlan_vs_find_vni(vs, ifindex, vni, NULL);
}

/* Fill in neighbour message in skbuff. */
static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
                          const struct vxlan_fdb *fdb,
                          u32 portid, u32 seq, int type, unsigned int flags,
                          const struct vxlan_rdst *rdst)
{
        unsigned long now = jiffies;
        struct nda_cacheinfo ci;
        bool send_ip, send_eth;
        struct nlmsghdr *nlh;
        struct nexthop *nh;
        struct ndmsg *ndm;
        int nh_family;
        u32 nh_id;

        nlh = nlmsg_put(skb, portid, seq, type, sizeof(*ndm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ndm = nlmsg_data(nlh);
        memset(ndm, 0, sizeof(*ndm));

        send_eth = send_ip = true;

        rcu_read_lock();
        nh = rcu_dereference(fdb->nh);
        if (nh) {
                nh_family = nexthop_get_family(nh);
                nh_id = nh->id;
        }
        rcu_read_unlock();

        if (type == RTM_GETNEIGH) {
                if (rdst) {
                        send_ip = !vxlan_addr_any(&rdst->remote_ip);
                        ndm->ndm_family = send_ip ? rdst->remote_ip.sa.sa_family : AF_INET;
                } else if (nh) {
                        ndm->ndm_family = nh_family;
                }
                send_eth = !is_zero_ether_addr(fdb->key.eth_addr);
        } else
                ndm->ndm_family        = AF_BRIDGE;
        ndm->ndm_state = fdb->state;
        ndm->ndm_ifindex = vxlan->dev->ifindex;
        ndm->ndm_flags = fdb->flags;
        if (rdst && rdst->offloaded)
                ndm->ndm_flags |= NTF_OFFLOADED;
        ndm->ndm_type = RTN_UNICAST;

        if (!net_eq(dev_net(vxlan->dev), vxlan->net) &&
            nla_put_s32(skb, NDA_LINK_NETNSID,
                        peernet2id(dev_net(vxlan->dev), vxlan->net)))
                goto nla_put_failure;

        if (send_eth && nla_put(skb, NDA_LLADDR, ETH_ALEN, &fdb->key.eth_addr))
                goto nla_put_failure;
        if (nh) {
                if (nla_put_u32(skb, NDA_NH_ID, nh_id))
                        goto nla_put_failure;
        } else if (rdst) {
                if (send_ip && vxlan_nla_put_addr(skb, NDA_DST,
                                                  &rdst->remote_ip))
                        goto nla_put_failure;

                if (rdst->remote_port &&
                    rdst->remote_port != vxlan->cfg.dst_port &&
                    nla_put_be16(skb, NDA_PORT, rdst->remote_port))
                        goto nla_put_failure;
                if (rdst->remote_vni != vxlan->default_dst.remote_vni &&
                    nla_put_u32(skb, NDA_VNI, be32_to_cpu(rdst->remote_vni)))
                        goto nla_put_failure;
                if (rdst->remote_ifindex &&
                    nla_put_u32(skb, NDA_IFINDEX, rdst->remote_ifindex))
                        goto nla_put_failure;
        }

        if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) && fdb->key.vni &&
            nla_put_u32(skb, NDA_SRC_VNI,
                        be32_to_cpu(fdb->key.vni)))
                goto nla_put_failure;

        ci.ndm_used         = jiffies_to_clock_t(now - READ_ONCE(fdb->used));
        ci.ndm_confirmed = 0;
        ci.ndm_updated         = jiffies_to_clock_t(now - READ_ONCE(fdb->updated));
        ci.ndm_refcnt         = 0;

        if (nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static inline size_t vxlan_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg))
                + nla_total_size(ETH_ALEN) /* NDA_LLADDR */
                + nla_total_size(sizeof(struct in6_addr)) /* NDA_DST */
                + nla_total_size(sizeof(__be16)) /* NDA_PORT */
                + nla_total_size(sizeof(__be32)) /* NDA_VNI */
                + nla_total_size(sizeof(__u32)) /* NDA_IFINDEX */
                + nla_total_size(sizeof(__s32)) /* NDA_LINK_NETNSID */
                + nla_total_size(sizeof(struct nda_cacheinfo));
}

static void __vxlan_fdb_notify(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
                               struct vxlan_rdst *rd, int type)
{
        struct net *net = dev_net(vxlan->dev);
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(vxlan_nlmsg_size(), GFP_ATOMIC);
        if (skb == NULL)
                goto errout;

        err = vxlan_fdb_info(skb, vxlan, fdb, 0, 0, type, 0, rd);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in vxlan_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
}

static void vxlan_fdb_switchdev_notifier_info(const struct vxlan_dev *vxlan,
                            const struct vxlan_fdb *fdb,
                            const struct vxlan_rdst *rd,
                            struct netlink_ext_ack *extack,
                            struct switchdev_notifier_vxlan_fdb_info *fdb_info)
{
        fdb_info->info.dev = vxlan->dev;
        fdb_info->info.extack = extack;
        fdb_info->remote_ip = rd->remote_ip;
        fdb_info->remote_port = rd->remote_port;
        fdb_info->remote_vni = rd->remote_vni;
        fdb_info->remote_ifindex = rd->remote_ifindex;
        memcpy(fdb_info->eth_addr, fdb->key.eth_addr, ETH_ALEN);
        fdb_info->vni = fdb->key.vni;
        fdb_info->offloaded = rd->offloaded;
        fdb_info->added_by_user = fdb->flags & NTF_VXLAN_ADDED_BY_USER;
}

static int vxlan_fdb_switchdev_call_notifiers(struct vxlan_dev *vxlan,
                                              struct vxlan_fdb *fdb,
                                              struct vxlan_rdst *rd,
                                              bool adding,
                                              struct netlink_ext_ack *extack)
{
        struct switchdev_notifier_vxlan_fdb_info info;
        enum switchdev_notifier_type notifier_type;
        int ret;

        if (WARN_ON(!rd))
                return 0;

        notifier_type = adding ? SWITCHDEV_VXLAN_FDB_ADD_TO_DEVICE
                               : SWITCHDEV_VXLAN_FDB_DEL_TO_DEVICE;
        vxlan_fdb_switchdev_notifier_info(vxlan, fdb, rd, NULL, &info);
        ret = call_switchdev_notifiers(notifier_type, vxlan->dev,
                                       &info.info, extack);
        return notifier_to_errno(ret);
}

static int vxlan_fdb_notify(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
                            struct vxlan_rdst *rd, int type, bool swdev_notify,
                            struct netlink_ext_ack *extack)
{
        int err;

        if (swdev_notify && rd) {
                switch (type) {
                case RTM_NEWNEIGH:
                        err = vxlan_fdb_switchdev_call_notifiers(vxlan, fdb, rd,
                                                                 true, extack);
                        if (err)
                                return err;
                        break;
                case RTM_DELNEIGH:
                        vxlan_fdb_switchdev_call_notifiers(vxlan, fdb, rd,
                                                           false, extack);
                        break;
                }
        }

        __vxlan_fdb_notify(vxlan, fdb, rd, type);
        return 0;
}

static void vxlan_ip_miss(struct net_device *dev, union vxlan_addr *ipa)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb f = {
                .state = NUD_STALE,
        };
        struct vxlan_rdst remote = {
                .remote_ip = *ipa, /* goes to NDA_DST */
                .remote_vni = cpu_to_be32(VXLAN_N_VID),
        };

        vxlan_fdb_notify(vxlan, &f, &remote, RTM_GETNEIGH, true, NULL);
}

static void vxlan_fdb_miss(struct vxlan_dev *vxlan, const u8 eth_addr[ETH_ALEN])
{
        struct vxlan_fdb f = {
                .state = NUD_STALE,
        };
        struct vxlan_rdst remote = { };

        memcpy(f.key.eth_addr, eth_addr, ETH_ALEN);

        vxlan_fdb_notify(vxlan, &f, &remote, RTM_GETNEIGH, true, NULL);
}

/* Look up Ethernet address in forwarding table */
static struct vxlan_fdb *vxlan_find_mac_rcu(struct vxlan_dev *vxlan,
                                            const u8 *mac, __be32 vni)
{
        struct vxlan_fdb_key key;

        memset(&key, 0, sizeof(key));
        memcpy(key.eth_addr, mac, sizeof(key.eth_addr));
        if (!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA))
                key.vni = vxlan->default_dst.remote_vni;
        else
                key.vni = vni;

        return rhashtable_lookup(&vxlan->fdb_hash_tbl, &key,
                                 vxlan_fdb_rht_params);
}

static struct vxlan_fdb *vxlan_find_mac_tx(struct vxlan_dev *vxlan,
                                           const u8 *mac, __be32 vni)
{
        struct vxlan_fdb *f;

        f = vxlan_find_mac_rcu(vxlan, mac, vni);
        if (f) {
                unsigned long now = jiffies;

                if (READ_ONCE(f->used) != now)
                        WRITE_ONCE(f->used, now);
        }

        return f;
}

static struct vxlan_fdb *vxlan_find_mac(struct vxlan_dev *vxlan,
                                        const u8 *mac, __be32 vni)
{
        struct vxlan_fdb *f;

        lockdep_assert_held_once(&vxlan->hash_lock);

        rcu_read_lock();
        f = vxlan_find_mac_rcu(vxlan, mac, vni);
        rcu_read_unlock();

        return f;
}

/* caller should hold vxlan->hash_lock */
static struct vxlan_rdst *vxlan_fdb_find_rdst(struct vxlan_fdb *f,
                                              union vxlan_addr *ip, __be16 port,
                                              __be32 vni, __u32 ifindex)
{
        struct vxlan_rdst *rd;

        list_for_each_entry(rd, &f->remotes, list) {
                if (vxlan_addr_equal(&rd->remote_ip, ip) &&
                    rd->remote_port == port &&
                    rd->remote_vni == vni &&
                    rd->remote_ifindex == ifindex)
                        return rd;
        }

        return NULL;
}

int vxlan_fdb_find_uc(struct net_device *dev, const u8 *mac, __be32 vni,
                      struct switchdev_notifier_vxlan_fdb_info *fdb_info)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        u8 eth_addr[ETH_ALEN + 2] = { 0 };
        struct vxlan_rdst *rdst = NULL;
        struct vxlan_fdb *f;
        int rc = 0;

        if (is_multicast_ether_addr(mac) ||
            is_zero_ether_addr(mac))
                return -EINVAL;

        ether_addr_copy(eth_addr, mac);

        rcu_read_lock();

        f = vxlan_find_mac_rcu(vxlan, eth_addr, vni);
        if (f)
                rdst = first_remote_rcu(f);
        if (!rdst) {
                rc = -ENOENT;
                goto out;
        }

        vxlan_fdb_switchdev_notifier_info(vxlan, f, rdst, NULL, fdb_info);

out:
        rcu_read_unlock();
        return rc;
}
EXPORT_SYMBOL_GPL(vxlan_fdb_find_uc);

static int vxlan_fdb_notify_one(struct notifier_block *nb,
                                const struct vxlan_dev *vxlan,
                                const struct vxlan_fdb *f,
                                const struct vxlan_rdst *rdst,
                                struct netlink_ext_ack *extack)
{
        struct switchdev_notifier_vxlan_fdb_info fdb_info;
        int rc;

        vxlan_fdb_switchdev_notifier_info(vxlan, f, rdst, extack, &fdb_info);
        rc = nb->notifier_call(nb, SWITCHDEV_VXLAN_FDB_ADD_TO_DEVICE,
                               &fdb_info);
        return notifier_to_errno(rc);
}

int vxlan_fdb_replay(const struct net_device *dev, __be32 vni,
                     struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan;
        struct vxlan_rdst *rdst;
        struct vxlan_fdb *f;
        int rc = 0;

        if (!netif_is_vxlan(dev))
                return -EINVAL;
        vxlan = netdev_priv(dev);

        spin_lock_bh(&vxlan->hash_lock);
        hlist_for_each_entry(f, &vxlan->fdb_list, fdb_node) {
                if (f->key.vni == vni) {
                        list_for_each_entry(rdst, &f->remotes, list) {
                                rc = vxlan_fdb_notify_one(nb, vxlan, f, rdst,
                                                          extack);
                                if (rc)
                                        goto unlock;
                        }
                }
        }
        spin_unlock_bh(&vxlan->hash_lock);
        return 0;

unlock:
        spin_unlock_bh(&vxlan->hash_lock);
        return rc;
}
EXPORT_SYMBOL_GPL(vxlan_fdb_replay);

void vxlan_fdb_clear_offload(const struct net_device *dev, __be32 vni)
{
        struct vxlan_dev *vxlan;
        struct vxlan_rdst *rdst;
        struct vxlan_fdb *f;

        if (!netif_is_vxlan(dev))
                return;
        vxlan = netdev_priv(dev);

        spin_lock_bh(&vxlan->hash_lock);
        hlist_for_each_entry(f, &vxlan->fdb_list, fdb_node) {
                if (f->key.vni == vni) {
                        list_for_each_entry(rdst, &f->remotes, list)
                                rdst->offloaded = false;
                }
        }
        spin_unlock_bh(&vxlan->hash_lock);

}
EXPORT_SYMBOL_GPL(vxlan_fdb_clear_offload);

/* Replace destination of unicast mac */
static int vxlan_fdb_replace(struct vxlan_fdb *f,
                             union vxlan_addr *ip, __be16 port, __be32 vni,
                             __u32 ifindex, struct vxlan_rdst *oldrd)
{
        struct vxlan_rdst *rd;

        rd = vxlan_fdb_find_rdst(f, ip, port, vni, ifindex);
        if (rd)
                return 0;

        rd = list_first_entry_or_null(&f->remotes, struct vxlan_rdst, list);
        if (!rd)
                return 0;

        *oldrd = *rd;
        dst_cache_reset(&rd->dst_cache);
        rd->remote_ip = *ip;
        rd->remote_port = port;
        rd->remote_vni = vni;
        rd->remote_ifindex = ifindex;
        rd->offloaded = false;
        return 1;
}

/* Add/update destinations for multicast */
static int vxlan_fdb_append(struct vxlan_fdb *f,
                            union vxlan_addr *ip, __be16 port, __be32 vni,
                            __u32 ifindex, struct vxlan_rdst **rdp)
{
        struct vxlan_rdst *rd;

        rd = vxlan_fdb_find_rdst(f, ip, port, vni, ifindex);
        if (rd)
                return 0;

        rd = kmalloc(sizeof(*rd), GFP_ATOMIC);
        if (rd == NULL)
                return -ENOMEM;

        /* The driver can work correctly without a dst cache, so do not treat
         * dst cache initialization errors as fatal.
         */
        dst_cache_init(&rd->dst_cache, GFP_ATOMIC | __GFP_NOWARN);

        rd->remote_ip = *ip;
        rd->remote_port = port;
        rd->offloaded = false;
        rd->remote_vni = vni;
        rd->remote_ifindex = ifindex;

        list_add_tail_rcu(&rd->list, &f->remotes);

        *rdp = rd;
        return 1;
}

static bool vxlan_parse_gpe_proto(const struct vxlanhdr *hdr, __be16 *protocol)
{
        const struct vxlanhdr_gpe *gpe = (const struct vxlanhdr_gpe *)hdr;

        /* Need to have Next Protocol set for interfaces in GPE mode. */
        if (!gpe->np_applied)
                return false;
        /* "The initial version is 0. If a receiver does not support the
         * version indicated it MUST drop the packet.
         */
        if (gpe->version != 0)
                return false;
        /* "When the O bit is set to 1, the packet is an OAM packet and OAM
         * processing MUST occur." However, we don't implement OAM
         * processing, thus drop the packet.
         */
        if (gpe->oam_flag)
                return false;

        *protocol = tun_p_to_eth_p(gpe->next_protocol);
        if (!*protocol)
                return false;

        return true;
}

static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb,
                                          unsigned int off,
                                          struct vxlanhdr *vh, size_t hdrlen,
                                          __be32 vni_field,
                                          struct gro_remcsum *grc,
                                          bool nopartial)
{
        size_t start, offset;

        if (skb->remcsum_offload)
                return vh;

        if (!NAPI_GRO_CB(skb)->csum_valid)
                return NULL;

        start = vxlan_rco_start(vni_field);
        offset = start + vxlan_rco_offset(vni_field);

        vh = skb_gro_remcsum_process(skb, (void *)vh, off, hdrlen,
                                     start, offset, grc, nopartial);

        skb->remcsum_offload = 1;

        return vh;
}

static struct vxlanhdr *vxlan_gro_prepare_receive(struct sock *sk,
                                                  struct list_head *head,
                                                  struct sk_buff *skb,
                                                  struct gro_remcsum *grc)
{
        struct sk_buff *p;
        struct vxlanhdr *vh, *vh2;
        unsigned int hlen, off_vx;
        struct vxlan_sock *vs = rcu_dereference_sk_user_data(sk);
        __be32 flags;

        skb_gro_remcsum_init(grc);

        off_vx = skb_gro_offset(skb);
        hlen = off_vx + sizeof(*vh);
        vh = skb_gro_header(skb, hlen, off_vx);
        if (unlikely(!vh))
                return NULL;

        skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));

        flags = vh->vx_flags;

        if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
                vh = vxlan_gro_remcsum(skb, off_vx, vh, sizeof(struct vxlanhdr),
                                       vh->vx_vni, grc,
                                       !!(vs->flags &
                                          VXLAN_F_REMCSUM_NOPARTIAL));

                if (!vh)
                        return NULL;
        }

        skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */

        list_for_each_entry(p, head, list) {
                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                vh2 = (struct vxlanhdr *)(p->data + off_vx);
                if (vh->vx_flags != vh2->vx_flags ||
                    vh->vx_vni != vh2->vx_vni) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
        }

        return vh;
}

static struct sk_buff *vxlan_gro_receive(struct sock *sk,
                                         struct list_head *head,
                                         struct sk_buff *skb)
{
        struct sk_buff *pp = NULL;
        struct gro_remcsum grc;
        int flush = 1;

        if (vxlan_gro_prepare_receive(sk, head, skb, &grc)) {
                pp = call_gro_receive(eth_gro_receive, head, skb);
                flush = 0;
        }
        skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
        return pp;
}

static struct sk_buff *vxlan_gpe_gro_receive(struct sock *sk,
                                             struct list_head *head,
                                             struct sk_buff *skb)
{
        const struct packet_offload *ptype;
        struct sk_buff *pp = NULL;
        struct gro_remcsum grc;
        struct vxlanhdr *vh;
        __be16 protocol;
        int flush = 1;

        vh = vxlan_gro_prepare_receive(sk, head, skb, &grc);
        if (vh) {
                if (!vxlan_parse_gpe_proto(vh, &protocol))
                        goto out;
                ptype = gro_find_receive_by_type(protocol);
                if (!ptype)
                        goto out;
                pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
                flush = 0;
        }
out:
        skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
        return pp;
}

static int vxlan_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
{
        /* Sets 'skb->inner_mac_header' since we are always called with
         * 'skb->encapsulation' set.
         */
        return eth_gro_complete(skb, nhoff + sizeof(struct vxlanhdr));
}

static int vxlan_gpe_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
{
        struct vxlanhdr *vh = (struct vxlanhdr *)(skb->data + nhoff);
        const struct packet_offload *ptype;
        int err = -ENOSYS;
        __be16 protocol;

        if (!vxlan_parse_gpe_proto(vh, &protocol))
                return err;
        ptype = gro_find_complete_by_type(protocol);
        if (ptype)
                err = ptype->callbacks.gro_complete(skb, nhoff + sizeof(struct vxlanhdr));
        return err;
}

static struct vxlan_fdb *vxlan_fdb_alloc(struct vxlan_dev *vxlan, const u8 *mac,
                                         __u16 state, __be32 src_vni,
                                         __u16 ndm_flags)
{
        struct vxlan_fdb *f;

        f = kmalloc(sizeof(*f), GFP_ATOMIC);
        if (!f)
                return NULL;
        memset(&f->key, 0, sizeof(f->key));
        f->state = state;
        f->flags = ndm_flags;
        f->updated = f->used = jiffies;
        f->key.vni = src_vni;
        f->nh = NULL;
        RCU_INIT_POINTER(f->vdev, vxlan);
        INIT_LIST_HEAD(&f->nh_list);
        INIT_LIST_HEAD(&f->remotes);
        memcpy(f->key.eth_addr, mac, ETH_ALEN);

        return f;
}

static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
                               u32 nhid, struct netlink_ext_ack *extack)
{
        struct nexthop *old_nh = rtnl_dereference(fdb->nh);
        struct nexthop *nh;
        int err = -EINVAL;

        if (old_nh && old_nh->id == nhid)
                return 0;

        nh = nexthop_find_by_id(vxlan->net, nhid);
        if (!nh) {
                NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
                goto err_inval;
        }

        if (!nexthop_get(nh)) {
                NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
                nh = NULL;
                goto err_inval;
        }
        if (!nexthop_is_fdb(nh)) {
                NL_SET_ERR_MSG(extack, "Nexthop is not a fdb nexthop");
                goto err_inval;
        }

        if (!nexthop_is_multipath(nh)) {
                NL_SET_ERR_MSG(extack, "Nexthop is not a multipath group");
                goto err_inval;
        }

        /* check nexthop group family */
        switch (vxlan->default_dst.remote_ip.sa.sa_family) {
        case AF_INET:
                if (!nexthop_has_v4(nh)) {
                        err = -EAFNOSUPPORT;
                        NL_SET_ERR_MSG(extack, "Nexthop group family not supported");
                        goto err_inval;
                }
                break;
        case AF_INET6:
                if (nexthop_has_v4(nh)) {
                        err = -EAFNOSUPPORT;
                        NL_SET_ERR_MSG(extack, "Nexthop group family not supported");
                        goto err_inval;
                }
        }

        if (old_nh) {
                list_del_rcu(&fdb->nh_list);
                nexthop_put(old_nh);
        }
        rcu_assign_pointer(fdb->nh, nh);
        list_add_tail_rcu(&fdb->nh_list, &nh->fdb_list);
        return 1;

err_inval:
        if (nh)
                nexthop_put(nh);
        return err;
}

int vxlan_fdb_create(struct vxlan_dev *vxlan,
                     const u8 *mac, union vxlan_addr *ip,
                     __u16 state, __be16 port, __be32 src_vni,
                     __be32 vni, __u32 ifindex, __u16 ndm_flags,
                     u32 nhid, struct vxlan_fdb **fdb,
                     struct netlink_ext_ack *extack)
{
        struct vxlan_rdst *rd = NULL;
        struct vxlan_fdb *f;
        int rc;

        if (vxlan->cfg.addrmax &&
            vxlan->addrcnt >= vxlan->cfg.addrmax)
                return -ENOSPC;

        netdev_dbg(vxlan->dev, "add %pM -> %pIS\n", mac, ip);
        f = vxlan_fdb_alloc(vxlan, mac, state, src_vni, ndm_flags);
        if (!f)
                return -ENOMEM;

        if (nhid)
                rc = vxlan_fdb_nh_update(vxlan, f, nhid, extack);
        else
                rc = vxlan_fdb_append(f, ip, port, vni, ifindex, &rd);
        if (rc < 0)
                goto errout;

        rc = rhashtable_lookup_insert_fast(&vxlan->fdb_hash_tbl, &f->rhnode,
                                           vxlan_fdb_rht_params);
        if (rc)
                goto destroy_remote;

        ++vxlan->addrcnt;
        hlist_add_head_rcu(&f->fdb_node, &vxlan->fdb_list);

        *fdb = f;

        return 0;

destroy_remote:
        if (rcu_access_pointer(f->nh)) {
                list_del_rcu(&f->nh_list);
                nexthop_put(rtnl_dereference(f->nh));
        } else {
                list_del(&rd->list);
                dst_cache_destroy(&rd->dst_cache);
                kfree(rd);
        }
errout:
        kfree(f);
        return rc;
}

static void __vxlan_fdb_free(struct vxlan_fdb *f)
{
        struct vxlan_rdst *rd, *nd;
        struct nexthop *nh;

        nh = rcu_dereference_raw(f->nh);
        if (nh) {
                rcu_assign_pointer(f->nh, NULL);
                rcu_assign_pointer(f->vdev, NULL);
                nexthop_put(nh);
        }

        list_for_each_entry_safe(rd, nd, &f->remotes, list) {
                dst_cache_destroy(&rd->dst_cache);
                kfree(rd);
        }
        kfree(f);
}

static void vxlan_fdb_free(struct rcu_head *head)
{
        struct vxlan_fdb *f = container_of(head, struct vxlan_fdb, rcu);

        __vxlan_fdb_free(f);
}

static void vxlan_fdb_destroy(struct vxlan_dev *vxlan, struct vxlan_fdb *f,
                              bool do_notify, bool swdev_notify)
{
        struct vxlan_rdst *rd;

        netdev_dbg(vxlan->dev, "delete %pM\n", f->key.eth_addr);

        --vxlan->addrcnt;
        if (do_notify) {
                if (rcu_access_pointer(f->nh))
                        vxlan_fdb_notify(vxlan, f, NULL, RTM_DELNEIGH,
                                         swdev_notify, NULL);
                else
                        list_for_each_entry(rd, &f->remotes, list)
                                vxlan_fdb_notify(vxlan, f, rd, RTM_DELNEIGH,
                                                 swdev_notify, NULL);
        }

        hlist_del_init_rcu(&f->fdb_node);
        rhashtable_remove_fast(&vxlan->fdb_hash_tbl, &f->rhnode,
                               vxlan_fdb_rht_params);
        list_del_rcu(&f->nh_list);
        call_rcu(&f->rcu, vxlan_fdb_free);
}

static void vxlan_dst_free(struct rcu_head *head)
{
        struct vxlan_rdst *rd = container_of(head, struct vxlan_rdst, rcu);

        dst_cache_destroy(&rd->dst_cache);
        kfree(rd);
}

static int vxlan_fdb_update_existing(struct vxlan_dev *vxlan,
                                     union vxlan_addr *ip,
                                     __u16 state, __u16 flags,
                                     __be16 port, __be32 vni,
                                     __u32 ifindex, __u16 ndm_flags,
                                     struct vxlan_fdb *f, u32 nhid,
                                     bool swdev_notify,
                                     struct netlink_ext_ack *extack)
{
        __u16 fdb_flags = (ndm_flags & ~NTF_USE);
        struct vxlan_rdst *rd = NULL;
        struct vxlan_rdst oldrd;
        int notify = 0;
        int rc = 0;
        int err;

        if (nhid && !rcu_access_pointer(f->nh)) {
                NL_SET_ERR_MSG(extack,
                               "Cannot replace an existing non nexthop fdb with a nexthop");
                return -EOPNOTSUPP;
        }

        if (nhid && (flags & NLM_F_APPEND)) {
                NL_SET_ERR_MSG(extack,
                               "Cannot append to a nexthop fdb");
                return -EOPNOTSUPP;
        }

        /* Do not allow an externally learned entry to take over an entry added
         * by the user.
         */
        if (!(fdb_flags & NTF_EXT_LEARNED) ||
            !(f->flags & NTF_VXLAN_ADDED_BY_USER)) {
                if (f->state != state) {
                        f->state = state;
                        notify = 1;
                }
                if (f->flags != fdb_flags) {
                        f->flags = fdb_flags;
                        notify = 1;
                }
        }

        if ((flags & NLM_F_REPLACE)) {
                /* Only change unicasts */
                if (!(is_multicast_ether_addr(f->key.eth_addr) ||
                      is_zero_ether_addr(f->key.eth_addr))) {
                        if (nhid) {
                                rc = vxlan_fdb_nh_update(vxlan, f, nhid, extack);
                                if (rc < 0)
                                        return rc;
                        } else {
                                rc = vxlan_fdb_replace(f, ip, port, vni,
                                                       ifindex, &oldrd);
                        }
                        notify |= rc;
                } else {
                        NL_SET_ERR_MSG(extack, "Cannot replace non-unicast fdb entries");
                        return -EOPNOTSUPP;
                }
        }
        if ((flags & NLM_F_APPEND) &&
            (is_multicast_ether_addr(f->key.eth_addr) ||
             is_zero_ether_addr(f->key.eth_addr))) {
                rc = vxlan_fdb_append(f, ip, port, vni, ifindex, &rd);

                if (rc < 0)
                        return rc;
                notify |= rc;
        }

        if (ndm_flags & NTF_USE)
                WRITE_ONCE(f->updated, jiffies);

        if (notify) {
                if (rd == NULL)
                        rd = first_remote_rtnl(f);

                WRITE_ONCE(f->updated, jiffies);
                err = vxlan_fdb_notify(vxlan, f, rd, RTM_NEWNEIGH,
                                       swdev_notify, extack);
                if (err)
                        goto err_notify;
        }

        return 0;

err_notify:
        if (nhid)
                return err;
        if ((flags & NLM_F_REPLACE) && rc)
                *rd = oldrd;
        else if ((flags & NLM_F_APPEND) && rc) {
                list_del_rcu(&rd->list);
                call_rcu(&rd->rcu, vxlan_dst_free);
        }
        return err;
}

static int vxlan_fdb_update_create(struct vxlan_dev *vxlan,
                                   const u8 *mac, union vxlan_addr *ip,
                                   __u16 state, __u16 flags,
                                   __be16 port, __be32 src_vni, __be32 vni,
                                   __u32 ifindex, __u16 ndm_flags, u32 nhid,
                                   bool swdev_notify,
                                   struct netlink_ext_ack *extack)
{
        __u16 fdb_flags = (ndm_flags & ~NTF_USE);
        struct vxlan_fdb *f;
        int rc;

        /* Disallow replace to add a multicast entry */
        if ((flags & NLM_F_REPLACE) &&
            (is_multicast_ether_addr(mac) || is_zero_ether_addr(mac)))
                return -EOPNOTSUPP;

        netdev_dbg(vxlan->dev, "add %pM -> %pIS\n", mac, ip);
        rc = vxlan_fdb_create(vxlan, mac, ip, state, port, src_vni,
                              vni, ifindex, fdb_flags, nhid, &f, extack);
        if (rc < 0)
                return rc;

        rc = vxlan_fdb_notify(vxlan, f, first_remote_rtnl(f), RTM_NEWNEIGH,
                              swdev_notify, extack);
        if (rc)
                goto err_notify;

        return 0;

err_notify:
        vxlan_fdb_destroy(vxlan, f, false, false);
        return rc;
}

/* Add new entry to forwarding table -- assumes lock held */
int vxlan_fdb_update(struct vxlan_dev *vxlan,
                     const u8 *mac, union vxlan_addr *ip,
                     __u16 state, __u16 flags,
                     __be16 port, __be32 src_vni, __be32 vni,
                     __u32 ifindex, __u16 ndm_flags, u32 nhid,
                     bool swdev_notify,
                     struct netlink_ext_ack *extack)
{
        struct vxlan_fdb *f;

        f = vxlan_find_mac(vxlan, mac, src_vni);
        if (f) {
                if (flags & NLM_F_EXCL) {
                        netdev_dbg(vxlan->dev,
                                   "lost race to create %pM\n", mac);
                        return -EEXIST;
                }

                return vxlan_fdb_update_existing(vxlan, ip, state, flags, port,
                                                 vni, ifindex, ndm_flags, f,
                                                 nhid, swdev_notify, extack);
        } else {
                if (!(flags & NLM_F_CREATE))
                        return -ENOENT;

                return vxlan_fdb_update_create(vxlan, mac, ip, state, flags,
                                               port, src_vni, vni, ifindex,
                                               ndm_flags, nhid, swdev_notify,
                                               extack);
        }
}

static void vxlan_fdb_dst_destroy(struct vxlan_dev *vxlan, struct vxlan_fdb *f,
                                  struct vxlan_rdst *rd, bool swdev_notify)
{
        list_del_rcu(&rd->list);
        vxlan_fdb_notify(vxlan, f, rd, RTM_DELNEIGH, swdev_notify, NULL);
        call_rcu(&rd->rcu, vxlan_dst_free);
}

static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
                           union vxlan_addr *ip, __be16 *port, __be32 *src_vni,
                           __be32 *vni, u32 *ifindex, u32 *nhid,
                           struct netlink_ext_ack *extack)
{
        struct net *net = dev_net(vxlan->dev);
        int err;

        if (tb[NDA_NH_ID] &&
            (tb[NDA_DST] || tb[NDA_VNI] || tb[NDA_IFINDEX] || tb[NDA_PORT])) {
                NL_SET_ERR_MSG(extack, "DST, VNI, ifindex and port are mutually exclusive with NH_ID");
                return -EINVAL;
        }

        if (tb[NDA_DST]) {
                err = vxlan_nla_get_addr(ip, tb[NDA_DST]);
                if (err) {
                        NL_SET_ERR_MSG(extack, "Unsupported address family");
                        return err;
                }
        } else {
                union vxlan_addr *remote = &vxlan->default_dst.remote_ip;

                if (remote->sa.sa_family == AF_INET) {
                        ip->sin.sin_addr.s_addr = htonl(INADDR_ANY);
                        ip->sa.sa_family = AF_INET;
#if IS_ENABLED(CONFIG_IPV6)
                } else {
                        ip->sin6.sin6_addr = in6addr_any;
                        ip->sa.sa_family = AF_INET6;
#endif
                }
        }

        if (tb[NDA_PORT]) {
                if (nla_len(tb[NDA_PORT]) != sizeof(__be16)) {
                        NL_SET_ERR_MSG(extack, "Invalid vxlan port");
                        return -EINVAL;
                }
                *port = nla_get_be16(tb[NDA_PORT]);
        } else {
                *port = vxlan->cfg.dst_port;
        }

        if (tb[NDA_VNI]) {
                if (nla_len(tb[NDA_VNI]) != sizeof(u32)) {
                        NL_SET_ERR_MSG(extack, "Invalid vni");
                        return -EINVAL;
                }
                *vni = cpu_to_be32(nla_get_u32(tb[NDA_VNI]));
        } else {
                *vni = vxlan->default_dst.remote_vni;
        }

        if (tb[NDA_SRC_VNI]) {
                if (nla_len(tb[NDA_SRC_VNI]) != sizeof(u32)) {
                        NL_SET_ERR_MSG(extack, "Invalid src vni");
                        return -EINVAL;
                }
                *src_vni = cpu_to_be32(nla_get_u32(tb[NDA_SRC_VNI]));
        } else {
                *src_vni = vxlan->default_dst.remote_vni;
        }

        if (tb[NDA_IFINDEX]) {
                struct net_device *tdev;

                if (nla_len(tb[NDA_IFINDEX]) != sizeof(u32)) {
                        NL_SET_ERR_MSG(extack, "Invalid ifindex");
                        return -EINVAL;
                }
                *ifindex = nla_get_u32(tb[NDA_IFINDEX]);
                tdev = __dev_get_by_index(net, *ifindex);
                if (!tdev) {
                        NL_SET_ERR_MSG(extack, "Device not found");
                        return -EADDRNOTAVAIL;
                }
        } else {
                *ifindex = 0;
        }

        *nhid = nla_get_u32_default(tb[NDA_NH_ID], 0);

        return 0;
}

/* Add static entry (via netlink) */
static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
                         struct net_device *dev,
                         const unsigned char *addr, u16 vid, u16 flags,
                         bool *notified, struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        /* struct net *net = dev_net(vxlan->dev); */
        union vxlan_addr ip;
        __be16 port;
        __be32 src_vni, vni;
        u32 ifindex, nhid;
        int err;

        if (!(ndm->ndm_state & (NUD_PERMANENT|NUD_REACHABLE))) {
                pr_info("RTM_NEWNEIGH with invalid state %#x\n",
                        ndm->ndm_state);
                return -EINVAL;
        }

        if (!tb || (!tb[NDA_DST] && !tb[NDA_NH_ID]))
                return -EINVAL;

        err = vxlan_fdb_parse(tb, vxlan, &ip, &port, &src_vni, &vni, &ifindex,
                              &nhid, extack);
        if (err)
                return err;

        if (vxlan->default_dst.remote_ip.sa.sa_family != ip.sa.sa_family)
                return -EAFNOSUPPORT;

        spin_lock_bh(&vxlan->hash_lock);
        err = vxlan_fdb_update(vxlan, addr, &ip, ndm->ndm_state, flags,
                               port, src_vni, vni, ifindex,
                               ndm->ndm_flags | NTF_VXLAN_ADDED_BY_USER,
                               nhid, true, extack);
        spin_unlock_bh(&vxlan->hash_lock);

        if (!err)
                *notified = true;

        return err;
}

int __vxlan_fdb_delete(struct vxlan_dev *vxlan,
                       const unsigned char *addr, union vxlan_addr ip,
                       __be16 port, __be32 src_vni, __be32 vni,
                       u32 ifindex, bool swdev_notify)
{
        struct vxlan_rdst *rd = NULL;
        struct vxlan_fdb *f;
        int err = -ENOENT;

        f = vxlan_find_mac(vxlan, addr, src_vni);
        if (!f)
                return err;

        if (!vxlan_addr_any(&ip)) {
                rd = vxlan_fdb_find_rdst(f, &ip, port, vni, ifindex);
                if (!rd)
                        goto out;
        }

        /* remove a destination if it's not the only one on the list,
         * otherwise destroy the fdb entry
         */
        if (rd && !list_is_singular(&f->remotes)) {
                vxlan_fdb_dst_destroy(vxlan, f, rd, swdev_notify);
                goto out;
        }

        vxlan_fdb_destroy(vxlan, f, true, swdev_notify);

out:
        return 0;
}

/* Delete entry (via netlink) */
static int vxlan_fdb_delete(struct ndmsg *ndm, struct nlattr *tb[],
                            struct net_device *dev,
                            const unsigned char *addr, u16 vid, bool *notified,
                            struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        union vxlan_addr ip;
        __be32 src_vni, vni;
        u32 ifindex, nhid;
        __be16 port;
        int err;

        err = vxlan_fdb_parse(tb, vxlan, &ip, &port, &src_vni, &vni, &ifindex,
                              &nhid, extack);
        if (err)
                return err;

        spin_lock_bh(&vxlan->hash_lock);
        err = __vxlan_fdb_delete(vxlan, addr, ip, port, src_vni, vni, ifindex,
                                 true);
        spin_unlock_bh(&vxlan->hash_lock);

        if (!err)
                *notified = true;

        return err;
}

/* Dump forwarding table */
static int vxlan_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
                          struct net_device *dev,
                          struct net_device *filter_dev, int *idx)
{
        struct ndo_fdb_dump_context *ctx = (void *)cb->ctx;
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb *f;
        int err = 0;

        rcu_read_lock();
        hlist_for_each_entry_rcu(f, &vxlan->fdb_list, fdb_node) {
                struct vxlan_rdst *rd;

                if (rcu_access_pointer(f->nh)) {
                        if (*idx < ctx->fdb_idx)
                                goto skip_nh;
                        err = vxlan_fdb_info(skb, vxlan, f,
                                             NETLINK_CB(cb->skb).portid,
                                             cb->nlh->nlmsg_seq,
                                             RTM_NEWNEIGH, NLM_F_MULTI, NULL);
                        if (err < 0) {
                                rcu_read_unlock();
                                goto out;
                        }
skip_nh:
                        *idx += 1;
                        continue;
                }

                list_for_each_entry_rcu(rd, &f->remotes, list) {
                        if (*idx < ctx->fdb_idx)
                                goto skip;

                        err = vxlan_fdb_info(skb, vxlan, f,
                                             NETLINK_CB(cb->skb).portid,
                                             cb->nlh->nlmsg_seq,
                                             RTM_NEWNEIGH, NLM_F_MULTI, rd);
                        if (err < 0) {
                                rcu_read_unlock();
                                goto out;
                        }
skip:
                        *idx += 1;
                }
        }
        rcu_read_unlock();
out:
        return err;
}

static int vxlan_fdb_get(struct sk_buff *skb,
                         struct nlattr *tb[],
                         struct net_device *dev,
                         const unsigned char *addr,
                         u16 vid, u32 portid, u32 seq,
                         struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb *f;
        __be32 vni;
        int err;

        if (tb[NDA_VNI])
                vni = cpu_to_be32(nla_get_u32(tb[NDA_VNI]));
        else
                vni = vxlan->default_dst.remote_vni;

        rcu_read_lock();

        f = vxlan_find_mac_rcu(vxlan, addr, vni);
        if (!f) {
                NL_SET_ERR_MSG(extack, "Fdb entry not found");
                err = -ENOENT;
                goto errout;
        }

        err = vxlan_fdb_info(skb, vxlan, f, portid, seq,
                             RTM_NEWNEIGH, 0, first_remote_rcu(f));
errout:
        rcu_read_unlock();
        return err;
}

/* Watch incoming packets to learn mapping between Ethernet address
 * and Tunnel endpoint.
 */
static enum skb_drop_reason vxlan_snoop(struct net_device *dev,
                                        union vxlan_addr *src_ip,
                                        const u8 *src_mac, u32 src_ifindex,
                                        __be32 vni)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb *f;
        u32 ifindex = 0;

        /* Ignore packets from invalid src-address */
        if (!is_valid_ether_addr(src_mac))
                return SKB_DROP_REASON_MAC_INVALID_SOURCE;

#if IS_ENABLED(CONFIG_IPV6)
        if (src_ip->sa.sa_family == AF_INET6 &&
            (ipv6_addr_type(&src_ip->sin6.sin6_addr) & IPV6_ADDR_LINKLOCAL))
                ifindex = src_ifindex;
#endif

        f = vxlan_find_mac_rcu(vxlan, src_mac, vni);
        if (likely(f)) {
                struct vxlan_rdst *rdst = first_remote_rcu(f);
                unsigned long now = jiffies;

                if (READ_ONCE(f->updated) != now)
                        WRITE_ONCE(f->updated, now);

                /* Don't override an fdb with nexthop with a learnt entry */
                if (rcu_access_pointer(f->nh))
                        return SKB_DROP_REASON_VXLAN_ENTRY_EXISTS;

                if (likely(vxlan_addr_equal(&rdst->remote_ip, src_ip) &&
                           rdst->remote_ifindex == ifindex))
                        return SKB_NOT_DROPPED_YET;

                /* Don't migrate static entries, drop packets */
                if (f->state & (NUD_PERMANENT | NUD_NOARP))
                        return SKB_DROP_REASON_VXLAN_ENTRY_EXISTS;

                if (net_ratelimit())
                        netdev_info(dev,
                                    "%pM migrated from %pIS to %pIS\n",
                                    src_mac, &rdst->remote_ip.sa, &src_ip->sa);

                rdst->remote_ip = *src_ip;
                vxlan_fdb_notify(vxlan, f, rdst, RTM_NEWNEIGH, true, NULL);
        } else {
                /* learned new entry */
                spin_lock(&vxlan->hash_lock);

                /* close off race between vxlan_flush and incoming packets */
                if (netif_running(dev))
                        vxlan_fdb_update(vxlan, src_mac, src_ip,
                                         NUD_REACHABLE,
                                         NLM_F_EXCL|NLM_F_CREATE,
                                         vxlan->cfg.dst_port,
                                         vni,
                                         vxlan->default_dst.remote_vni,
                                         ifindex, NTF_SELF, 0, true, NULL);
                spin_unlock(&vxlan->hash_lock);
        }

        return SKB_NOT_DROPPED_YET;
}

static bool __vxlan_sock_release_prep(struct vxlan_sock *vs)
{
        ASSERT_RTNL();

        if (!vs)
                return false;
        if (!refcount_dec_and_test(&vs->refcnt))
                return false;

        hlist_del_rcu(&vs->hlist);
        udp_tunnel_notify_del_rx_port(vs->sock,
                                      (vs->flags & VXLAN_F_GPE) ?
                                      UDP_TUNNEL_TYPE_VXLAN_GPE :
                                      UDP_TUNNEL_TYPE_VXLAN);

        return true;
}

static void vxlan_sock_release(struct vxlan_dev *vxlan)
{
        struct vxlan_sock *sock4 = rtnl_dereference(vxlan->vn4_sock);
#if IS_ENABLED(CONFIG_IPV6)
        struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock);

        RCU_INIT_POINTER(vxlan->vn6_sock, NULL);
#endif

        RCU_INIT_POINTER(vxlan->vn4_sock, NULL);
        synchronize_net();

        if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
                vxlan_vs_del_vnigrp(vxlan);
        else
                vxlan_vs_del_dev(vxlan);

        if (__vxlan_sock_release_prep(sock4)) {
                udp_tunnel_sock_release(sock4->sock);
                kfree(sock4);
        }

#if IS_ENABLED(CONFIG_IPV6)
        if (__vxlan_sock_release_prep(sock6)) {
                udp_tunnel_sock_release(sock6->sock);
                kfree(sock6);
        }
#endif
}

static enum skb_drop_reason vxlan_remcsum(struct sk_buff *skb, u32 vxflags)
{
        const struct vxlanhdr *vh = vxlan_hdr(skb);
        enum skb_drop_reason reason;
        size_t start, offset;

        if (!(vh->vx_flags & VXLAN_HF_RCO) || skb->remcsum_offload)
                return SKB_NOT_DROPPED_YET;

        start = vxlan_rco_start(vh->vx_vni);
        offset = start + vxlan_rco_offset(vh->vx_vni);

        reason = pskb_may_pull_reason(skb, offset + sizeof(u16));
        if (reason)
                return reason;

        skb_remcsum_process(skb, (void *)(vxlan_hdr(skb) + 1), start, offset,
                            !!(vxflags & VXLAN_F_REMCSUM_NOPARTIAL));
        return SKB_NOT_DROPPED_YET;
}

static void vxlan_parse_gbp_hdr(struct sk_buff *skb, u32 vxflags,
                                struct vxlan_metadata *md)
{
        const struct vxlanhdr *vh = vxlan_hdr(skb);
        const struct vxlanhdr_gbp *gbp;
        struct metadata_dst *tun_dst;

        gbp = (const struct vxlanhdr_gbp *)vh;

        if (!(vh->vx_flags & VXLAN_HF_GBP))
                return;

        md->gbp = ntohs(gbp->policy_id);

        tun_dst = (struct metadata_dst *)skb_dst(skb);
        if (tun_dst) {
                __set_bit(IP_TUNNEL_VXLAN_OPT_BIT,
                          tun_dst->u.tun_info.key.tun_flags);
                tun_dst->u.tun_info.options_len = sizeof(*md);
        }
        if (gbp->dont_learn)
                md->gbp |= VXLAN_GBP_DONT_LEARN;

        if (gbp->policy_applied)
                md->gbp |= VXLAN_GBP_POLICY_APPLIED;

        /* In flow-based mode, GBP is carried in dst_metadata */
        if (!(vxflags & VXLAN_F_COLLECT_METADATA))
                skb->mark = md->gbp;
}

static enum skb_drop_reason vxlan_set_mac(struct vxlan_dev *vxlan,
                                          struct vxlan_sock *vs,
                                          struct sk_buff *skb, __be32 vni)
{
        union vxlan_addr saddr;
        u32 ifindex = skb->dev->ifindex;

        skb_reset_mac_header(skb);
        skb->protocol = eth_type_trans(skb, vxlan->dev);
        skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);

        /* Ignore packet loops (and multicast echo) */
        if (ether_addr_equal(eth_hdr(skb)->h_source, vxlan->dev->dev_addr))
                return SKB_DROP_REASON_LOCAL_MAC;

        /* Get address from the outer IP header */
        if (vxlan_get_sk_family(vs) == AF_INET) {
                saddr.sin.sin_addr.s_addr = ip_hdr(skb)->saddr;
                saddr.sa.sa_family = AF_INET;
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                saddr.sin6.sin6_addr = ipv6_hdr(skb)->saddr;
                saddr.sa.sa_family = AF_INET6;
#endif
        }

        if (!(vxlan->cfg.flags & VXLAN_F_LEARN))
                return SKB_NOT_DROPPED_YET;

        return vxlan_snoop(skb->dev, &saddr, eth_hdr(skb)->h_source,
                           ifindex, vni);
}

static bool vxlan_ecn_decapsulate(struct vxlan_sock *vs, void *oiph,
                                  struct sk_buff *skb)
{
        int err = 0;

        if (vxlan_get_sk_family(vs) == AF_INET)
                err = IP_ECN_decapsulate(oiph, skb);
#if IS_ENABLED(CONFIG_IPV6)
        else
                err = IP6_ECN_decapsulate(oiph, skb);
#endif

        if (unlikely(err) && log_ecn_error) {
                if (vxlan_get_sk_family(vs) == AF_INET)
                        net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
                                             &((struct iphdr *)oiph)->saddr,
                                             ((struct iphdr *)oiph)->tos);
                else
                        net_info_ratelimited("non-ECT from %pI6\n",
                                             &((struct ipv6hdr *)oiph)->saddr);
        }
        return err <= 1;
}

static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
{
        struct vxlan_vni_node *vninode = NULL;
        const struct vxlanhdr *vh;
        struct vxlan_dev *vxlan;
        struct vxlan_sock *vs;
        struct vxlan_metadata _md;
        struct vxlan_metadata *md = &_md;
        __be16 protocol = htons(ETH_P_TEB);
        enum skb_drop_reason reason;
        bool raw_proto = false;
        void *oiph;
        __be32 vni = 0;
        int nh;

        /* Need UDP and VXLAN header to be present */
        reason = pskb_may_pull_reason(skb, VXLAN_HLEN);
        if (reason)
                goto drop;

        vh = vxlan_hdr(skb);
        /* VNI flag always required to be set */
        if (!(vh->vx_flags & VXLAN_HF_VNI)) {
                netdev_dbg(skb->dev, "invalid vxlan flags=%#x vni=%#x\n",
                           ntohl(vh->vx_flags), ntohl(vh->vx_vni));
                reason = SKB_DROP_REASON_VXLAN_INVALID_HDR;
                /* Return non vxlan pkt */
                goto drop;
        }

        vs = rcu_dereference_sk_user_data(sk);
        if (!vs)
                goto drop;

        vni = vxlan_vni(vh->vx_vni);

        vxlan = vxlan_vs_find_vni(vs, skb->dev->ifindex, vni, &vninode);
        if (!vxlan) {
                reason = SKB_DROP_REASON_VXLAN_VNI_NOT_FOUND;
                goto drop;
        }

        if (vh->vx_flags & vxlan->cfg.reserved_bits.vx_flags ||
            vh->vx_vni & vxlan->cfg.reserved_bits.vx_vni) {
                /* If the header uses bits besides those enabled by the
                 * netdevice configuration, treat this as a malformed packet.
                 * This behavior diverges from VXLAN RFC (RFC7348) which
                 * stipulates that bits in reserved in reserved fields are to be
                 * ignored. The approach here maintains compatibility with
                 * previous stack code, and also is more robust and provides a
                 * little more security in adding extensions to VXLAN.
                 */
                reason = SKB_DROP_REASON_VXLAN_INVALID_HDR;
                DEV_STATS_INC(vxlan->dev, rx_frame_errors);
                DEV_STATS_INC(vxlan->dev, rx_errors);
                vxlan_vnifilter_count(vxlan, vni, vninode,
                                      VXLAN_VNI_STATS_RX_ERRORS, 0);
                goto drop;
        }

        if (vxlan->cfg.flags & VXLAN_F_GPE) {
                if (!vxlan_parse_gpe_proto(vh, &protocol))
                        goto drop;
                raw_proto = true;
        }

        if (__iptunnel_pull_header(skb, VXLAN_HLEN, protocol, raw_proto,
                                   !net_eq(vxlan->net, dev_net(vxlan->dev)))) {
                reason = SKB_DROP_REASON_NOMEM;
                goto drop;
        }

        if (vxlan->cfg.flags & VXLAN_F_REMCSUM_RX) {
                reason = vxlan_remcsum(skb, vxlan->cfg.flags);
                if (unlikely(reason))
                        goto drop;
        }

        if (vxlan_collect_metadata(vs)) {
                IP_TUNNEL_DECLARE_FLAGS(flags) = { };
                struct metadata_dst *tun_dst;

                __set_bit(IP_TUNNEL_KEY_BIT, flags);
                tun_dst = udp_tun_rx_dst(skb, vxlan_get_sk_family(vs), flags,
                                         key32_to_tunnel_id(vni), sizeof(*md));

                if (!tun_dst) {
                        reason = SKB_DROP_REASON_NOMEM;
                        goto drop;
                }

                md = ip_tunnel_info_opts(&tun_dst->u.tun_info);

                skb_dst_set(skb, (struct dst_entry *)tun_dst);
        } else {
                memset(md, 0, sizeof(*md));
        }

        if (vxlan->cfg.flags & VXLAN_F_GBP)
                vxlan_parse_gbp_hdr(skb, vxlan->cfg.flags, md);
        /* Note that GBP and GPE can never be active together. This is
         * ensured in vxlan_dev_configure.
         */

        if (!raw_proto) {
                reason = vxlan_set_mac(vxlan, vs, skb, vni);
                if (reason)
                        goto drop;
        } else {
                skb_reset_mac_header(skb);
                skb->dev = vxlan->dev;
                skb->pkt_type = PACKET_HOST;
        }

        /* Save offset of outer header relative to skb->head,
         * because we are going to reset the network header to the inner header
         * and might change skb->head.
         */
        nh = skb_network_header(skb) - skb->head;

        skb_reset_network_header(skb);

        reason = pskb_inet_may_pull_reason(skb);
        if (reason) {
                DEV_STATS_INC(vxlan->dev, rx_length_errors);
                DEV_STATS_INC(vxlan->dev, rx_errors);
                vxlan_vnifilter_count(vxlan, vni, vninode,
                                      VXLAN_VNI_STATS_RX_ERRORS, 0);
                goto drop;
        }

        /* Get the outer header. */
        oiph = skb->head + nh;

        if (!vxlan_ecn_decapsulate(vs, oiph, skb)) {
                reason = SKB_DROP_REASON_IP_TUNNEL_ECN;
                DEV_STATS_INC(vxlan->dev, rx_frame_errors);
                DEV_STATS_INC(vxlan->dev, rx_errors);
                vxlan_vnifilter_count(vxlan, vni, vninode,
                                      VXLAN_VNI_STATS_RX_ERRORS, 0);
                goto drop;
        }

        rcu_read_lock();

        if (unlikely(!(vxlan->dev->flags & IFF_UP))) {
                rcu_read_unlock();
                dev_dstats_rx_dropped(vxlan->dev);
                vxlan_vnifilter_count(vxlan, vni, vninode,
                                      VXLAN_VNI_STATS_RX_DROPS, 0);
                reason = SKB_DROP_REASON_DEV_READY;
                goto drop;
        }

        dev_dstats_rx_add(vxlan->dev, skb->len);
        vxlan_vnifilter_count(vxlan, vni, vninode, VXLAN_VNI_STATS_RX, skb->len);
        gro_cells_receive(&vxlan->gro_cells, skb);

        rcu_read_unlock();

        return 0;

drop:
        reason = reason ?: SKB_DROP_REASON_NOT_SPECIFIED;
        /* Consume bad packet */
        kfree_skb_reason(skb, reason);
        return 0;
}

static int vxlan_err_lookup(struct sock *sk, struct sk_buff *skb)
{
        struct vxlan_dev *vxlan;
        struct vxlan_sock *vs;
        struct vxlanhdr *hdr;
        __be32 vni;

        if (!pskb_may_pull(skb, skb_transport_offset(skb) + VXLAN_HLEN))
                return -EINVAL;

        hdr = vxlan_hdr(skb);

        if (!(hdr->vx_flags & VXLAN_HF_VNI))
                return -EINVAL;

        vs = rcu_dereference_sk_user_data(sk);
        if (!vs)
                return -ENOENT;

        vni = vxlan_vni(hdr->vx_vni);
        vxlan = vxlan_vs_find_vni(vs, skb->dev->ifindex, vni, NULL);
        if (!vxlan)
                return -ENOENT;

        return 0;
}

static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct arphdr *parp;
        u8 *arpptr, *sha;
        __be32 sip, tip;
        struct neighbour *n;

        if (dev->flags & IFF_NOARP)
                goto out;

        if (!pskb_may_pull(skb, arp_hdr_len(dev))) {
                dev_dstats_tx_dropped(dev);
                vxlan_vnifilter_count(vxlan, vni, NULL,
                                      VXLAN_VNI_STATS_TX_DROPS, 0);
                goto out;
        }
        parp = arp_hdr(skb);

        if ((parp->ar_hrd != htons(ARPHRD_ETHER) &&
             parp->ar_hrd != htons(ARPHRD_IEEE802)) ||
            parp->ar_pro != htons(ETH_P_IP) ||
            parp->ar_op != htons(ARPOP_REQUEST) ||
            parp->ar_hln != dev->addr_len ||
            parp->ar_pln != 4)
                goto out;
        arpptr = (u8 *)parp + sizeof(struct arphdr);
        sha = arpptr;
        arpptr += dev->addr_len;        /* sha */
        memcpy(&sip, arpptr, sizeof(sip));
        arpptr += sizeof(sip);
        arpptr += dev->addr_len;        /* tha */
        memcpy(&tip, arpptr, sizeof(tip));

        if (ipv4_is_loopback(tip) ||
            ipv4_is_multicast(tip))
                goto out;

        n = neigh_lookup(&arp_tbl, &tip, dev);

        if (n) {
                struct vxlan_rdst *rdst = NULL;
                struct vxlan_fdb *f;
                struct sk_buff        *reply;

                if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
                        neigh_release(n);
                        goto out;
                }

                rcu_read_lock();
                f = vxlan_find_mac_tx(vxlan, n->ha, vni);
                if (f)
                        rdst = first_remote_rcu(f);
                if (rdst && vxlan_addr_any(&rdst->remote_ip)) {
                        /* bridge-local neighbor */
                        neigh_release(n);
                        rcu_read_unlock();
                        goto out;
                }
                rcu_read_unlock();

                reply = arp_create(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
                                n->ha, sha);

                neigh_release(n);

                if (reply == NULL)
                        goto out;

                skb_reset_mac_header(reply);
                __skb_pull(reply, skb_network_offset(reply));
                reply->ip_summed = CHECKSUM_UNNECESSARY;
                reply->pkt_type = PACKET_HOST;

                if (netif_rx(reply) == NET_RX_DROP) {
                        dev_dstats_rx_dropped(dev);
                        vxlan_vnifilter_count(vxlan, vni, NULL,
                                              VXLAN_VNI_STATS_RX_DROPS, 0);
                }

        } else if (vxlan->cfg.flags & VXLAN_F_L3MISS) {
                union vxlan_addr ipa = {
                        .sin.sin_addr.s_addr = tip,
                        .sin.sin_family = AF_INET,
                };

                vxlan_ip_miss(dev, &ipa);
        }
out:
        consume_skb(skb);
        return NETDEV_TX_OK;
}

#if IS_ENABLED(CONFIG_IPV6)
static struct sk_buff *vxlan_na_create(struct sk_buff *request,
        struct neighbour *n, bool isrouter)
{
        struct net_device *dev = request->dev;
        struct sk_buff *reply;
        struct nd_msg *ns, *na;
        struct ipv6hdr *pip6;
        u8 *daddr;
        int na_olen = 8; /* opt hdr + ETH_ALEN for target */
        int ns_olen;
        int i, len;

        if (dev == NULL || !pskb_may_pull(request, request->len))
                return NULL;

        len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) +
                sizeof(*na) + na_olen + dev->needed_tailroom;
        reply = alloc_skb(len, GFP_ATOMIC);
        if (reply == NULL)
                return NULL;

        reply->protocol = htons(ETH_P_IPV6);
        reply->dev = dev;
        skb_reserve(reply, LL_RESERVED_SPACE(request->dev));
        skb_push(reply, sizeof(struct ethhdr));
        skb_reset_mac_header(reply);

        ns = (struct nd_msg *)(ipv6_hdr(request) + 1);

        daddr = eth_hdr(request)->h_source;
        ns_olen = request->len - skb_network_offset(request) -
                sizeof(struct ipv6hdr) - sizeof(*ns);
        for (i = 0; i < ns_olen-1; i += (ns->opt[i+1]<<3)) {
                if (!ns->opt[i + 1]) {
                        kfree_skb(reply);
                        return NULL;
                }
                if (ns->opt[i] == ND_OPT_SOURCE_LL_ADDR) {
                        daddr = ns->opt + i + sizeof(struct nd_opt_hdr);
                        break;
                }
        }

        /* Ethernet header */
        ether_addr_copy(eth_hdr(reply)->h_dest, daddr);
        ether_addr_copy(eth_hdr(reply)->h_source, n->ha);
        eth_hdr(reply)->h_proto = htons(ETH_P_IPV6);
        reply->protocol = htons(ETH_P_IPV6);

        skb_pull(reply, sizeof(struct ethhdr));
        skb_reset_network_header(reply);
        skb_put(reply, sizeof(struct ipv6hdr));

        /* IPv6 header */

        pip6 = ipv6_hdr(reply);
        memset(pip6, 0, sizeof(struct ipv6hdr));
        pip6->version = 6;
        pip6->priority = ipv6_hdr(request)->priority;
        pip6->nexthdr = IPPROTO_ICMPV6;
        pip6->hop_limit = 255;
        pip6->daddr = ipv6_hdr(request)->saddr;
        pip6->saddr = *(struct in6_addr *)n->primary_key;

        skb_pull(reply, sizeof(struct ipv6hdr));
        skb_reset_transport_header(reply);

        /* Neighbor Advertisement */
        na = skb_put_zero(reply, sizeof(*na) + na_olen);
        na->icmph.icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
        na->icmph.icmp6_router = isrouter;
        na->icmph.icmp6_override = 1;
        na->icmph.icmp6_solicited = 1;
        na->target = ns->target;
        ether_addr_copy(&na->opt[2], n->ha);
        na->opt[0] = ND_OPT_TARGET_LL_ADDR;
        na->opt[1] = na_olen >> 3;

        na->icmph.icmp6_cksum = csum_ipv6_magic(&pip6->saddr,
                &pip6->daddr, sizeof(*na)+na_olen, IPPROTO_ICMPV6,
                csum_partial(na, sizeof(*na)+na_olen, 0));

        pip6->payload_len = htons(sizeof(*na)+na_olen);

        skb_push(reply, sizeof(struct ipv6hdr));

        reply->ip_summed = CHECKSUM_UNNECESSARY;

        return reply;
}

static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        const struct in6_addr *daddr;
        const struct ipv6hdr *iphdr;
        struct inet6_dev *in6_dev;
        struct neighbour *n;
        struct nd_msg *msg;

        rcu_read_lock();
        in6_dev = __in6_dev_get(dev);
        if (!in6_dev)
                goto out;

        iphdr = ipv6_hdr(skb);
        daddr = &iphdr->daddr;
        msg = (struct nd_msg *)(iphdr + 1);

        if (ipv6_addr_loopback(daddr) ||
            ipv6_addr_is_multicast(&msg->target))
                goto out;

        n = neigh_lookup(ipv6_stub->nd_tbl, &msg->target, dev);

        if (n) {
                struct vxlan_rdst *rdst = NULL;
                struct vxlan_fdb *f;
                struct sk_buff *reply;

                if (!(READ_ONCE(n->nud_state) & NUD_CONNECTED)) {
                        neigh_release(n);
                        goto out;
                }

                f = vxlan_find_mac_tx(vxlan, n->ha, vni);
                if (f)
                        rdst = first_remote_rcu(f);
                if (rdst && vxlan_addr_any(&rdst->remote_ip)) {
                        /* bridge-local neighbor */
                        neigh_release(n);
                        goto out;
                }

                reply = vxlan_na_create(skb, n,
                                        !!(f ? f->flags & NTF_ROUTER : 0));

                neigh_release(n);

                if (reply == NULL)
                        goto out;

                if (netif_rx(reply) == NET_RX_DROP) {
                        dev_dstats_rx_dropped(dev);
                        vxlan_vnifilter_count(vxlan, vni, NULL,
                                              VXLAN_VNI_STATS_RX_DROPS, 0);
                }
        } else if (vxlan->cfg.flags & VXLAN_F_L3MISS) {
                union vxlan_addr ipa = {
                        .sin6.sin6_addr = msg->target,
                        .sin6.sin6_family = AF_INET6,
                };

                vxlan_ip_miss(dev, &ipa);
        }

out:
        rcu_read_unlock();
        consume_skb(skb);
        return NETDEV_TX_OK;
}
#endif

static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct neighbour *n;

        if (is_multicast_ether_addr(eth_hdr(skb)->h_dest))
                return false;

        n = NULL;
        switch (ntohs(eth_hdr(skb)->h_proto)) {
        case ETH_P_IP:
        {
                struct iphdr *pip;

                if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                        return false;
                pip = ip_hdr(skb);
                n = neigh_lookup(&arp_tbl, &pip->daddr, dev);
                if (!n && (vxlan->cfg.flags & VXLAN_F_L3MISS)) {
                        union vxlan_addr ipa = {
                                .sin.sin_addr.s_addr = pip->daddr,
                                .sin.sin_family = AF_INET,
                        };

                        vxlan_ip_miss(dev, &ipa);
                        return false;
                }

                break;
        }
#if IS_ENABLED(CONFIG_IPV6)
        case ETH_P_IPV6:
        {
                struct ipv6hdr *pip6;

                if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                        return false;
                pip6 = ipv6_hdr(skb);
                n = neigh_lookup(ipv6_stub->nd_tbl, &pip6->daddr, dev);
                if (!n && (vxlan->cfg.flags & VXLAN_F_L3MISS)) {
                        union vxlan_addr ipa = {
                                .sin6.sin6_addr = pip6->daddr,
                                .sin6.sin6_family = AF_INET6,
                        };

                        vxlan_ip_miss(dev, &ipa);
                        return false;
                }

                break;
        }
#endif
        default:
                return false;
        }

        if (n) {
                bool diff;

                diff = !ether_addr_equal(eth_hdr(skb)->h_dest, n->ha);
                if (diff) {
                        memcpy(eth_hdr(skb)->h_source, eth_hdr(skb)->h_dest,
                                dev->addr_len);
                        memcpy(eth_hdr(skb)->h_dest, n->ha, dev->addr_len);
                }
                neigh_release(n);
                return diff;
        }

        return false;
}

static int vxlan_build_gpe_hdr(struct vxlanhdr *vxh, __be16 protocol)
{
        struct vxlanhdr_gpe *gpe = (struct vxlanhdr_gpe *)vxh;

        gpe->np_applied = 1;
        gpe->next_protocol = tun_p_from_eth_p(protocol);
        if (!gpe->next_protocol)
                return -EPFNOSUPPORT;
        return 0;
}

static int vxlan_build_skb(struct sk_buff *skb, struct dst_entry *dst,
                           int iphdr_len, __be32 vni,
                           struct vxlan_metadata *md, u32 vxflags,
                           bool udp_sum)
{
        struct vxlanhdr *vxh;
        int min_headroom;
        int err;
        int type = udp_sum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
        __be16 inner_protocol = htons(ETH_P_TEB);

        if ((vxflags & VXLAN_F_REMCSUM_TX) &&
            skb->ip_summed == CHECKSUM_PARTIAL) {
                int csum_start = skb_checksum_start_offset(skb);

                if (csum_start <= VXLAN_MAX_REMCSUM_START &&
                    !(csum_start & VXLAN_RCO_SHIFT_MASK) &&
                    (skb->csum_offset == offsetof(struct udphdr, check) ||
                     skb->csum_offset == offsetof(struct tcphdr, check)))
                        type |= SKB_GSO_TUNNEL_REMCSUM;
        }

        min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len
                        + VXLAN_HLEN + iphdr_len;

        /* Need space for new headers (invalidates iph ptr) */
        err = skb_cow_head(skb, min_headroom);
        if (unlikely(err))
                return err;

        err = iptunnel_handle_offloads(skb, type);
        if (err)
                return err;

        vxh = __skb_push(skb, sizeof(*vxh));
        vxh->vx_flags = VXLAN_HF_VNI;
        vxh->vx_vni = vxlan_vni_field(vni);

        if (type & SKB_GSO_TUNNEL_REMCSUM) {
                unsigned int start;

                start = skb_checksum_start_offset(skb) - sizeof(struct vxlanhdr);
                vxh->vx_vni |= vxlan_compute_rco(start, skb->csum_offset);
                vxh->vx_flags |= VXLAN_HF_RCO;

                if (!skb_is_gso(skb)) {
                        skb->ip_summed = CHECKSUM_NONE;
                        skb->encapsulation = 0;
                }
        }

        if (vxflags & VXLAN_F_GBP)
                vxlan_build_gbp_hdr(vxh, md);
        if (vxflags & VXLAN_F_GPE) {
                err = vxlan_build_gpe_hdr(vxh, skb->protocol);
                if (err < 0)
                        return err;
                inner_protocol = skb->protocol;
        }

        skb_set_inner_protocol(skb, inner_protocol);
        return 0;
}

/* Bypass encapsulation if the destination is local */
static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
                               struct vxlan_dev *dst_vxlan, __be32 vni,
                               bool snoop)
{
        union vxlan_addr loopback;
        union vxlan_addr *remote_ip = &dst_vxlan->default_dst.remote_ip;
        unsigned int len = skb->len;
        struct net_device *dev;

        skb->pkt_type = PACKET_HOST;
        skb->encapsulation = 0;
        skb->dev = dst_vxlan->dev;
        __skb_pull(skb, skb_network_offset(skb));

        if (remote_ip->sa.sa_family == AF_INET) {
                loopback.sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
                loopback.sa.sa_family =  AF_INET;
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                loopback.sin6.sin6_addr = in6addr_loopback;
                loopback.sa.sa_family =  AF_INET6;
#endif
        }

        rcu_read_lock();
        dev = skb->dev;
        if (unlikely(!(dev->flags & IFF_UP))) {
                kfree_skb_reason(skb, SKB_DROP_REASON_DEV_READY);
                goto drop;
        }

        if ((dst_vxlan->cfg.flags & VXLAN_F_LEARN) && snoop)
                vxlan_snoop(dev, &loopback, eth_hdr(skb)->h_source, 0, vni);

        dev_dstats_tx_add(src_vxlan->dev, len);
        vxlan_vnifilter_count(src_vxlan, vni, NULL, VXLAN_VNI_STATS_TX, len);

        if (__netif_rx(skb) == NET_RX_SUCCESS) {
                dev_dstats_rx_add(dst_vxlan->dev, len);
                vxlan_vnifilter_count(dst_vxlan, vni, NULL, VXLAN_VNI_STATS_RX,
                                      len);
        } else {
drop:
                dev_dstats_rx_dropped(dev);
                vxlan_vnifilter_count(dst_vxlan, vni, NULL,
                                      VXLAN_VNI_STATS_RX_DROPS, 0);
        }
        rcu_read_unlock();
}

static int encap_bypass_if_local(struct sk_buff *skb, struct net_device *dev,
                                 struct vxlan_dev *vxlan,
                                 int addr_family,
                                 __be16 dst_port, int dst_ifindex, __be32 vni,
                                 struct dst_entry *dst,
                                 u32 rt_flags)
{
#if IS_ENABLED(CONFIG_IPV6)
        /* IPv6 rt-flags are checked against RTF_LOCAL, but the value of
         * RTF_LOCAL is equal to RTCF_LOCAL. So to keep code simple
         * we can use RTCF_LOCAL which works for ipv4 and ipv6 route entry.
         */
        BUILD_BUG_ON(RTCF_LOCAL != RTF_LOCAL);
#endif
        /* Bypass encapsulation if the destination is local */
        if (rt_flags & RTCF_LOCAL &&
            !(rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
            vxlan->cfg.flags & VXLAN_F_LOCALBYPASS) {
                struct vxlan_dev *dst_vxlan;

                dst_release(dst);
                dst_vxlan = vxlan_find_vni(vxlan->net, dst_ifindex, vni,
                                           addr_family, dst_port,
                                           vxlan->cfg.flags);
                if (!dst_vxlan) {
                        DEV_STATS_INC(dev, tx_errors);
                        vxlan_vnifilter_count(vxlan, vni, NULL,
                                              VXLAN_VNI_STATS_TX_ERRORS, 0);
                        kfree_skb_reason(skb, SKB_DROP_REASON_VXLAN_VNI_NOT_FOUND);

                        return -ENOENT;
                }
                vxlan_encap_bypass(skb, vxlan, dst_vxlan, vni, true);
                return 1;
        }

        return 0;
}

void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
                    __be32 default_vni, struct vxlan_rdst *rdst, bool did_rsc)
{
        struct dst_cache *dst_cache;
        struct ip_tunnel_info *info;
        struct ip_tunnel_key *pkey;
        struct ip_tunnel_key key;
        struct vxlan_dev *vxlan = netdev_priv(dev);
        const struct iphdr *old_iph;
        struct vxlan_metadata _md;
        struct vxlan_metadata *md = &_md;
        unsigned int pkt_len = skb->len;
        __be16 src_port = 0, dst_port;
        struct dst_entry *ndst = NULL;
        int addr_family;
        __u8 tos, ttl;
        int ifindex;
        int err;
        u32 flags = vxlan->cfg.flags;
        bool use_cache;
        bool udp_sum = false;
        bool xnet = !net_eq(vxlan->net, dev_net(vxlan->dev));
        enum skb_drop_reason reason;
        bool no_eth_encap;
        __be32 vni = 0;

        no_eth_encap = flags & VXLAN_F_GPE && skb->protocol != htons(ETH_P_TEB);
        reason = skb_vlan_inet_prepare(skb, no_eth_encap);
        if (reason)
                goto drop;

        reason = SKB_DROP_REASON_NOT_SPECIFIED;
        old_iph = ip_hdr(skb);

        info = skb_tunnel_info(skb);
        use_cache = ip_tunnel_dst_cache_usable(skb, info);

        if (rdst) {
                memset(&key, 0, sizeof(key));
                pkey = &key;

                if (vxlan_addr_any(&rdst->remote_ip)) {
                        if (did_rsc) {
                                /* short-circuited back to local bridge */
                                vxlan_encap_bypass(skb, vxlan, vxlan,
                                                   default_vni, true);
                                return;
                        }
                        goto drop;
                }

                addr_family = vxlan->cfg.saddr.sa.sa_family;
                dst_port = rdst->remote_port ? rdst->remote_port : vxlan->cfg.dst_port;
                vni = (rdst->remote_vni) ? : default_vni;
                ifindex = rdst->remote_ifindex;

                if (addr_family == AF_INET) {
                        key.u.ipv4.src = vxlan->cfg.saddr.sin.sin_addr.s_addr;
                        key.u.ipv4.dst = rdst->remote_ip.sin.sin_addr.s_addr;
                } else {
                        key.u.ipv6.src = vxlan->cfg.saddr.sin6.sin6_addr;
                        key.u.ipv6.dst = rdst->remote_ip.sin6.sin6_addr;
                }

                dst_cache = &rdst->dst_cache;
                md->gbp = skb->mark;
                if (flags & VXLAN_F_TTL_INHERIT) {
                        ttl = ip_tunnel_get_ttl(old_iph, skb);
                } else {
                        ttl = vxlan->cfg.ttl;
                        if (!ttl && vxlan_addr_multicast(&rdst->remote_ip))
                                ttl = 1;
                }
                tos = vxlan->cfg.tos;
                if (tos == 1)
                        tos = ip_tunnel_get_dsfield(old_iph, skb);
                if (tos && !info)
                        use_cache = false;

                if (addr_family == AF_INET)
                        udp_sum = !(flags & VXLAN_F_UDP_ZERO_CSUM_TX);
                else
                        udp_sum = !(flags & VXLAN_F_UDP_ZERO_CSUM6_TX);
#if IS_ENABLED(CONFIG_IPV6)
                switch (vxlan->cfg.label_policy) {
                case VXLAN_LABEL_FIXED:
                        key.label = vxlan->cfg.label;
                        break;
                case VXLAN_LABEL_INHERIT:
                        key.label = ip_tunnel_get_flowlabel(old_iph, skb);
                        break;
                default:
                        DEBUG_NET_WARN_ON_ONCE(1);
                        goto drop;
                }
#endif
        } else {
                if (!info) {
                        WARN_ONCE(1, "%s: Missing encapsulation instructions\n",
                                  dev->name);
                        goto drop;
                }
                pkey = &info->key;
                addr_family = ip_tunnel_info_af(info);
                dst_port = info->key.tp_dst ? : vxlan->cfg.dst_port;
                vni = tunnel_id_to_key32(info->key.tun_id);
                ifindex = 0;
                dst_cache = &info->dst_cache;
                if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) {
                        if (info->options_len < sizeof(*md))
                                goto drop;
                        md = ip_tunnel_info_opts(info);
                }
                ttl = info->key.ttl;
                tos = info->key.tos;
                udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
        }
        src_port = udp_flow_src_port(dev_net(dev), skb, vxlan->cfg.port_min,
                                     vxlan->cfg.port_max, true);

        rcu_read_lock();
        if (addr_family == AF_INET) {
                struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock);
                u16 ipcb_flags = 0;
                struct rtable *rt;
                __be16 df = 0;
                __be32 saddr;

                if (!ifindex)
                        ifindex = sock4->sock->sk->sk_bound_dev_if;

                rt = udp_tunnel_dst_lookup(skb, dev, vxlan->net, ifindex,
                                           &saddr, pkey, src_port, dst_port,
                                           tos, use_cache ? dst_cache : NULL);
                if (IS_ERR(rt)) {
                        err = PTR_ERR(rt);
                        reason = SKB_DROP_REASON_IP_OUTNOROUTES;
                        goto tx_error;
                }

                if (flags & VXLAN_F_MC_ROUTE)
                        ipcb_flags |= IPSKB_MCROUTE;

                if (!info) {
                        /* Bypass encapsulation if the destination is local */
                        err = encap_bypass_if_local(skb, dev, vxlan, AF_INET,
                                                    dst_port, ifindex, vni,
                                                    &rt->dst, rt->rt_flags);
                        if (err)
                                goto out_unlock;

                        if (vxlan->cfg.df == VXLAN_DF_SET) {
                                df = htons(IP_DF);
                        } else if (vxlan->cfg.df == VXLAN_DF_INHERIT) {
                                struct ethhdr *eth = eth_hdr(skb);

                                if (ntohs(eth->h_proto) == ETH_P_IPV6 ||
                                    (ntohs(eth->h_proto) == ETH_P_IP &&
                                     old_iph->frag_off & htons(IP_DF)))
                                        df = htons(IP_DF);
                        }
                } else if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT,
                                    info->key.tun_flags)) {
                        df = htons(IP_DF);
                }

                ndst = &rt->dst;
                err = skb_tunnel_check_pmtu(skb, ndst, vxlan_headroom(flags & VXLAN_F_GPE),
                                            netif_is_any_bridge_port(dev));
                if (err < 0) {
                        goto tx_error;
                } else if (err) {
                        if (info) {
                                struct ip_tunnel_info *unclone;

                                unclone = skb_tunnel_info_unclone(skb);
                                if (unlikely(!unclone))
                                        goto tx_error;

                                unclone->key.u.ipv4.src = pkey->u.ipv4.dst;
                                unclone->key.u.ipv4.dst = saddr;
                        }
                        vxlan_encap_bypass(skb, vxlan, vxlan, vni, false);
                        dst_release(ndst);
                        goto out_unlock;
                }

                tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
                ttl = ttl ? : ip4_dst_hoplimit(&rt->dst);
                err = vxlan_build_skb(skb, ndst, sizeof(struct iphdr),
                                      vni, md, flags, udp_sum);
                if (err < 0) {
                        reason = SKB_DROP_REASON_NOMEM;
                        goto tx_error;
                }

                udp_tunnel_xmit_skb(rt, sock4->sock->sk, skb, saddr,
                                    pkey->u.ipv4.dst, tos, ttl, df,
                                    src_port, dst_port, xnet, !udp_sum,
                                    ipcb_flags);
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                struct vxlan_sock *sock6 = rcu_dereference(vxlan->vn6_sock);
                struct in6_addr saddr;
                u16 ip6cb_flags = 0;

                if (!ifindex)
                        ifindex = sock6->sock->sk->sk_bound_dev_if;

                ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sock,
                                              ifindex, &saddr, pkey,
                                              src_port, dst_port, tos,
                                              use_cache ? dst_cache : NULL);
                if (IS_ERR(ndst)) {
                        err = PTR_ERR(ndst);
                        ndst = NULL;
                        reason = SKB_DROP_REASON_IP_OUTNOROUTES;
                        goto tx_error;
                }

                if (flags & VXLAN_F_MC_ROUTE)
                        ip6cb_flags |= IP6SKB_MCROUTE;

                if (!info) {
                        u32 rt6i_flags = dst_rt6_info(ndst)->rt6i_flags;

                        err = encap_bypass_if_local(skb, dev, vxlan, AF_INET6,
                                                    dst_port, ifindex, vni,
                                                    ndst, rt6i_flags);
                        if (err)
                                goto out_unlock;
                }

                err = skb_tunnel_check_pmtu(skb, ndst,
                                            vxlan_headroom((flags & VXLAN_F_GPE) | VXLAN_F_IPV6),
                                            netif_is_any_bridge_port(dev));
                if (err < 0) {
                        goto tx_error;
                } else if (err) {
                        if (info) {
                                struct ip_tunnel_info *unclone;

                                unclone = skb_tunnel_info_unclone(skb);
                                if (unlikely(!unclone))
                                        goto tx_error;

                                unclone->key.u.ipv6.src = pkey->u.ipv6.dst;
                                unclone->key.u.ipv6.dst = saddr;
                        }

                        vxlan_encap_bypass(skb, vxlan, vxlan, vni, false);
                        dst_release(ndst);
                        goto out_unlock;
                }

                tos = ip_tunnel_ecn_encap(tos, old_iph, skb);
                ttl = ttl ? : ip6_dst_hoplimit(ndst);
                skb_scrub_packet(skb, xnet);
                err = vxlan_build_skb(skb, ndst, sizeof(struct ipv6hdr),
                                      vni, md, flags, udp_sum);
                if (err < 0) {
                        reason = SKB_DROP_REASON_NOMEM;
                        goto tx_error;
                }

                udp_tunnel6_xmit_skb(ndst, sock6->sock->sk, skb, dev,
                                     &saddr, &pkey->u.ipv6.dst, tos, ttl,
                                     pkey->label, src_port, dst_port, !udp_sum,
                                     ip6cb_flags);
#endif
        }
        vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX, pkt_len);
out_unlock:
        rcu_read_unlock();
        return;

drop:
        dev_dstats_tx_dropped(dev);
        vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX_DROPS, 0);
        kfree_skb_reason(skb, reason);
        return;

tx_error:
        rcu_read_unlock();
        if (err == -ELOOP)
                DEV_STATS_INC(dev, collisions);
        else if (err == -ENETUNREACH)
                DEV_STATS_INC(dev, tx_carrier_errors);
        dst_release(ndst);
        DEV_STATS_INC(dev, tx_errors);
        vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX_ERRORS, 0);
        kfree_skb_reason(skb, reason);
}

static void vxlan_xmit_nh(struct sk_buff *skb, struct net_device *dev,
                          struct vxlan_fdb *f, __be32 vni, bool did_rsc)
{
        struct vxlan_rdst nh_rdst;
        struct nexthop *nh;
        bool do_xmit;
        u32 hash;

        memset(&nh_rdst, 0, sizeof(struct vxlan_rdst));
        hash = skb_get_hash(skb);

        nh = rcu_dereference(f->nh);
        if (!nh)
                goto drop;
        do_xmit = vxlan_fdb_nh_path_select(nh, hash, &nh_rdst);

        if (likely(do_xmit))
                vxlan_xmit_one(skb, dev, vni, &nh_rdst, did_rsc);
        else
                goto drop;

        return;

drop:
        dev_dstats_tx_dropped(dev);
        vxlan_vnifilter_count(netdev_priv(dev), vni, NULL,
                              VXLAN_VNI_STATS_TX_DROPS, 0);
        dev_kfree_skb(skb);
}

static netdev_tx_t vxlan_xmit_nhid(struct sk_buff *skb, struct net_device *dev,
                                   u32 nhid, __be32 vni)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst nh_rdst;
        struct nexthop *nh;
        bool do_xmit;
        u32 hash;

        memset(&nh_rdst, 0, sizeof(struct vxlan_rdst));
        hash = skb_get_hash(skb);

        rcu_read_lock();
        nh = nexthop_find_by_id(dev_net(dev), nhid);
        if (unlikely(!nh || !nexthop_is_fdb(nh) || !nexthop_is_multipath(nh))) {
                rcu_read_unlock();
                goto drop;
        }
        do_xmit = vxlan_fdb_nh_path_select(nh, hash, &nh_rdst);
        rcu_read_unlock();

        if (vxlan->cfg.saddr.sa.sa_family != nh_rdst.remote_ip.sa.sa_family)
                goto drop;

        if (likely(do_xmit))
                vxlan_xmit_one(skb, dev, vni, &nh_rdst, false);
        else
                goto drop;

        return NETDEV_TX_OK;

drop:
        dev_dstats_tx_dropped(dev);
        vxlan_vnifilter_count(netdev_priv(dev), vni, NULL,
                              VXLAN_VNI_STATS_TX_DROPS, 0);
        dev_kfree_skb(skb);
        return NETDEV_TX_OK;
}

/* Transmit local packets over Vxlan
 *
 * Outer IP header inherits ECN and DF from inner header.
 * Outer UDP destination is the VXLAN assigned port.
 *           source port is based on hash of flow
 */
static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst *rdst, *fdst = NULL;
        const struct ip_tunnel_info *info;
        struct vxlan_fdb *f;
        struct ethhdr *eth;
        __be32 vni = 0;
        u32 nhid = 0;
        bool did_rsc;

        info = skb_tunnel_info(skb);

        skb_reset_mac_header(skb);

        if (vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) {
                if (info && info->mode & IP_TUNNEL_INFO_BRIDGE &&
                    info->mode & IP_TUNNEL_INFO_TX) {
                        vni = tunnel_id_to_key32(info->key.tun_id);
                        nhid = info->key.nhid;
                } else {
                        if (info && info->mode & IP_TUNNEL_INFO_TX)
                                vxlan_xmit_one(skb, dev, vni, NULL, false);
                        else
                                kfree_skb_reason(skb, SKB_DROP_REASON_TUNNEL_TXINFO);
                        return NETDEV_TX_OK;
                }
        }

        if (vxlan->cfg.flags & VXLAN_F_PROXY) {
                eth = eth_hdr(skb);
                if (ntohs(eth->h_proto) == ETH_P_ARP)
                        return arp_reduce(dev, skb, vni);
#if IS_ENABLED(CONFIG_IPV6)
                else if (ntohs(eth->h_proto) == ETH_P_IPV6 &&
                         pskb_may_pull(skb, sizeof(struct ipv6hdr) +
                                            sizeof(struct nd_msg)) &&
                         ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6) {
                        struct nd_msg *m = (struct nd_msg *)(ipv6_hdr(skb) + 1);

                        if (m->icmph.icmp6_code == 0 &&
                            m->icmph.icmp6_type == NDISC_NEIGHBOUR_SOLICITATION)
                                return neigh_reduce(dev, skb, vni);
                }
#endif
        }

        if (nhid)
                return vxlan_xmit_nhid(skb, dev, nhid, vni);

        if (vxlan->cfg.flags & VXLAN_F_MDB) {
                struct vxlan_mdb_entry *mdb_entry;

                rcu_read_lock();
                mdb_entry = vxlan_mdb_entry_skb_get(vxlan, skb, vni);
                if (mdb_entry) {
                        netdev_tx_t ret;

                        ret = vxlan_mdb_xmit(vxlan, mdb_entry, skb);
                        rcu_read_unlock();
                        return ret;
                }
                rcu_read_unlock();
        }

        eth = eth_hdr(skb);
        rcu_read_lock();
        f = vxlan_find_mac_tx(vxlan, eth->h_dest, vni);
        did_rsc = false;

        if (f && (f->flags & NTF_ROUTER) && (vxlan->cfg.flags & VXLAN_F_RSC) &&
            (ntohs(eth->h_proto) == ETH_P_IP ||
             ntohs(eth->h_proto) == ETH_P_IPV6)) {
                did_rsc = route_shortcircuit(dev, skb);
                if (did_rsc)
                        f = vxlan_find_mac_tx(vxlan, eth->h_dest, vni);
        }

        if (f == NULL) {
                f = vxlan_find_mac_tx(vxlan, all_zeros_mac, vni);
                if (f == NULL) {
                        if ((vxlan->cfg.flags & VXLAN_F_L2MISS) &&
                            !is_multicast_ether_addr(eth->h_dest))
                                vxlan_fdb_miss(vxlan, eth->h_dest);

                        dev_dstats_tx_dropped(dev);
                        vxlan_vnifilter_count(vxlan, vni, NULL,
                                              VXLAN_VNI_STATS_TX_DROPS, 0);
                        kfree_skb_reason(skb, SKB_DROP_REASON_NO_TX_TARGET);
                        goto out;
                }
        }

        if (rcu_access_pointer(f->nh)) {
                vxlan_xmit_nh(skb, dev, f,
                              (vni ? : vxlan->default_dst.remote_vni), did_rsc);
        } else {
                list_for_each_entry_rcu(rdst, &f->remotes, list) {
                        struct sk_buff *skb1;

                        if (!fdst) {
                                fdst = rdst;
                                continue;
                        }
                        skb1 = skb_clone(skb, GFP_ATOMIC);
                        if (skb1)
                                vxlan_xmit_one(skb1, dev, vni, rdst, did_rsc);
                }
                if (fdst)
                        vxlan_xmit_one(skb, dev, vni, fdst, did_rsc);
                else
                        kfree_skb_reason(skb, SKB_DROP_REASON_NO_TX_TARGET);
        }

out:
        rcu_read_unlock();
        return NETDEV_TX_OK;
}

/* Walk the forwarding table and purge stale entries */
static void vxlan_cleanup(struct timer_list *t)
{
        struct vxlan_dev *vxlan = timer_container_of(vxlan, t, age_timer);
        unsigned long next_timer = jiffies + FDB_AGE_INTERVAL;
        struct vxlan_fdb *f;

        if (!netif_running(vxlan->dev))
                return;

        rcu_read_lock();
        hlist_for_each_entry_rcu(f, &vxlan->fdb_list, fdb_node) {
                unsigned long timeout;

                if (f->state & (NUD_PERMANENT | NUD_NOARP))
                        continue;

                if (f->flags & NTF_EXT_LEARNED)
                        continue;

                timeout = READ_ONCE(f->updated) + vxlan->cfg.age_interval * HZ;
                if (time_before_eq(timeout, jiffies)) {
                        spin_lock(&vxlan->hash_lock);
                        if (!hlist_unhashed(&f->fdb_node)) {
                                netdev_dbg(vxlan->dev, "garbage collect %pM\n",
                                           f->key.eth_addr);
                                f->state = NUD_STALE;
                                vxlan_fdb_destroy(vxlan, f, true, true);
                        }
                        spin_unlock(&vxlan->hash_lock);
                } else if (time_before(timeout, next_timer)) {
                        next_timer = timeout;
                }
        }
        rcu_read_unlock();

        mod_timer(&vxlan->age_timer, next_timer);
}

static void vxlan_vs_del_dev(struct vxlan_dev *vxlan)
{
        ASSERT_RTNL();

        hlist_del_init_rcu(&vxlan->hlist4.hlist);
#if IS_ENABLED(CONFIG_IPV6)
        hlist_del_init_rcu(&vxlan->hlist6.hlist);
#endif
}

static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan,
                             struct vxlan_dev_node *node)
{
        __be32 vni = vxlan->default_dst.remote_vni;

        ASSERT_RTNL();

        node->vxlan = vxlan;
        hlist_add_head_rcu(&node->hlist, vni_head(vs, vni));
}

/* Setup stats when device is created */
static int vxlan_init(struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        int err;

        err = rhashtable_init(&vxlan->fdb_hash_tbl, &vxlan_fdb_rht_params);
        if (err)
                return err;

        if (vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
                err = vxlan_vnigroup_init(vxlan);
                if (err)
                        goto err_rhashtable_destroy;
        }

        err = gro_cells_init(&vxlan->gro_cells, dev);
        if (err)
                goto err_vnigroup_uninit;

        err = vxlan_mdb_init(vxlan);
        if (err)
                goto err_gro_cells_destroy;

        netdev_lockdep_set_classes(dev);
        return 0;

err_gro_cells_destroy:
        gro_cells_destroy(&vxlan->gro_cells);
err_vnigroup_uninit:
        if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
                vxlan_vnigroup_uninit(vxlan);
err_rhashtable_destroy:
        rhashtable_destroy(&vxlan->fdb_hash_tbl);
        return err;
}

static void vxlan_uninit(struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);

        vxlan_mdb_fini(vxlan);

        if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
                vxlan_vnigroup_uninit(vxlan);

        gro_cells_destroy(&vxlan->gro_cells);

        rhashtable_destroy(&vxlan->fdb_hash_tbl);
}

/* Start ageing timer and join group when device is brought up */
static int vxlan_open(struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        int ret;

        ret = vxlan_sock_add(vxlan);
        if (ret < 0)
                return ret;

        ret = vxlan_multicast_join(vxlan);
        if (ret) {
                vxlan_sock_release(vxlan);
                return ret;
        }

        if (vxlan->cfg.age_interval)
                mod_timer(&vxlan->age_timer, jiffies + FDB_AGE_INTERVAL);

        return ret;
}

struct vxlan_fdb_flush_desc {
        bool                                ignore_default_entry;
        unsigned long                   state;
        unsigned long                        state_mask;
        unsigned long                   flags;
        unsigned long                        flags_mask;
        __be32                                src_vni;
        u32                                nhid;
        __be32                                vni;
        __be16                                port;
        union vxlan_addr                dst_ip;
};

static bool vxlan_fdb_is_default_entry(const struct vxlan_fdb *f,
                                       const struct vxlan_dev *vxlan)
{
        return is_zero_ether_addr(f->key.eth_addr) &&
               f->key.vni == vxlan->cfg.vni;
}

static bool vxlan_fdb_nhid_matches(const struct vxlan_fdb *f, u32 nhid)
{
        struct nexthop *nh = rtnl_dereference(f->nh);

        return nh && nh->id == nhid;
}

static bool vxlan_fdb_flush_matches(const struct vxlan_fdb *f,
                                    const struct vxlan_dev *vxlan,
                                    const struct vxlan_fdb_flush_desc *desc)
{
        if (desc->state_mask && (f->state & desc->state_mask) != desc->state)
                return false;

        if (desc->flags_mask && (f->flags & desc->flags_mask) != desc->flags)
                return false;

        if (desc->ignore_default_entry && vxlan_fdb_is_default_entry(f, vxlan))
                return false;

        if (desc->src_vni && f->key.vni != desc->src_vni)
                return false;

        if (desc->nhid && !vxlan_fdb_nhid_matches(f, desc->nhid))
                return false;

        return true;
}

static bool
vxlan_fdb_flush_should_match_remotes(const struct vxlan_fdb_flush_desc *desc)
{
        return desc->vni || desc->port || desc->dst_ip.sa.sa_family;
}

static bool
vxlan_fdb_flush_remote_matches(const struct vxlan_fdb_flush_desc *desc,
                               const struct vxlan_rdst *rd)
{
        if (desc->vni && rd->remote_vni != desc->vni)
                return false;

        if (desc->port && rd->remote_port != desc->port)
                return false;

        if (desc->dst_ip.sa.sa_family &&
            !vxlan_addr_equal(&rd->remote_ip, &desc->dst_ip))
                return false;

        return true;
}

static void
vxlan_fdb_flush_match_remotes(struct vxlan_fdb *f, struct vxlan_dev *vxlan,
                              const struct vxlan_fdb_flush_desc *desc,
                              bool *p_destroy_fdb)
{
        bool remotes_flushed = false;
        struct vxlan_rdst *rd, *tmp;

        list_for_each_entry_safe(rd, tmp, &f->remotes, list) {
                if (!vxlan_fdb_flush_remote_matches(desc, rd))
                        continue;

                vxlan_fdb_dst_destroy(vxlan, f, rd, true);
                remotes_flushed = true;
        }

        *p_destroy_fdb = remotes_flushed && list_empty(&f->remotes);
}

/* Purge the forwarding table */
static void vxlan_flush(struct vxlan_dev *vxlan,
                        const struct vxlan_fdb_flush_desc *desc)
{
        bool match_remotes = vxlan_fdb_flush_should_match_remotes(desc);
        struct vxlan_fdb *f;

        rcu_read_lock();
        hlist_for_each_entry_rcu(f, &vxlan->fdb_list, fdb_node) {
                if (!vxlan_fdb_flush_matches(f, vxlan, desc))
                        continue;

                spin_lock_bh(&vxlan->hash_lock);
                if (hlist_unhashed(&f->fdb_node))
                        goto unlock;

                if (match_remotes) {
                        bool destroy_fdb = false;

                        vxlan_fdb_flush_match_remotes(f, vxlan, desc,
                                                      &destroy_fdb);

                        if (!destroy_fdb)
                                goto unlock;
                }

                vxlan_fdb_destroy(vxlan, f, true, true);
unlock:
                spin_unlock_bh(&vxlan->hash_lock);
        }
        rcu_read_unlock();
}

static const struct nla_policy vxlan_del_bulk_policy[NDA_MAX + 1] = {
        [NDA_SRC_VNI]   = { .type = NLA_U32 },
        [NDA_NH_ID]        = { .type = NLA_U32 },
        [NDA_VNI]        = { .type = NLA_U32 },
        [NDA_PORT]        = { .type = NLA_U16 },
        [NDA_DST]        = NLA_POLICY_RANGE(NLA_BINARY, sizeof(struct in_addr),
                                           sizeof(struct in6_addr)),
        [NDA_NDM_STATE_MASK]        = { .type = NLA_U16 },
        [NDA_NDM_FLAGS_MASK]        = { .type = NLA_U8 },
};

#define VXLAN_FDB_FLUSH_IGNORED_NDM_FLAGS (NTF_MASTER | NTF_SELF)
#define VXLAN_FDB_FLUSH_ALLOWED_NDM_STATES (NUD_PERMANENT | NUD_NOARP)
#define VXLAN_FDB_FLUSH_ALLOWED_NDM_FLAGS (NTF_EXT_LEARNED | NTF_OFFLOADED | \
                                           NTF_ROUTER)

static int vxlan_fdb_delete_bulk(struct nlmsghdr *nlh, struct net_device *dev,
                                 struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb_flush_desc desc = {};
        struct ndmsg *ndm = nlmsg_data(nlh);
        struct nlattr *tb[NDA_MAX + 1];
        u8 ndm_flags;
        int err;

        ndm_flags = ndm->ndm_flags & ~VXLAN_FDB_FLUSH_IGNORED_NDM_FLAGS;

        err = nlmsg_parse(nlh, sizeof(*ndm), tb, NDA_MAX, vxlan_del_bulk_policy,
                          extack);
        if (err)
                return err;

        if (ndm_flags & ~VXLAN_FDB_FLUSH_ALLOWED_NDM_FLAGS) {
                NL_SET_ERR_MSG(extack, "Unsupported fdb flush ndm flag bits set");
                return -EINVAL;
        }
        if (ndm->ndm_state & ~VXLAN_FDB_FLUSH_ALLOWED_NDM_STATES) {
                NL_SET_ERR_MSG(extack, "Unsupported fdb flush ndm state bits set");
                return -EINVAL;
        }

        desc.state = ndm->ndm_state;
        desc.flags = ndm_flags;

        if (tb[NDA_NDM_STATE_MASK])
                desc.state_mask = nla_get_u16(tb[NDA_NDM_STATE_MASK]);

        if (tb[NDA_NDM_FLAGS_MASK])
                desc.flags_mask = nla_get_u8(tb[NDA_NDM_FLAGS_MASK]);

        if (tb[NDA_SRC_VNI])
                desc.src_vni = cpu_to_be32(nla_get_u32(tb[NDA_SRC_VNI]));

        if (tb[NDA_NH_ID])
                desc.nhid = nla_get_u32(tb[NDA_NH_ID]);

        if (tb[NDA_VNI])
                desc.vni = cpu_to_be32(nla_get_u32(tb[NDA_VNI]));

        if (tb[NDA_PORT])
                desc.port = nla_get_be16(tb[NDA_PORT]);

        if (tb[NDA_DST]) {
                union vxlan_addr ip;

                err = vxlan_nla_get_addr(&ip, tb[NDA_DST]);
                if (err) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[NDA_DST],
                                            "Unsupported address family");
                        return err;
                }
                desc.dst_ip = ip;
        }

        vxlan_flush(vxlan, &desc);

        return 0;
}

/* Cleanup timer and forwarding table on shutdown */
static int vxlan_stop(struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb_flush_desc desc = {
                /* Default entry is deleted at vxlan_dellink. */
                .ignore_default_entry = true,
                .state = 0,
                .state_mask = NUD_PERMANENT | NUD_NOARP,
        };

        vxlan_multicast_leave(vxlan);

        timer_delete_sync(&vxlan->age_timer);

        vxlan_flush(vxlan, &desc);
        vxlan_sock_release(vxlan);

        return 0;
}

/* Stub, nothing needs to be done. */
static void vxlan_set_multicast_list(struct net_device *dev)
{
}

static int vxlan_change_mtu(struct net_device *dev, int new_mtu)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst *dst = &vxlan->default_dst;
        struct net_device *lowerdev = __dev_get_by_index(vxlan->net,
                                                         dst->remote_ifindex);

        /* This check is different than dev->max_mtu, because it looks at
         * the lowerdev->mtu, rather than the static dev->max_mtu
         */
        if (lowerdev) {
                int max_mtu = lowerdev->mtu - vxlan_headroom(vxlan->cfg.flags);
                if (new_mtu > max_mtu)
                        return -EINVAL;
        }

        WRITE_ONCE(dev->mtu, new_mtu);
        return 0;
}

static int vxlan_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct ip_tunnel_info *info = skb_tunnel_info(skb);
        __be16 sport, dport;

        sport = udp_flow_src_port(dev_net(dev), skb, vxlan->cfg.port_min,
                                  vxlan->cfg.port_max, true);
        dport = info->key.tp_dst ? : vxlan->cfg.dst_port;

        if (ip_tunnel_info_af(info) == AF_INET) {
                struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock);
                struct rtable *rt;

                if (!sock4)
                        return -EIO;

                rt = udp_tunnel_dst_lookup(skb, dev, vxlan->net, 0,
                                           &info->key.u.ipv4.src,
                                           &info->key,
                                           sport, dport, info->key.tos,
                                           &info->dst_cache);
                if (IS_ERR(rt))
                        return PTR_ERR(rt);
                ip_rt_put(rt);
        } else {
#if IS_ENABLED(CONFIG_IPV6)
                struct vxlan_sock *sock6 = rcu_dereference(vxlan->vn6_sock);
                struct dst_entry *ndst;

                if (!sock6)
                        return -EIO;

                ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sock,
                                              0, &info->key.u.ipv6.src,
                                              &info->key,
                                              sport, dport, info->key.tos,
                                              &info->dst_cache);
                if (IS_ERR(ndst))
                        return PTR_ERR(ndst);
                dst_release(ndst);
#else /* !CONFIG_IPV6 */
                return -EPFNOSUPPORT;
#endif
        }
        info->key.tp_src = sport;
        info->key.tp_dst = dport;
        return 0;
}

static const struct net_device_ops vxlan_netdev_ether_ops = {
        .ndo_init                = vxlan_init,
        .ndo_uninit                = vxlan_uninit,
        .ndo_open                = vxlan_open,
        .ndo_stop                = vxlan_stop,
        .ndo_start_xmit                = vxlan_xmit,
        .ndo_set_rx_mode        = vxlan_set_multicast_list,
        .ndo_change_mtu                = vxlan_change_mtu,
        .ndo_validate_addr        = eth_validate_addr,
        .ndo_set_mac_address        = eth_mac_addr,
        .ndo_fdb_add                = vxlan_fdb_add,
        .ndo_fdb_del                = vxlan_fdb_delete,
        .ndo_fdb_del_bulk        = vxlan_fdb_delete_bulk,
        .ndo_fdb_dump                = vxlan_fdb_dump,
        .ndo_fdb_get                = vxlan_fdb_get,
        .ndo_mdb_add                = vxlan_mdb_add,
        .ndo_mdb_del                = vxlan_mdb_del,
        .ndo_mdb_del_bulk        = vxlan_mdb_del_bulk,
        .ndo_mdb_dump                = vxlan_mdb_dump,
        .ndo_mdb_get                = vxlan_mdb_get,
        .ndo_fill_metadata_dst        = vxlan_fill_metadata_dst,
};

static const struct net_device_ops vxlan_netdev_raw_ops = {
        .ndo_init                = vxlan_init,
        .ndo_uninit                = vxlan_uninit,
        .ndo_open                = vxlan_open,
        .ndo_stop                = vxlan_stop,
        .ndo_start_xmit                = vxlan_xmit,
        .ndo_change_mtu                = vxlan_change_mtu,
        .ndo_fill_metadata_dst        = vxlan_fill_metadata_dst,
};

/* Info for udev, that this is a virtual tunnel endpoint */
static const struct device_type vxlan_type = {
        .name = "vxlan",
};

/* Calls the ndo_udp_tunnel_add of the caller in order to
 * supply the listening VXLAN udp ports. Callers are expected
 * to implement the ndo_udp_tunnel_add.
 */
static void vxlan_offload_rx_ports(struct net_device *dev, bool push)
{
        struct vxlan_sock *vs;
        struct net *net = dev_net(dev);
        struct vxlan_net *vn = net_generic(net, vxlan_net_id);
        unsigned int i;

        ASSERT_RTNL();

        for (i = 0; i < PORT_HASH_SIZE; ++i) {
                hlist_for_each_entry(vs, &vn->sock_list[i], hlist) {
                        unsigned short type;

                        if (vs->flags & VXLAN_F_GPE)
                                type = UDP_TUNNEL_TYPE_VXLAN_GPE;
                        else
                                type = UDP_TUNNEL_TYPE_VXLAN;

                        if (push)
                                udp_tunnel_push_rx_port(dev, vs->sock, type);
                        else
                                udp_tunnel_drop_rx_port(dev, vs->sock, type);
                }
        }
}

/* Initialize the device structure. */
static void vxlan_setup(struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);

        eth_hw_addr_random(dev);
        ether_setup(dev);

        dev->needs_free_netdev = true;
        SET_NETDEV_DEVTYPE(dev, &vxlan_type);

        dev->features        |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->features   |= NETIF_F_RXCSUM;
        dev->features   |= NETIF_F_GSO_SOFTWARE;

        dev->vlan_features = dev->features;
        dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_FRAGLIST;
        dev->hw_features |= NETIF_F_RXCSUM;
        dev->hw_features |= NETIF_F_GSO_SOFTWARE;
        netif_keep_dst(dev);
        dev->priv_flags |= IFF_NO_QUEUE;
        dev->change_proto_down = true;
        dev->lltx = true;

        /* MTU range: 68 - 65535 */
        dev->min_mtu = ETH_MIN_MTU;
        dev->max_mtu = ETH_MAX_MTU;

        dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
        INIT_LIST_HEAD(&vxlan->next);
        spin_lock_init(&vxlan->hash_lock);

        timer_setup(&vxlan->age_timer, vxlan_cleanup, TIMER_DEFERRABLE);

        vxlan->dev = dev;

        INIT_HLIST_HEAD(&vxlan->fdb_list);
}

static void vxlan_ether_setup(struct net_device *dev)
{
        dev->priv_flags &= ~IFF_TX_SKB_SHARING;
        dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
        dev->netdev_ops = &vxlan_netdev_ether_ops;
}

static void vxlan_raw_setup(struct net_device *dev)
{
        dev->header_ops = NULL;
        dev->type = ARPHRD_NONE;
        dev->hard_header_len = 0;
        dev->addr_len = 0;
        dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
        dev->netdev_ops = &vxlan_netdev_raw_ops;
}

static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = {
        [IFLA_VXLAN_UNSPEC]     = { .strict_start_type = IFLA_VXLAN_LOCALBYPASS },
        [IFLA_VXLAN_ID]                = { .type = NLA_U32 },
        [IFLA_VXLAN_GROUP]        = { .len = sizeof_field(struct iphdr, daddr) },
        [IFLA_VXLAN_GROUP6]        = { .len = sizeof(struct in6_addr) },
        [IFLA_VXLAN_LINK]        = { .type = NLA_U32 },
        [IFLA_VXLAN_LOCAL]        = { .len = sizeof_field(struct iphdr, saddr) },
        [IFLA_VXLAN_LOCAL6]        = { .len = sizeof(struct in6_addr) },
        [IFLA_VXLAN_TOS]        = { .type = NLA_U8 },
        [IFLA_VXLAN_TTL]        = { .type = NLA_U8 },
        [IFLA_VXLAN_LABEL]        = { .type = NLA_U32 },
        [IFLA_VXLAN_LEARNING]        = { .type = NLA_U8 },
        [IFLA_VXLAN_AGEING]        = { .type = NLA_U32 },
        [IFLA_VXLAN_LIMIT]        = { .type = NLA_U32 },
        [IFLA_VXLAN_PORT_RANGE] = { .len  = sizeof(struct ifla_vxlan_port_range) },
        [IFLA_VXLAN_PROXY]        = { .type = NLA_U8 },
        [IFLA_VXLAN_RSC]        = { .type = NLA_U8 },
        [IFLA_VXLAN_L2MISS]        = { .type = NLA_U8 },
        [IFLA_VXLAN_L3MISS]        = { .type = NLA_U8 },
        [IFLA_VXLAN_COLLECT_METADATA]        = { .type = NLA_U8 },
        [IFLA_VXLAN_PORT]        = { .type = NLA_U16 },
        [IFLA_VXLAN_UDP_CSUM]        = { .type = NLA_U8 },
        [IFLA_VXLAN_UDP_ZERO_CSUM6_TX]        = { .type = NLA_U8 },
        [IFLA_VXLAN_UDP_ZERO_CSUM6_RX]        = { .type = NLA_U8 },
        [IFLA_VXLAN_REMCSUM_TX]        = { .type = NLA_U8 },
        [IFLA_VXLAN_REMCSUM_RX]        = { .type = NLA_U8 },
        [IFLA_VXLAN_GBP]        = { .type = NLA_FLAG, },
        [IFLA_VXLAN_GPE]        = { .type = NLA_FLAG, },
        [IFLA_VXLAN_REMCSUM_NOPARTIAL]        = { .type = NLA_FLAG },
        [IFLA_VXLAN_TTL_INHERIT]        = { .type = NLA_FLAG },
        [IFLA_VXLAN_DF]                = { .type = NLA_U8 },
        [IFLA_VXLAN_VNIFILTER]        = { .type = NLA_U8 },
        [IFLA_VXLAN_LOCALBYPASS]        = NLA_POLICY_MAX(NLA_U8, 1),
        [IFLA_VXLAN_LABEL_POLICY]       = NLA_POLICY_MAX(NLA_U32, VXLAN_LABEL_MAX),
        [IFLA_VXLAN_RESERVED_BITS] = NLA_POLICY_EXACT_LEN(sizeof(struct vxlanhdr)),
        [IFLA_VXLAN_MC_ROUTE]                = NLA_POLICY_MAX(NLA_U8, 1),
};

static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        if (tb[IFLA_ADDRESS]) {
                if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
                                            "Provided link layer address is not Ethernet");
                        return -EINVAL;
                }

                if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
                                            "Provided Ethernet address is not unicast");
                        return -EADDRNOTAVAIL;
                }
        }

        if (tb[IFLA_MTU]) {
                u32 mtu = nla_get_u32(tb[IFLA_MTU]);

                if (mtu < ETH_MIN_MTU || mtu > ETH_MAX_MTU) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_MTU],
                                            "MTU must be between 68 and 65535");
                        return -EINVAL;
                }
        }

        if (!data) {
                NL_SET_ERR_MSG(extack,
                               "Required attributes not provided to perform the operation");
                return -EINVAL;
        }

        if (data[IFLA_VXLAN_ID]) {
                u32 id = nla_get_u32(data[IFLA_VXLAN_ID]);

                if (id >= VXLAN_N_VID) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VXLAN_ID],
                                            "VXLAN ID must be lower than 16777216");
                        return -ERANGE;
                }
        }

        if (data[IFLA_VXLAN_PORT_RANGE]) {
                const struct ifla_vxlan_port_range *p
                        = nla_data(data[IFLA_VXLAN_PORT_RANGE]);

                if (ntohs(p->high) < ntohs(p->low)) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VXLAN_PORT_RANGE],
                                            "Invalid source port range");
                        return -EINVAL;
                }
        }

        if (data[IFLA_VXLAN_DF]) {
                enum ifla_vxlan_df df = nla_get_u8(data[IFLA_VXLAN_DF]);

                if (df < 0 || df > VXLAN_DF_MAX) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VXLAN_DF],
                                            "Invalid DF attribute");
                        return -EINVAL;
                }
        }

        return 0;
}

static void vxlan_get_drvinfo(struct net_device *netdev,
                              struct ethtool_drvinfo *drvinfo)
{
        strscpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version));
        strscpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver));
}

static int vxlan_get_link_ksettings(struct net_device *dev,
                                    struct ethtool_link_ksettings *cmd)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst *dst = &vxlan->default_dst;
        struct net_device *lowerdev = __dev_get_by_index(vxlan->net,
                                                         dst->remote_ifindex);

        if (!lowerdev) {
                cmd->base.duplex = DUPLEX_UNKNOWN;
                cmd->base.port = PORT_OTHER;
                cmd->base.speed = SPEED_UNKNOWN;

                return 0;
        }

        return __ethtool_get_link_ksettings(lowerdev, cmd);
}

static const struct ethtool_ops vxlan_ethtool_ops = {
        .get_drvinfo                = vxlan_get_drvinfo,
        .get_link                = ethtool_op_get_link,
        .get_link_ksettings        = vxlan_get_link_ksettings,
};

static struct socket *vxlan_create_sock(struct net *net, bool ipv6,
                                        __be16 port, u32 flags, int ifindex)
{
        struct socket *sock;
        struct udp_port_cfg udp_conf;
        int err;

        memset(&udp_conf, 0, sizeof(udp_conf));

        if (ipv6) {
                udp_conf.family = AF_INET6;
                udp_conf.use_udp6_rx_checksums =
                    !(flags & VXLAN_F_UDP_ZERO_CSUM6_RX);
                udp_conf.ipv6_v6only = 1;
        } else {
                udp_conf.family = AF_INET;
        }

        udp_conf.local_udp_port = port;
        udp_conf.bind_ifindex = ifindex;

        /* Open UDP socket */
        err = udp_sock_create(net, &udp_conf, &sock);
        if (err < 0)
                return ERR_PTR(err);

        udp_allow_gso(sock->sk);
        return sock;
}

/* Create new listen socket if needed */
static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
                                              __be16 port, u32 flags,
                                              int ifindex)
{
        struct vxlan_sock *vs;
        struct socket *sock;
        unsigned int h;
        struct udp_tunnel_sock_cfg tunnel_cfg;

        ASSERT_RTNL();

        vs = kzalloc(sizeof(*vs), GFP_KERNEL);
        if (!vs)
                return ERR_PTR(-ENOMEM);

        for (h = 0; h < VNI_HASH_SIZE; ++h)
                INIT_HLIST_HEAD(&vs->vni_list[h]);

        sock = vxlan_create_sock(net, ipv6, port, flags, ifindex);
        if (IS_ERR(sock)) {
                kfree(vs);
                return ERR_CAST(sock);
        }

        vs->sock = sock;
        refcount_set(&vs->refcnt, 1);
        vs->flags = (flags & VXLAN_F_RCV_FLAGS);

        hlist_add_head_rcu(&vs->hlist, vs_head(net, port));
        udp_tunnel_notify_add_rx_port(sock,
                                      (vs->flags & VXLAN_F_GPE) ?
                                      UDP_TUNNEL_TYPE_VXLAN_GPE :
                                      UDP_TUNNEL_TYPE_VXLAN);

        /* Mark socket as an encapsulation socket. */
        memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
        tunnel_cfg.sk_user_data = vs;
        tunnel_cfg.encap_type = 1;
        tunnel_cfg.encap_rcv = vxlan_rcv;
        tunnel_cfg.encap_err_lookup = vxlan_err_lookup;
        tunnel_cfg.encap_destroy = NULL;
        if (vs->flags & VXLAN_F_GPE) {
                tunnel_cfg.gro_receive = vxlan_gpe_gro_receive;
                tunnel_cfg.gro_complete = vxlan_gpe_gro_complete;
        } else {
                tunnel_cfg.gro_receive = vxlan_gro_receive;
                tunnel_cfg.gro_complete = vxlan_gro_complete;
        }

        setup_udp_tunnel_sock(net, sock, &tunnel_cfg);

        return vs;
}

static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6)
{
        bool metadata = vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA;
        struct vxlan_sock *vs = NULL;
        struct vxlan_dev_node *node;
        int l3mdev_index = 0;

        ASSERT_RTNL();

        if (vxlan->cfg.remote_ifindex)
                l3mdev_index = l3mdev_master_upper_ifindex_by_index(
                        vxlan->net, vxlan->cfg.remote_ifindex);

        if (!vxlan->cfg.no_share) {
                rcu_read_lock();
                vs = vxlan_find_sock(vxlan->net, ipv6 ? AF_INET6 : AF_INET,
                                     vxlan->cfg.dst_port, vxlan->cfg.flags,
                                     l3mdev_index);
                if (vs && !refcount_inc_not_zero(&vs->refcnt)) {
                        rcu_read_unlock();
                        return -EBUSY;
                }
                rcu_read_unlock();
        }
        if (!vs)
                vs = vxlan_socket_create(vxlan->net, ipv6,
                                         vxlan->cfg.dst_port, vxlan->cfg.flags,
                                         l3mdev_index);
        if (IS_ERR(vs))
                return PTR_ERR(vs);
#if IS_ENABLED(CONFIG_IPV6)
        if (ipv6) {
                rcu_assign_pointer(vxlan->vn6_sock, vs);
                node = &vxlan->hlist6;
        } else
#endif
        {
                rcu_assign_pointer(vxlan->vn4_sock, vs);
                node = &vxlan->hlist4;
        }

        if (metadata && (vxlan->cfg.flags & VXLAN_F_VNIFILTER))
                vxlan_vs_add_vnigrp(vxlan, vs, ipv6);
        else
                vxlan_vs_add_dev(vs, vxlan, node);

        return 0;
}

static int vxlan_sock_add(struct vxlan_dev *vxlan)
{
        bool metadata = vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA;
        bool ipv6 = vxlan->cfg.flags & VXLAN_F_IPV6 || metadata;
        bool ipv4 = !ipv6 || metadata;
        int ret = 0;

        RCU_INIT_POINTER(vxlan->vn4_sock, NULL);
#if IS_ENABLED(CONFIG_IPV6)
        RCU_INIT_POINTER(vxlan->vn6_sock, NULL);
        if (ipv6) {
                ret = __vxlan_sock_add(vxlan, true);
                if (ret < 0 && ret != -EAFNOSUPPORT)
                        ipv4 = false;
        }
#endif
        if (ipv4)
                ret = __vxlan_sock_add(vxlan, false);
        if (ret < 0)
                vxlan_sock_release(vxlan);
        return ret;
}

int vxlan_vni_in_use(struct net *src_net, struct vxlan_dev *vxlan,
                     struct vxlan_config *conf, __be32 vni)
{
        struct vxlan_net *vn = net_generic(src_net, vxlan_net_id);
        struct vxlan_dev *tmp;

        list_for_each_entry(tmp, &vn->vxlan_list, next) {
                if (tmp == vxlan)
                        continue;
                if (tmp->cfg.flags & VXLAN_F_VNIFILTER) {
                        if (!vxlan_vnifilter_lookup(tmp, vni))
                                continue;
                } else if (tmp->cfg.vni != vni) {
                        continue;
                }
                if (tmp->cfg.dst_port != conf->dst_port)
                        continue;
                if ((tmp->cfg.flags & (VXLAN_F_RCV_FLAGS | VXLAN_F_IPV6)) !=
                    (conf->flags & (VXLAN_F_RCV_FLAGS | VXLAN_F_IPV6)))
                        continue;

                if ((conf->flags & VXLAN_F_IPV6_LINKLOCAL) &&
                    tmp->cfg.remote_ifindex != conf->remote_ifindex)
                        continue;

                return -EEXIST;
        }

        return 0;
}

static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
                                 struct net_device **lower,
                                 struct vxlan_dev *old,
                                 struct netlink_ext_ack *extack)
{
        bool use_ipv6 = false;

        if (conf->flags & VXLAN_F_GPE) {
                /* For now, allow GPE only together with
                 * COLLECT_METADATA. This can be relaxed later; in such
                 * case, the other side of the PtP link will have to be
                 * provided.
                 */
                if ((conf->flags & ~VXLAN_F_ALLOWED_GPE) ||
                    !(conf->flags & VXLAN_F_COLLECT_METADATA)) {
                        NL_SET_ERR_MSG(extack,
                                       "VXLAN GPE does not support this combination of attributes");
                        return -EINVAL;
                }
        }

        if (!conf->remote_ip.sa.sa_family && !conf->saddr.sa.sa_family) {
                /* Unless IPv6 is explicitly requested, assume IPv4 */
                conf->remote_ip.sa.sa_family = AF_INET;
                conf->saddr.sa.sa_family = AF_INET;
        } else if (!conf->remote_ip.sa.sa_family) {
                conf->remote_ip.sa.sa_family = conf->saddr.sa.sa_family;
        } else if (!conf->saddr.sa.sa_family) {
                conf->saddr.sa.sa_family = conf->remote_ip.sa.sa_family;
        }

        if (conf->saddr.sa.sa_family != conf->remote_ip.sa.sa_family) {
                NL_SET_ERR_MSG(extack,
                               "Local and remote address must be from the same family");
                return -EINVAL;
        }

        if (vxlan_addr_multicast(&conf->saddr)) {
                NL_SET_ERR_MSG(extack, "Local address cannot be multicast");
                return -EINVAL;
        }

        if (conf->saddr.sa.sa_family == AF_INET6) {
                if (!IS_ENABLED(CONFIG_IPV6)) {
                        NL_SET_ERR_MSG(extack,
                                       "IPv6 support not enabled in the kernel");
                        return -EPFNOSUPPORT;
                }
                use_ipv6 = true;
                conf->flags |= VXLAN_F_IPV6;

                if (!(conf->flags & VXLAN_F_COLLECT_METADATA)) {
                        int local_type =
                                ipv6_addr_type(&conf->saddr.sin6.sin6_addr);
                        int remote_type =
                                ipv6_addr_type(&conf->remote_ip.sin6.sin6_addr);

                        if (local_type & IPV6_ADDR_LINKLOCAL) {
                                if (!(remote_type & IPV6_ADDR_LINKLOCAL) &&
                                    (remote_type != IPV6_ADDR_ANY)) {
                                        NL_SET_ERR_MSG(extack,
                                                       "Invalid combination of local and remote address scopes");
                                        return -EINVAL;
                                }

                                conf->flags |= VXLAN_F_IPV6_LINKLOCAL;
                        } else {
                                if (remote_type ==
                                    (IPV6_ADDR_UNICAST | IPV6_ADDR_LINKLOCAL)) {
                                        NL_SET_ERR_MSG(extack,
                                                       "Invalid combination of local and remote address scopes");
                                        return -EINVAL;
                                }

                                conf->flags &= ~VXLAN_F_IPV6_LINKLOCAL;
                        }
                }
        }

        if (conf->label && !use_ipv6) {
                NL_SET_ERR_MSG(extack,
                               "Label attribute only applies to IPv6 VXLAN devices");
                return -EINVAL;
        }

        if (conf->label_policy && !use_ipv6) {
                NL_SET_ERR_MSG(extack,
                               "Label policy only applies to IPv6 VXLAN devices");
                return -EINVAL;
        }

        if (conf->remote_ifindex) {
                struct net_device *lowerdev;

                lowerdev = __dev_get_by_index(src_net, conf->remote_ifindex);
                if (!lowerdev) {
                        NL_SET_ERR_MSG(extack,
                                       "Invalid local interface, device not found");
                        return -ENODEV;
                }

#if IS_ENABLED(CONFIG_IPV6)
                if (use_ipv6) {
                        struct inet6_dev *idev = __in6_dev_get(lowerdev);

                        if (idev && idev->cnf.disable_ipv6) {
                                NL_SET_ERR_MSG(extack,
                                               "IPv6 support disabled by administrator");
                                return -EPERM;
                        }
                }
#endif

                *lower = lowerdev;
        } else {
                if (vxlan_addr_multicast(&conf->remote_ip)) {
                        NL_SET_ERR_MSG(extack,
                                       "Local interface required for multicast remote destination");

                        return -EINVAL;
                }

#if IS_ENABLED(CONFIG_IPV6)
                if (conf->flags & VXLAN_F_IPV6_LINKLOCAL) {
                        NL_SET_ERR_MSG(extack,
                                       "Local interface required for link-local local/remote addresses");
                        return -EINVAL;
                }
#endif

                *lower = NULL;
        }

        if (!conf->dst_port) {
                if (conf->flags & VXLAN_F_GPE)
                        conf->dst_port = htons(IANA_VXLAN_GPE_UDP_PORT);
                else
                        conf->dst_port = htons(vxlan_port);
        }

        if (!conf->age_interval)
                conf->age_interval = FDB_AGE_DEFAULT;

        if (vxlan_vni_in_use(src_net, old, conf, conf->vni)) {
                NL_SET_ERR_MSG(extack,
                               "A VXLAN device with the specified VNI already exists");
                return -EEXIST;
        }

        return 0;
}

static void vxlan_config_apply(struct net_device *dev,
                               struct vxlan_config *conf,
                               struct net_device *lowerdev,
                               struct net *src_net,
                               bool changelink)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst *dst = &vxlan->default_dst;
        unsigned short needed_headroom = ETH_HLEN;
        int max_mtu = ETH_MAX_MTU;
        u32 flags = conf->flags;

        if (!changelink) {
                if (flags & VXLAN_F_GPE)
                        vxlan_raw_setup(dev);
                else
                        vxlan_ether_setup(dev);

                if (conf->mtu)
                        dev->mtu = conf->mtu;

                vxlan->net = src_net;
        }

        dst->remote_vni = conf->vni;

        memcpy(&dst->remote_ip, &conf->remote_ip, sizeof(conf->remote_ip));

        if (lowerdev) {
                dst->remote_ifindex = conf->remote_ifindex;

                netif_inherit_tso_max(dev, lowerdev);

                needed_headroom = lowerdev->hard_header_len;
                needed_headroom += lowerdev->needed_headroom;

                dev->needed_tailroom = lowerdev->needed_tailroom;

                max_mtu = lowerdev->mtu - vxlan_headroom(flags);
                if (max_mtu < ETH_MIN_MTU)
                        max_mtu = ETH_MIN_MTU;

                if (!changelink && !conf->mtu)
                        dev->mtu = max_mtu;
        }

        if (dev->mtu > max_mtu)
                dev->mtu = max_mtu;

        if (flags & VXLAN_F_COLLECT_METADATA)
                flags |= VXLAN_F_IPV6;
        needed_headroom += vxlan_headroom(flags);
        dev->needed_headroom = needed_headroom;

        memcpy(&vxlan->cfg, conf, sizeof(*conf));
}

static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
                               struct vxlan_config *conf,
                               struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct net_device *lowerdev;
        int ret;

        ret = vxlan_config_validate(src_net, conf, &lowerdev, vxlan, extack);
        if (ret)
                return ret;

        vxlan_config_apply(dev, conf, lowerdev, src_net, false);

        return 0;
}

static int __vxlan_dev_create(struct net *net, struct net_device *dev,
                              struct vxlan_config *conf,
                              struct netlink_ext_ack *extack)
{
        struct vxlan_net *vn = net_generic(net, vxlan_net_id);
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct net_device *remote_dev = NULL;
        struct vxlan_rdst *dst;
        int err;

        dst = &vxlan->default_dst;
        err = vxlan_dev_configure(net, dev, conf, extack);
        if (err)
                return err;

        dev->ethtool_ops = &vxlan_ethtool_ops;

        err = register_netdevice(dev);
        if (err)
                return err;

        if (dst->remote_ifindex) {
                remote_dev = __dev_get_by_index(net, dst->remote_ifindex);
                if (!remote_dev) {
                        err = -ENODEV;
                        goto unregister;
                }

                err = netdev_upper_dev_link(remote_dev, dev, extack);
                if (err)
                        goto unregister;

                dst->remote_dev = remote_dev;
        }

        err = rtnl_configure_link(dev, NULL, 0, NULL);
        if (err < 0)
                goto unlink;

        /* create an fdb entry for a valid default destination */
        if (!vxlan_addr_any(&dst->remote_ip)) {
                spin_lock_bh(&vxlan->hash_lock);
                err = vxlan_fdb_update(vxlan, all_zeros_mac,
                                       &dst->remote_ip,
                                       NUD_REACHABLE | NUD_PERMANENT,
                                       NLM_F_EXCL | NLM_F_CREATE,
                                       vxlan->cfg.dst_port,
                                       dst->remote_vni,
                                       dst->remote_vni,
                                       dst->remote_ifindex,
                                       NTF_SELF, 0, true, extack);
                spin_unlock_bh(&vxlan->hash_lock);
                if (err)
                        goto unlink;
        }

        list_add(&vxlan->next, &vn->vxlan_list);

        return 0;

unlink:
        if (remote_dev)
                netdev_upper_dev_unlink(remote_dev, dev);
unregister:
        unregister_netdevice(dev);
        return err;
}

/* Set/clear flags based on attribute */
static int vxlan_nl2flag(struct vxlan_config *conf, struct nlattr *tb[],
                          int attrtype, unsigned long mask, bool changelink,
                          bool changelink_supported,
                          struct netlink_ext_ack *extack)
{
        unsigned long flags;

        if (!tb[attrtype])
                return 0;

        if (changelink && !changelink_supported) {
                vxlan_flag_attr_error(attrtype, extack);
                return -EOPNOTSUPP;
        }

        if (vxlan_policy[attrtype].type == NLA_FLAG)
                flags = conf->flags | mask;
        else if (nla_get_u8(tb[attrtype]))
                flags = conf->flags | mask;
        else
                flags = conf->flags & ~mask;

        conf->flags = flags;

        return 0;
}

static int vxlan_nl2conf(struct nlattr *tb[], struct nlattr *data[],
                         struct net_device *dev, struct vxlan_config *conf,
                         bool changelink, struct netlink_ext_ack *extack)
{
        struct vxlanhdr used_bits = {
                .vx_flags = VXLAN_HF_VNI,
                .vx_vni = VXLAN_VNI_MASK,
        };
        struct vxlan_dev *vxlan = netdev_priv(dev);
        int err = 0;

        memset(conf, 0, sizeof(*conf));

        /* if changelink operation, start with old existing cfg */
        if (changelink)
                memcpy(conf, &vxlan->cfg, sizeof(*conf));

        if (data[IFLA_VXLAN_ID]) {
                __be32 vni = cpu_to_be32(nla_get_u32(data[IFLA_VXLAN_ID]));

                if (changelink && (vni != conf->vni)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_ID], "Cannot change VNI");
                        return -EOPNOTSUPP;
                }
                conf->vni = vni;
        }

        if (data[IFLA_VXLAN_GROUP]) {
                if (changelink && (conf->remote_ip.sa.sa_family != AF_INET)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_GROUP], "New group address family does not match old group");
                        return -EOPNOTSUPP;
                }

                conf->remote_ip.sin.sin_addr.s_addr = nla_get_in_addr(data[IFLA_VXLAN_GROUP]);
                conf->remote_ip.sa.sa_family = AF_INET;
        } else if (data[IFLA_VXLAN_GROUP6]) {
                if (!IS_ENABLED(CONFIG_IPV6)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_GROUP6], "IPv6 support not enabled in the kernel");
                        return -EPFNOSUPPORT;
                }

                if (changelink && (conf->remote_ip.sa.sa_family != AF_INET6)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_GROUP6], "New group address family does not match old group");
                        return -EOPNOTSUPP;
                }

                conf->remote_ip.sin6.sin6_addr = nla_get_in6_addr(data[IFLA_VXLAN_GROUP6]);
                conf->remote_ip.sa.sa_family = AF_INET6;
        }

        if (data[IFLA_VXLAN_LOCAL]) {
                if (changelink && (conf->saddr.sa.sa_family != AF_INET)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_LOCAL], "New local address family does not match old");
                        return -EOPNOTSUPP;
                }

                conf->saddr.sin.sin_addr.s_addr = nla_get_in_addr(data[IFLA_VXLAN_LOCAL]);
                conf->saddr.sa.sa_family = AF_INET;
        } else if (data[IFLA_VXLAN_LOCAL6]) {
                if (!IS_ENABLED(CONFIG_IPV6)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_LOCAL6], "IPv6 support not enabled in the kernel");
                        return -EPFNOSUPPORT;
                }

                if (changelink && (conf->saddr.sa.sa_family != AF_INET6)) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_LOCAL6], "New local address family does not match old");
                        return -EOPNOTSUPP;
                }

                /* TODO: respect scope id */
                conf->saddr.sin6.sin6_addr = nla_get_in6_addr(data[IFLA_VXLAN_LOCAL6]);
                conf->saddr.sa.sa_family = AF_INET6;
        }

        if (data[IFLA_VXLAN_LINK])
                conf->remote_ifindex = nla_get_u32(data[IFLA_VXLAN_LINK]);

        if (data[IFLA_VXLAN_TOS])
                conf->tos  = nla_get_u8(data[IFLA_VXLAN_TOS]);

        if (data[IFLA_VXLAN_TTL])
                conf->ttl = nla_get_u8(data[IFLA_VXLAN_TTL]);

        if (data[IFLA_VXLAN_TTL_INHERIT]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_TTL_INHERIT,
                                    VXLAN_F_TTL_INHERIT, changelink, false,
                                    extack);
                if (err)
                        return err;

        }

        if (data[IFLA_VXLAN_LABEL])
                conf->label = nla_get_be32(data[IFLA_VXLAN_LABEL]) &
                             IPV6_FLOWLABEL_MASK;
        if (data[IFLA_VXLAN_LABEL_POLICY])
                conf->label_policy = nla_get_u32(data[IFLA_VXLAN_LABEL_POLICY]);

        if (data[IFLA_VXLAN_LEARNING]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_LEARNING,
                                    VXLAN_F_LEARN, changelink, true,
                                    extack);
                if (err)
                        return err;
        } else if (!changelink) {
                /* default to learn on a new device */
                conf->flags |= VXLAN_F_LEARN;
        }

        if (data[IFLA_VXLAN_AGEING])
                conf->age_interval = nla_get_u32(data[IFLA_VXLAN_AGEING]);

        if (data[IFLA_VXLAN_PROXY]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_PROXY,
                                    VXLAN_F_PROXY, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_RSC]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_RSC,
                                    VXLAN_F_RSC, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_L2MISS]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_L2MISS,
                                    VXLAN_F_L2MISS, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_L3MISS]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_L3MISS,
                                    VXLAN_F_L3MISS, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_LIMIT]) {
                if (changelink) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_LIMIT],
                                            "Cannot change limit");
                        return -EOPNOTSUPP;
                }
                conf->addrmax = nla_get_u32(data[IFLA_VXLAN_LIMIT]);
        }

        if (data[IFLA_VXLAN_COLLECT_METADATA]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_COLLECT_METADATA,
                                    VXLAN_F_COLLECT_METADATA, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_PORT_RANGE]) {
                if (!changelink) {
                        const struct ifla_vxlan_port_range *p
                                = nla_data(data[IFLA_VXLAN_PORT_RANGE]);
                        conf->port_min = ntohs(p->low);
                        conf->port_max = ntohs(p->high);
                } else {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_PORT_RANGE],
                                            "Cannot change port range");
                        return -EOPNOTSUPP;
                }
        }

        if (data[IFLA_VXLAN_PORT]) {
                if (changelink) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_PORT],
                                            "Cannot change port");
                        return -EOPNOTSUPP;
                }
                conf->dst_port = nla_get_be16(data[IFLA_VXLAN_PORT]);
        }

        if (data[IFLA_VXLAN_UDP_CSUM]) {
                if (changelink) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_UDP_CSUM],
                                            "Cannot change UDP_CSUM flag");
                        return -EOPNOTSUPP;
                }
                if (!nla_get_u8(data[IFLA_VXLAN_UDP_CSUM]))
                        conf->flags |= VXLAN_F_UDP_ZERO_CSUM_TX;
        }

        if (data[IFLA_VXLAN_LOCALBYPASS]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_LOCALBYPASS,
                                    VXLAN_F_LOCALBYPASS, changelink,
                                    true, extack);
                if (err)
                        return err;
        } else if (!changelink) {
                /* default to local bypass on a new device */
                conf->flags |= VXLAN_F_LOCALBYPASS;
        }

        if (data[IFLA_VXLAN_UDP_ZERO_CSUM6_TX]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
                                    VXLAN_F_UDP_ZERO_CSUM6_TX, changelink,
                                    false, extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_UDP_ZERO_CSUM6_RX]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
                                    VXLAN_F_UDP_ZERO_CSUM6_RX, changelink,
                                    false, extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_REMCSUM_TX]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_REMCSUM_TX,
                                    VXLAN_F_REMCSUM_TX, changelink, false,
                                    extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_REMCSUM_RX]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_REMCSUM_RX,
                                    VXLAN_F_REMCSUM_RX, changelink, false,
                                    extack);
                if (err)
                        return err;
                used_bits.vx_flags |= VXLAN_HF_RCO;
                used_bits.vx_vni |= ~VXLAN_VNI_MASK;
        }

        if (data[IFLA_VXLAN_GBP]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_GBP,
                                    VXLAN_F_GBP, changelink, false, extack);
                if (err)
                        return err;
                used_bits.vx_flags |= VXLAN_GBP_USED_BITS;
        }

        if (data[IFLA_VXLAN_GPE]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_GPE,
                                    VXLAN_F_GPE, changelink, false,
                                    extack);
                if (err)
                        return err;

                used_bits.vx_flags |= VXLAN_GPE_USED_BITS;
        }

        if (data[IFLA_VXLAN_RESERVED_BITS]) {
                struct vxlanhdr reserved_bits;

                if (changelink) {
                        NL_SET_ERR_MSG_ATTR(extack,
                                            data[IFLA_VXLAN_RESERVED_BITS],
                                            "Cannot change reserved_bits");
                        return -EOPNOTSUPP;
                }

                nla_memcpy(&reserved_bits, data[IFLA_VXLAN_RESERVED_BITS],
                           sizeof(reserved_bits));
                if (used_bits.vx_flags & reserved_bits.vx_flags ||
                    used_bits.vx_vni & reserved_bits.vx_vni) {
                        __be64 ub_be64, rb_be64;

                        memcpy(&ub_be64, &used_bits, sizeof(ub_be64));
                        memcpy(&rb_be64, &reserved_bits, sizeof(rb_be64));

                        NL_SET_ERR_MSG_ATTR_FMT(extack,
                                                data[IFLA_VXLAN_RESERVED_BITS],
                                                "Used bits %#018llx cannot overlap reserved bits %#018llx",
                                                be64_to_cpu(ub_be64),
                                                be64_to_cpu(rb_be64));
                        return -EINVAL;
                }

                conf->reserved_bits = reserved_bits;
        } else {
                /* For backwards compatibility, only allow reserved fields to be
                 * used by VXLAN extensions if explicitly requested.
                 */
                conf->reserved_bits = (struct vxlanhdr) {
                        .vx_flags = ~used_bits.vx_flags,
                        .vx_vni = ~used_bits.vx_vni,
                };
        }

        if (data[IFLA_VXLAN_REMCSUM_NOPARTIAL]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_REMCSUM_NOPARTIAL,
                                    VXLAN_F_REMCSUM_NOPARTIAL, changelink,
                                    false, extack);
                if (err)
                        return err;
        }

        if (data[IFLA_VXLAN_MC_ROUTE]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_MC_ROUTE,
                                    VXLAN_F_MC_ROUTE, changelink,
                                    true, extack);
                if (err)
                        return err;
        }

        if (tb[IFLA_MTU]) {
                if (changelink) {
                        NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_MTU],
                                            "Cannot change mtu");
                        return -EOPNOTSUPP;
                }
                conf->mtu = nla_get_u32(tb[IFLA_MTU]);
        }

        if (data[IFLA_VXLAN_DF])
                conf->df = nla_get_u8(data[IFLA_VXLAN_DF]);

        if (data[IFLA_VXLAN_VNIFILTER]) {
                err = vxlan_nl2flag(conf, data, IFLA_VXLAN_VNIFILTER,
                                    VXLAN_F_VNIFILTER, changelink, false,
                                    extack);
                if (err)
                        return err;

                if ((conf->flags & VXLAN_F_VNIFILTER) &&
                    !(conf->flags & VXLAN_F_COLLECT_METADATA)) {
                        NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VXLAN_VNIFILTER],
                                            "vxlan vnifilter only valid in collect metadata mode");
                        return -EINVAL;
                }
        }

        return 0;
}

static int vxlan_newlink(struct net_device *dev,
                         struct rtnl_newlink_params *params,
                         struct netlink_ext_ack *extack)
{
        struct net *link_net = rtnl_newlink_link_net(params);
        struct nlattr **data = params->data;
        struct nlattr **tb = params->tb;
        struct vxlan_config conf;
        int err;

        err = vxlan_nl2conf(tb, data, dev, &conf, false, extack);
        if (err)
                return err;

        return __vxlan_dev_create(link_net, dev, &conf, extack);
}

static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
                            struct nlattr *data[],
                            struct netlink_ext_ack *extack)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        bool rem_ip_changed, change_igmp;
        struct net_device *lowerdev;
        struct vxlan_config conf;
        struct vxlan_rdst *dst;
        int err;

        dst = &vxlan->default_dst;
        err = vxlan_nl2conf(tb, data, dev, &conf, true, extack);
        if (err)
                return err;

        err = vxlan_config_validate(vxlan->net, &conf, &lowerdev,
                                    vxlan, extack);
        if (err)
                return err;

        if (dst->remote_dev == lowerdev)
                lowerdev = NULL;

        err = netdev_adjacent_change_prepare(dst->remote_dev, lowerdev, dev,
                                             extack);
        if (err)
                return err;

        rem_ip_changed = !vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip);
        change_igmp = vxlan->dev->flags & IFF_UP &&
                      (rem_ip_changed ||
                       dst->remote_ifindex != conf.remote_ifindex);

        /* handle default dst entry */
        if (rem_ip_changed) {
                spin_lock_bh(&vxlan->hash_lock);
                if (!vxlan_addr_any(&conf.remote_ip)) {
                        err = vxlan_fdb_update(vxlan, all_zeros_mac,
                                               &conf.remote_ip,
                                               NUD_REACHABLE | NUD_PERMANENT,
                                               NLM_F_APPEND | NLM_F_CREATE,
                                               vxlan->cfg.dst_port,
                                               conf.vni, conf.vni,
                                               conf.remote_ifindex,
                                               NTF_SELF, 0, true, extack);
                        if (err) {
                                spin_unlock_bh(&vxlan->hash_lock);
                                netdev_adjacent_change_abort(dst->remote_dev,
                                                             lowerdev, dev);
                                return err;
                        }
                }
                if (!vxlan_addr_any(&dst->remote_ip))
                        __vxlan_fdb_delete(vxlan, all_zeros_mac,
                                           dst->remote_ip,
                                           vxlan->cfg.dst_port,
                                           dst->remote_vni,
                                           dst->remote_vni,
                                           dst->remote_ifindex,
                                           true);
                spin_unlock_bh(&vxlan->hash_lock);

                /* If vni filtering device, also update fdb entries of
                 * all vnis that were using default remote ip
                 */
                if (vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
                        err = vxlan_vnilist_update_group(vxlan, &dst->remote_ip,
                                                         &conf.remote_ip, extack);
                        if (err) {
                                netdev_adjacent_change_abort(dst->remote_dev,
                                                             lowerdev, dev);
                                return err;
                        }
                }
        }

        if (change_igmp && vxlan_addr_multicast(&dst->remote_ip))
                err = vxlan_multicast_leave(vxlan);

        if (conf.age_interval != vxlan->cfg.age_interval)
                mod_timer(&vxlan->age_timer, jiffies);

        netdev_adjacent_change_commit(dst->remote_dev, lowerdev, dev);
        if (lowerdev && lowerdev != dst->remote_dev)
                dst->remote_dev = lowerdev;
        vxlan_config_apply(dev, &conf, lowerdev, vxlan->net, true);

        if (!err && change_igmp &&
            vxlan_addr_multicast(&dst->remote_ip))
                err = vxlan_multicast_join(vxlan);

        return err;
}

static void vxlan_dellink(struct net_device *dev, struct list_head *head)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb_flush_desc desc = {};

        vxlan_flush(vxlan, &desc);

        list_del(&vxlan->next);
        unregister_netdevice_queue(dev, head);
        if (vxlan->default_dst.remote_dev)
                netdev_upper_dev_unlink(vxlan->default_dst.remote_dev, dev);
}

static size_t vxlan_get_size(const struct net_device *dev)
{
        return nla_total_size(sizeof(__u32)) +        /* IFLA_VXLAN_ID */
                nla_total_size(sizeof(struct in6_addr)) + /* IFLA_VXLAN_GROUP{6} */
                nla_total_size(sizeof(__u32)) +        /* IFLA_VXLAN_LINK */
                nla_total_size(sizeof(struct in6_addr)) + /* IFLA_VXLAN_LOCAL{6} */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_TTL */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_TTL_INHERIT */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_TOS */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_DF */
                nla_total_size(sizeof(__be32)) + /* IFLA_VXLAN_LABEL */
                nla_total_size(sizeof(__u32)) +  /* IFLA_VXLAN_LABEL_POLICY */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_LEARNING */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_PROXY */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_RSC */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_L2MISS */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_L3MISS */
                nla_total_size(sizeof(__u8)) +        /* IFLA_VXLAN_COLLECT_METADATA */
                nla_total_size(sizeof(__u32)) +        /* IFLA_VXLAN_AGEING */
                nla_total_size(sizeof(__u32)) +        /* IFLA_VXLAN_LIMIT */
                nla_total_size(sizeof(__be16)) + /* IFLA_VXLAN_PORT */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_CSUM */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_TX */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_RX */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_TX */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_RX */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_LOCALBYPASS */
                /* IFLA_VXLAN_PORT_RANGE */
                nla_total_size(sizeof(struct ifla_vxlan_port_range)) +
                nla_total_size(0) + /* IFLA_VXLAN_GBP */
                nla_total_size(0) + /* IFLA_VXLAN_GPE */
                nla_total_size(0) + /* IFLA_VXLAN_REMCSUM_NOPARTIAL */
                nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_VNIFILTER */
                /* IFLA_VXLAN_RESERVED_BITS */
                nla_total_size(sizeof(struct vxlanhdr)) +
                0;
}

static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        const struct vxlan_dev *vxlan = netdev_priv(dev);
        const struct vxlan_rdst *dst = &vxlan->default_dst;
        struct ifla_vxlan_port_range ports = {
                .low =  htons(vxlan->cfg.port_min),
                .high = htons(vxlan->cfg.port_max),
        };

        if (nla_put_u32(skb, IFLA_VXLAN_ID, be32_to_cpu(dst->remote_vni)))
                goto nla_put_failure;

        if (!vxlan_addr_any(&dst->remote_ip)) {
                if (dst->remote_ip.sa.sa_family == AF_INET) {
                        if (nla_put_in_addr(skb, IFLA_VXLAN_GROUP,
                                            dst->remote_ip.sin.sin_addr.s_addr))
                                goto nla_put_failure;
#if IS_ENABLED(CONFIG_IPV6)
                } else {
                        if (nla_put_in6_addr(skb, IFLA_VXLAN_GROUP6,
                                             &dst->remote_ip.sin6.sin6_addr))
                                goto nla_put_failure;
#endif
                }
        }

        if (dst->remote_ifindex && nla_put_u32(skb, IFLA_VXLAN_LINK, dst->remote_ifindex))
                goto nla_put_failure;

        if (!vxlan_addr_any(&vxlan->cfg.saddr)) {
                if (vxlan->cfg.saddr.sa.sa_family == AF_INET) {
                        if (nla_put_in_addr(skb, IFLA_VXLAN_LOCAL,
                                            vxlan->cfg.saddr.sin.sin_addr.s_addr))
                                goto nla_put_failure;
#if IS_ENABLED(CONFIG_IPV6)
                } else {
                        if (nla_put_in6_addr(skb, IFLA_VXLAN_LOCAL6,
                                             &vxlan->cfg.saddr.sin6.sin6_addr))
                                goto nla_put_failure;
#endif
                }
        }

        if (nla_put_u8(skb, IFLA_VXLAN_TTL, vxlan->cfg.ttl) ||
            nla_put_u8(skb, IFLA_VXLAN_TTL_INHERIT,
                       !!(vxlan->cfg.flags & VXLAN_F_TTL_INHERIT)) ||
            nla_put_u8(skb, IFLA_VXLAN_TOS, vxlan->cfg.tos) ||
            nla_put_u8(skb, IFLA_VXLAN_DF, vxlan->cfg.df) ||
            nla_put_be32(skb, IFLA_VXLAN_LABEL, vxlan->cfg.label) ||
            nla_put_u32(skb, IFLA_VXLAN_LABEL_POLICY, vxlan->cfg.label_policy) ||
            nla_put_u8(skb, IFLA_VXLAN_LEARNING,
                       !!(vxlan->cfg.flags & VXLAN_F_LEARN)) ||
            nla_put_u8(skb, IFLA_VXLAN_PROXY,
                       !!(vxlan->cfg.flags & VXLAN_F_PROXY)) ||
            nla_put_u8(skb, IFLA_VXLAN_RSC,
                       !!(vxlan->cfg.flags & VXLAN_F_RSC)) ||
            nla_put_u8(skb, IFLA_VXLAN_L2MISS,
                       !!(vxlan->cfg.flags & VXLAN_F_L2MISS)) ||
            nla_put_u8(skb, IFLA_VXLAN_L3MISS,
                       !!(vxlan->cfg.flags & VXLAN_F_L3MISS)) ||
            nla_put_u8(skb, IFLA_VXLAN_COLLECT_METADATA,
                       !!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA)) ||
            nla_put_u32(skb, IFLA_VXLAN_AGEING, vxlan->cfg.age_interval) ||
            nla_put_u32(skb, IFLA_VXLAN_LIMIT, vxlan->cfg.addrmax) ||
            nla_put_be16(skb, IFLA_VXLAN_PORT, vxlan->cfg.dst_port) ||
            nla_put_u8(skb, IFLA_VXLAN_UDP_CSUM,
                       !(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM_TX)) ||
            nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
                       !!(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||
            nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
                       !!(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM6_RX)) ||
            nla_put_u8(skb, IFLA_VXLAN_REMCSUM_TX,
                       !!(vxlan->cfg.flags & VXLAN_F_REMCSUM_TX)) ||
            nla_put_u8(skb, IFLA_VXLAN_REMCSUM_RX,
                       !!(vxlan->cfg.flags & VXLAN_F_REMCSUM_RX)) ||
            nla_put_u8(skb, IFLA_VXLAN_LOCALBYPASS,
                       !!(vxlan->cfg.flags & VXLAN_F_LOCALBYPASS)))
                goto nla_put_failure;

        if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
                goto nla_put_failure;

        if (vxlan->cfg.flags & VXLAN_F_GBP &&
            nla_put_flag(skb, IFLA_VXLAN_GBP))
                goto nla_put_failure;

        if (vxlan->cfg.flags & VXLAN_F_GPE &&
            nla_put_flag(skb, IFLA_VXLAN_GPE))
                goto nla_put_failure;

        if (vxlan->cfg.flags & VXLAN_F_REMCSUM_NOPARTIAL &&
            nla_put_flag(skb, IFLA_VXLAN_REMCSUM_NOPARTIAL))
                goto nla_put_failure;

        if (vxlan->cfg.flags & VXLAN_F_VNIFILTER &&
            nla_put_u8(skb, IFLA_VXLAN_VNIFILTER,
                       !!(vxlan->cfg.flags & VXLAN_F_VNIFILTER)))
                goto nla_put_failure;

        if (nla_put(skb, IFLA_VXLAN_RESERVED_BITS,
                    sizeof(vxlan->cfg.reserved_bits),
                    &vxlan->cfg.reserved_bits))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static struct net *vxlan_get_link_net(const struct net_device *dev)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);

        return READ_ONCE(vxlan->net);
}

static struct rtnl_link_ops vxlan_link_ops __read_mostly = {
        .kind                = "vxlan",
        .maxtype        = IFLA_VXLAN_MAX,
        .policy                = vxlan_policy,
        .priv_size        = sizeof(struct vxlan_dev),
        .setup                = vxlan_setup,
        .validate        = vxlan_validate,
        .newlink        = vxlan_newlink,
        .changelink        = vxlan_changelink,
        .dellink        = vxlan_dellink,
        .get_size        = vxlan_get_size,
        .fill_info        = vxlan_fill_info,
        .get_link_net        = vxlan_get_link_net,
};

struct net_device *vxlan_dev_create(struct net *net, const char *name,
                                    u8 name_assign_type,
                                    struct vxlan_config *conf)
{
        struct nlattr *tb[IFLA_MAX + 1];
        struct net_device *dev;
        int err;

        memset(&tb, 0, sizeof(tb));

        dev = rtnl_create_link(net, name, name_assign_type,
                               &vxlan_link_ops, tb, NULL);
        if (IS_ERR(dev))
                return dev;

        err = __vxlan_dev_create(net, dev, conf, NULL);
        if (err < 0) {
                free_netdev(dev);
                return ERR_PTR(err);
        }

        err = rtnl_configure_link(dev, NULL, 0, NULL);
        if (err < 0) {
                LIST_HEAD(list_kill);

                vxlan_dellink(dev, &list_kill);
                unregister_netdevice_many(&list_kill);
                return ERR_PTR(err);
        }

        return dev;
}
EXPORT_SYMBOL_GPL(vxlan_dev_create);

static void vxlan_handle_lowerdev_unregister(struct vxlan_net *vn,
                                             struct net_device *dev)
{
        struct vxlan_dev *vxlan, *next;
        LIST_HEAD(list_kill);

        list_for_each_entry_safe(vxlan, next, &vn->vxlan_list, next) {
                struct vxlan_rdst *dst = &vxlan->default_dst;

                /* In case we created vxlan device with carrier
                 * and we loose the carrier due to module unload
                 * we also need to remove vxlan device. In other
                 * cases, it's not necessary and remote_ifindex
                 * is 0 here, so no matches.
                 */
                if (dst->remote_ifindex == dev->ifindex)
                        vxlan_dellink(vxlan->dev, &list_kill);
        }

        unregister_netdevice_many(&list_kill);
}

static int vxlan_netdevice_event(struct notifier_block *unused,
                                 unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);

        if (event == NETDEV_UNREGISTER)
                vxlan_handle_lowerdev_unregister(vn, dev);
        else if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
                vxlan_offload_rx_ports(dev, true);
        else if (event == NETDEV_UDP_TUNNEL_DROP_INFO)
                vxlan_offload_rx_ports(dev, false);

        return NOTIFY_DONE;
}

static struct notifier_block vxlan_notifier_block __read_mostly = {
        .notifier_call = vxlan_netdevice_event,
};

static void
vxlan_fdb_offloaded_set(struct net_device *dev,
                        struct switchdev_notifier_vxlan_fdb_info *fdb_info)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_rdst *rdst;
        struct vxlan_fdb *f;

        spin_lock_bh(&vxlan->hash_lock);

        f = vxlan_find_mac(vxlan, fdb_info->eth_addr, fdb_info->vni);
        if (!f)
                goto out;

        rdst = vxlan_fdb_find_rdst(f, &fdb_info->remote_ip,
                                   fdb_info->remote_port,
                                   fdb_info->remote_vni,
                                   fdb_info->remote_ifindex);
        if (!rdst)
                goto out;

        rdst->offloaded = fdb_info->offloaded;

out:
        spin_unlock_bh(&vxlan->hash_lock);
}

static int
vxlan_fdb_external_learn_add(struct net_device *dev,
                             struct switchdev_notifier_vxlan_fdb_info *fdb_info)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct netlink_ext_ack *extack;
        int err;

        extack = switchdev_notifier_info_to_extack(&fdb_info->info);

        spin_lock_bh(&vxlan->hash_lock);
        err = vxlan_fdb_update(vxlan, fdb_info->eth_addr, &fdb_info->remote_ip,
                               NUD_REACHABLE,
                               NLM_F_CREATE | NLM_F_REPLACE,
                               fdb_info->remote_port,
                               fdb_info->vni,
                               fdb_info->remote_vni,
                               fdb_info->remote_ifindex,
                               NTF_USE | NTF_SELF | NTF_EXT_LEARNED,
                               0, false, extack);
        spin_unlock_bh(&vxlan->hash_lock);

        return err;
}

static int
vxlan_fdb_external_learn_del(struct net_device *dev,
                             struct switchdev_notifier_vxlan_fdb_info *fdb_info)
{
        struct vxlan_dev *vxlan = netdev_priv(dev);
        struct vxlan_fdb *f;
        int err = 0;

        spin_lock_bh(&vxlan->hash_lock);

        f = vxlan_find_mac(vxlan, fdb_info->eth_addr, fdb_info->vni);
        if (!f)
                err = -ENOENT;
        else if (f->flags & NTF_EXT_LEARNED)
                err = __vxlan_fdb_delete(vxlan, fdb_info->eth_addr,
                                         fdb_info->remote_ip,
                                         fdb_info->remote_port,
                                         fdb_info->vni,
                                         fdb_info->remote_vni,
                                         fdb_info->remote_ifindex,
                                         false);

        spin_unlock_bh(&vxlan->hash_lock);

        return err;
}

static int vxlan_switchdev_event(struct notifier_block *unused,
                                 unsigned long event, void *ptr)
{
        struct net_device *dev = switchdev_notifier_info_to_dev(ptr);
        struct switchdev_notifier_vxlan_fdb_info *fdb_info;
        int err = 0;

        switch (event) {
        case SWITCHDEV_VXLAN_FDB_OFFLOADED:
                vxlan_fdb_offloaded_set(dev, ptr);
                break;
        case SWITCHDEV_VXLAN_FDB_ADD_TO_BRIDGE:
                fdb_info = ptr;
                err = vxlan_fdb_external_learn_add(dev, fdb_info);
                if (err) {
                        err = notifier_from_errno(err);
                        break;
                }
                fdb_info->offloaded = true;
                vxlan_fdb_offloaded_set(dev, fdb_info);
                break;
        case SWITCHDEV_VXLAN_FDB_DEL_TO_BRIDGE:
                fdb_info = ptr;
                err = vxlan_fdb_external_learn_del(dev, fdb_info);
                if (err) {
                        err = notifier_from_errno(err);
                        break;
                }
                fdb_info->offloaded = false;
                vxlan_fdb_offloaded_set(dev, fdb_info);
                break;
        }

        return err;
}

static struct notifier_block vxlan_switchdev_notifier_block __read_mostly = {
        .notifier_call = vxlan_switchdev_event,
};

static void vxlan_fdb_nh_flush(struct nexthop *nh)
{
        struct vxlan_fdb *fdb;
        struct vxlan_dev *vxlan;

        rcu_read_lock();
        list_for_each_entry_rcu(fdb, &nh->fdb_list, nh_list) {
                vxlan = rcu_dereference(fdb->vdev);
                WARN_ON(!vxlan);
                spin_lock_bh(&vxlan->hash_lock);
                if (!hlist_unhashed(&fdb->fdb_node))
                        vxlan_fdb_destroy(vxlan, fdb, false, false);
                spin_unlock_bh(&vxlan->hash_lock);
        }
        rcu_read_unlock();
}

static int vxlan_nexthop_event(struct notifier_block *nb,
                               unsigned long event, void *ptr)
{
        struct nh_notifier_info *info = ptr;
        struct nexthop *nh;

        if (event != NEXTHOP_EVENT_DEL)
                return NOTIFY_DONE;

        nh = nexthop_find_by_id(info->net, info->id);
        if (!nh)
                return NOTIFY_DONE;

        vxlan_fdb_nh_flush(nh);

        return NOTIFY_DONE;
}

static __net_init int vxlan_init_net(struct net *net)
{
        struct vxlan_net *vn = net_generic(net, vxlan_net_id);
        unsigned int h;

        INIT_LIST_HEAD(&vn->vxlan_list);
        vn->nexthop_notifier_block.notifier_call = vxlan_nexthop_event;

        for (h = 0; h < PORT_HASH_SIZE; ++h)
                INIT_HLIST_HEAD(&vn->sock_list[h]);

        return register_nexthop_notifier(net, &vn->nexthop_notifier_block,
                                         NULL);
}

static void __net_exit vxlan_destroy_tunnels(struct vxlan_net *vn,
                                             struct list_head *dev_to_kill)
{
        struct vxlan_dev *vxlan, *next;

        list_for_each_entry_safe(vxlan, next, &vn->vxlan_list, next)
                vxlan_dellink(vxlan->dev, dev_to_kill);
}

static void __net_exit vxlan_exit_rtnl(struct net *net,
                                       struct list_head *dev_to_kill)
{
        struct vxlan_net *vn = net_generic(net, vxlan_net_id);

        ASSERT_RTNL_NET(net);

        __unregister_nexthop_notifier(net, &vn->nexthop_notifier_block);
        vxlan_destroy_tunnels(vn, dev_to_kill);
}

static void __net_exit vxlan_exit_net(struct net *net)
{
        struct vxlan_net *vn = net_generic(net, vxlan_net_id);
        unsigned int h;

        for (h = 0; h < PORT_HASH_SIZE; ++h)
                WARN_ON_ONCE(!hlist_empty(&vn->sock_list[h]));
}

static struct pernet_operations vxlan_net_ops = {
        .init = vxlan_init_net,
        .exit_rtnl = vxlan_exit_rtnl,
        .exit = vxlan_exit_net,
        .id   = &vxlan_net_id,
        .size = sizeof(struct vxlan_net),
};

static int __init vxlan_init_module(void)
{
        int rc;

        rc = register_pernet_subsys(&vxlan_net_ops);
        if (rc)
                goto out1;

        rc = register_netdevice_notifier(&vxlan_notifier_block);
        if (rc)
                goto out2;

        rc = register_switchdev_notifier(&vxlan_switchdev_notifier_block);
        if (rc)
                goto out3;

        rc = rtnl_link_register(&vxlan_link_ops);
        if (rc)
                goto out4;

        rc = vxlan_vnifilter_init();
        if (rc)
                goto out5;

        return 0;
out5:
        rtnl_link_unregister(&vxlan_link_ops);
out4:
        unregister_switchdev_notifier(&vxlan_switchdev_notifier_block);
out3:
        unregister_netdevice_notifier(&vxlan_notifier_block);
out2:
        unregister_pernet_subsys(&vxlan_net_ops);
out1:
        return rc;
}
late_initcall(vxlan_init_module);

static void __exit vxlan_cleanup_module(void)
{
        vxlan_vnifilter_uninit();
        rtnl_link_unregister(&vxlan_link_ops);
        unregister_switchdev_notifier(&vxlan_switchdev_notifier_block);
        unregister_netdevice_notifier(&vxlan_notifier_block);
        unregister_pernet_subsys(&vxlan_net_ops);
        /* rcu_barrier() is called by netns */
}
module_exit(vxlan_cleanup_module);

MODULE_LICENSE("GPL");
MODULE_VERSION(VXLAN_VERSION);
MODULE_AUTHOR("Stephen Hemminger <stephen@networkplumber.org>");
MODULE_DESCRIPTION("Driver for VXLAN encapsulated traffic");
MODULE_ALIAS_RTNL_LINK("vxlan");






















  714 

  711 


















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
// SPDX-License-Identifier: GPL-2.0
/*
 * property.c - Unified device property interface.
 *
 * Copyright (C) 2014, Intel Corporation
 * Authors: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
 *          Mika Westerberg <mika.westerberg@linux.intel.com>
 */

#include <linux/device.h>
#include <linux/err.h>
#include <linux/export.h>
#include <linux/kconfig.h>
#include <linux/of.h>
#include <linux/property.h>
#include <linux/phy.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/types.h>

struct fwnode_handle *__dev_fwnode(struct device *dev)
{
        return IS_ENABLED(CONFIG_OF) && dev->of_node ?
                of_fwnode_handle(dev->of_node) : dev->fwnode;
}
EXPORT_SYMBOL_GPL(__dev_fwnode);

const struct fwnode_handle *__dev_fwnode_const(const struct device *dev)
{
        return IS_ENABLED(CONFIG_OF) && dev->of_node ?
                of_fwnode_handle(dev->of_node) : dev->fwnode;
}
EXPORT_SYMBOL_GPL(__dev_fwnode_const);

/**
 * device_property_present - check if a property of a device is present
 * @dev: Device whose property is being checked
 * @propname: Name of the property
 *
 * Check if property @propname is present in the device firmware description.
 *
 * Return: true if property @propname is present. Otherwise, returns false.
 */
bool device_property_present(const struct device *dev, const char *propname)
{
        return fwnode_property_present(dev_fwnode(dev), propname);
}
EXPORT_SYMBOL_GPL(device_property_present);

/**
 * fwnode_property_present - check if a property of a firmware node is present
 * @fwnode: Firmware node whose property to check
 * @propname: Name of the property
 *
 * Return: true if property @propname is present. Otherwise, returns false.
 */
bool fwnode_property_present(const struct fwnode_handle *fwnode,
                             const char *propname)
{
        bool ret;

        if (IS_ERR_OR_NULL(fwnode))
                return false;

        ret = fwnode_call_bool_op(fwnode, property_present, propname);
        if (ret)
                return ret;

        return fwnode_call_bool_op(fwnode->secondary, property_present, propname);
}
EXPORT_SYMBOL_GPL(fwnode_property_present);

/**
 * device_property_read_bool - Return the value for a boolean property of a device
 * @dev: Device whose property is being checked
 * @propname: Name of the property
 *
 * Return if property @propname is true or false in the device firmware description.
 *
 * Return: true if property @propname is present. Otherwise, returns false.
 */
bool device_property_read_bool(const struct device *dev, const char *propname)
{
        return fwnode_property_read_bool(dev_fwnode(dev), propname);
}
EXPORT_SYMBOL_GPL(device_property_read_bool);

/**
 * fwnode_property_read_bool - Return the value for a boolean property of a firmware node
 * @fwnode: Firmware node whose property to check
 * @propname: Name of the property
 *
 * Return if property @propname is true or false in the firmware description.
 */
bool fwnode_property_read_bool(const struct fwnode_handle *fwnode,
                             const char *propname)
{
        bool ret;

        if (IS_ERR_OR_NULL(fwnode))
                return false;

        ret = fwnode_call_bool_op(fwnode, property_read_bool, propname);
        if (ret)
                return ret;

        return fwnode_call_bool_op(fwnode->secondary, property_read_bool, propname);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_bool);

/**
 * device_property_read_u8_array - return a u8 array property of a device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Function reads an array of u8 properties with @propname from the device
 * firmware description and stores them to @val if found.
 *
 * It's recommended to call device_property_count_u8() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_u8_array(const struct device *dev, const char *propname,
                                  u8 *val, size_t nval)
{
        return fwnode_property_read_u8_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u8_array);

/**
 * device_property_read_u16_array - return a u16 array property of a device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Function reads an array of u16 properties with @propname from the device
 * firmware description and stores them to @val if found.
 *
 * It's recommended to call device_property_count_u16() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_u16_array(const struct device *dev, const char *propname,
                                   u16 *val, size_t nval)
{
        return fwnode_property_read_u16_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u16_array);

/**
 * device_property_read_u32_array - return a u32 array property of a device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Function reads an array of u32 properties with @propname from the device
 * firmware description and stores them to @val if found.
 *
 * It's recommended to call device_property_count_u32() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_u32_array(const struct device *dev, const char *propname,
                                   u32 *val, size_t nval)
{
        return fwnode_property_read_u32_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u32_array);

/**
 * device_property_read_u64_array - return a u64 array property of a device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Function reads an array of u64 properties with @propname from the device
 * firmware description and stores them to @val if found.
 *
 * It's recommended to call device_property_count_u64() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_u64_array(const struct device *dev, const char *propname,
                                   u64 *val, size_t nval)
{
        return fwnode_property_read_u64_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_u64_array);

/**
 * device_property_read_string_array - return a string array property of device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Function reads an array of string properties with @propname from the device
 * firmware description and stores them to @val if found.
 *
 * It's recommended to call device_property_string_array_count() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values read on success if @val is non-NULL,
 *           number of values available on success if @val is NULL,
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO or %-EILSEQ if the property is not an array of strings,
 *           %-EOVERFLOW if the size of the property is not as expected.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_string_array(const struct device *dev, const char *propname,
                                      const char **val, size_t nval)
{
        return fwnode_property_read_string_array(dev_fwnode(dev), propname, val, nval);
}
EXPORT_SYMBOL_GPL(device_property_read_string_array);

/**
 * device_property_read_string - return a string property of a device
 * @dev: Device to get the property of
 * @propname: Name of the property
 * @val: The value is stored here
 *
 * Function reads property @propname from the device firmware description and
 * stores the value into @val if found. The value is checked to be a string.
 *
 * Return: %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO or %-EILSEQ if the property type is not a string.
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_read_string(const struct device *dev, const char *propname,
                                const char **val)
{
        return fwnode_property_read_string(dev_fwnode(dev), propname, val);
}
EXPORT_SYMBOL_GPL(device_property_read_string);

/**
 * device_property_match_string - find a string in an array and return index
 * @dev: Device to get the property of
 * @propname: Name of the property holding the array
 * @string: String to look for
 *
 * Find a given string in a string array and if it is found return the
 * index back.
 *
 * Return: index, starting from %0, if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of strings,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int device_property_match_string(const struct device *dev, const char *propname,
                                 const char *string)
{
        return fwnode_property_match_string(dev_fwnode(dev), propname, string);
}
EXPORT_SYMBOL_GPL(device_property_match_string);

static int fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
                                          const char *propname,
                                          unsigned int elem_size, void *val,
                                          size_t nval)
{
        int ret;

        if (IS_ERR_OR_NULL(fwnode))
                return -EINVAL;

        ret = fwnode_call_int_op(fwnode, property_read_int_array, propname,
                                 elem_size, val, nval);
        if (ret != -EINVAL)
                return ret;

        return fwnode_call_int_op(fwnode->secondary, property_read_int_array, propname,
                                  elem_size, val, nval);
}

/**
 * fwnode_property_read_u8_array - return a u8 array property of firmware node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Read an array of u8 properties with @propname from @fwnode and stores them to
 * @val if found.
 *
 * It's recommended to call fwnode_property_count_u8() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_u8_array(const struct fwnode_handle *fwnode,
                                  const char *propname, u8 *val, size_t nval)
{
        return fwnode_property_read_int_array(fwnode, propname, sizeof(u8),
                                              val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u8_array);

/**
 * fwnode_property_read_u16_array - return a u16 array property of firmware node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Read an array of u16 properties with @propname from @fwnode and store them to
 * @val if found.
 *
 * It's recommended to call fwnode_property_count_u16() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_u16_array(const struct fwnode_handle *fwnode,
                                   const char *propname, u16 *val, size_t nval)
{
        return fwnode_property_read_int_array(fwnode, propname, sizeof(u16),
                                              val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u16_array);

/**
 * fwnode_property_read_u32_array - return a u32 array property of firmware node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Read an array of u32 properties with @propname from @fwnode store them to
 * @val if found.
 *
 * It's recommended to call fwnode_property_count_u32() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_u32_array(const struct fwnode_handle *fwnode,
                                   const char *propname, u32 *val, size_t nval)
{
        return fwnode_property_read_int_array(fwnode, propname, sizeof(u32),
                                              val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u32_array);

/**
 * fwnode_property_read_u64_array - return a u64 array property firmware node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Read an array of u64 properties with @propname from @fwnode and store them to
 * @val if found.
 *
 * It's recommended to call fwnode_property_count_u64() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values if @val was %NULL,
 *         %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of numbers,
 *           %-EOVERFLOW if the size of the property is not as expected,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_u64_array(const struct fwnode_handle *fwnode,
                                   const char *propname, u64 *val, size_t nval)
{
        return fwnode_property_read_int_array(fwnode, propname, sizeof(u64),
                                              val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_u64_array);

/**
 * fwnode_property_read_string_array - return string array property of a node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The values are stored here or %NULL to return the number of values
 * @nval: Size of the @val array
 *
 * Read an string list property @propname from the given firmware node and store
 * them to @val if found.
 *
 * It's recommended to call fwnode_property_string_array_count() instead of calling
 * this function with @val equals %NULL and @nval equals 0.
 *
 * Return: number of values read on success if @val is non-NULL,
 *           number of values available on success if @val is NULL,
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO or %-EILSEQ if the property is not an array of strings,
 *           %-EOVERFLOW if the size of the property is not as expected,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
                                      const char *propname, const char **val,
                                      size_t nval)
{
        int ret;

        if (IS_ERR_OR_NULL(fwnode))
                return -EINVAL;

        ret = fwnode_call_int_op(fwnode, property_read_string_array, propname,
                                 val, nval);
        if (ret != -EINVAL)
                return ret;

        return fwnode_call_int_op(fwnode->secondary, property_read_string_array, propname,
                                  val, nval);
}
EXPORT_SYMBOL_GPL(fwnode_property_read_string_array);

/**
 * fwnode_property_read_string - return a string property of a firmware node
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property
 * @val: The value is stored here
 *
 * Read property @propname from the given firmware node and store the value into
 * @val if found.  The value is checked to be a string.
 *
 * Return: %0 if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO or %-EILSEQ if the property is not a string,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_read_string(const struct fwnode_handle *fwnode,
                                const char *propname, const char **val)
{
        int ret = fwnode_property_read_string_array(fwnode, propname, val, 1);

        return ret < 0 ? ret : 0;
}
EXPORT_SYMBOL_GPL(fwnode_property_read_string);

/**
 * fwnode_property_match_string - find a string in an array and return index
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property holding the array
 * @string: String to look for
 *
 * Find a given string in a string array and if it is found return the
 * index back.
 *
 * Return: index, starting from %0, if the property was found (success),
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO if the property is not an array of strings,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_match_string(const struct fwnode_handle *fwnode,
        const char *propname, const char *string)
{
        const char **values;
        int nval, ret;

        nval = fwnode_property_string_array_count(fwnode, propname);
        if (nval < 0)
                return nval;

        if (nval == 0)
                return -ENODATA;

        values = kcalloc(nval, sizeof(*values), GFP_KERNEL);
        if (!values)
                return -ENOMEM;

        ret = fwnode_property_read_string_array(fwnode, propname, values, nval);
        if (ret < 0)
                goto out_free;

        ret = match_string(values, nval, string);
        if (ret < 0)
                ret = -ENODATA;

out_free:
        kfree(values);
        return ret;
}
EXPORT_SYMBOL_GPL(fwnode_property_match_string);

/**
 * fwnode_property_match_property_string - find a property string value in an array and return index
 * @fwnode: Firmware node to get the property of
 * @propname: Name of the property holding the string value
 * @array: String array to search in
 * @n: Size of the @array
 *
 * Find a property string value in a given @array and if it is found return
 * the index back.
 *
 * Return: index, starting from %0, if the string value was found in the @array (success),
 *           %-ENOENT when the string value was not found in the @array,
 *           %-EINVAL if given arguments are not valid,
 *           %-ENODATA if the property does not have a value,
 *           %-EPROTO or %-EILSEQ if the property is not a string,
 *           %-ENXIO if no suitable firmware interface is present.
 */
int fwnode_property_match_property_string(const struct fwnode_handle *fwnode,
        const char *propname, const char * const *array, size_t n)
{
        const char *string;
        int ret;

        ret = fwnode_property_read_string(fwnode, propname, &string);
        if (ret)
                return ret;

        ret = match_string(array, n, string);
        if (ret < 0)
                ret = -ENOENT;

        return ret;
}
EXPORT_SYMBOL_GPL(fwnode_property_match_property_string);

/**
 * fwnode_property_get_reference_args() - Find a reference with arguments
 * @fwnode:        Firmware node where to look for the reference
 * @prop:        The name of the property
 * @nargs_prop:        The name of the property telling the number of
 *                arguments in the referred node. NULL if @nargs is known,
 *                otherwise @nargs is ignored.
 * @nargs:        Number of arguments. Ignored if @nargs_prop is non-NULL.
 * @index:        Index of the reference, from zero onwards.
 * @args:        Result structure with reference and integer arguments.
 *                May be NULL.
 *
 * Obtain a reference based on a named property in an fwnode, with
 * integer arguments.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * @args->fwnode pointer.
 *
 * Return: %0 on success
 *            %-ENOENT when the index is out of bounds, the index has an empty
 *                     reference or the property was not found
 *            %-EINVAL on parse error
 */
int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
                                       const char *prop, const char *nargs_prop,
                                       unsigned int nargs, unsigned int index,
                                       struct fwnode_reference_args *args)
{
        int ret;

        if (IS_ERR_OR_NULL(fwnode))
                return -ENOENT;

        ret = fwnode_call_int_op(fwnode, get_reference_args, prop, nargs_prop,
                                 nargs, index, args);
        if (ret == 0)
                return ret;

        if (IS_ERR_OR_NULL(fwnode->secondary))
                return ret;

        return fwnode_call_int_op(fwnode->secondary, get_reference_args, prop, nargs_prop,
                                  nargs, index, args);
}
EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);

/**
 * fwnode_find_reference - Find named reference to a fwnode_handle
 * @fwnode: Firmware node where to look for the reference
 * @name: The name of the reference
 * @index: Index of the reference
 *
 * @index can be used when the named reference holds a table of references.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: a pointer to the reference fwnode, when found. Otherwise,
 * returns an error pointer.
 */
struct fwnode_handle *fwnode_find_reference(const struct fwnode_handle *fwnode,
                                            const char *name,
                                            unsigned int index)
{
        struct fwnode_reference_args args;
        int ret;

        ret = fwnode_property_get_reference_args(fwnode, name, NULL, 0, index,
                                                 &args);
        return ret ? ERR_PTR(ret) : args.fwnode;
}
EXPORT_SYMBOL_GPL(fwnode_find_reference);

/**
 * fwnode_get_name - Return the name of a node
 * @fwnode: The firmware node
 *
 * Return: a pointer to the node name, or %NULL.
 */
const char *fwnode_get_name(const struct fwnode_handle *fwnode)
{
        return fwnode_call_ptr_op(fwnode, get_name);
}
EXPORT_SYMBOL_GPL(fwnode_get_name);

/**
 * fwnode_get_name_prefix - Return the prefix of node for printing purposes
 * @fwnode: The firmware node
 *
 * Return: the prefix of a node, intended to be printed right before the node.
 * The prefix works also as a separator between the nodes.
 */
const char *fwnode_get_name_prefix(const struct fwnode_handle *fwnode)
{
        return fwnode_call_ptr_op(fwnode, get_name_prefix);
}

/**
 * fwnode_name_eq - Return true if node name is equal
 * @fwnode: The firmware node
 * @name: The name to which to compare the node name
 *
 * Compare the name provided as an argument to the name of the node, stopping
 * the comparison at either NUL or '@' character, whichever comes first. This
 * function is generally used for comparing node names while ignoring the
 * possible unit address of the node.
 *
 * Return: true if the node name matches with the name provided in the @name
 * argument, false otherwise.
 */
bool fwnode_name_eq(const struct fwnode_handle *fwnode, const char *name)
{
        const char *node_name;
        ptrdiff_t len;

        node_name = fwnode_get_name(fwnode);
        if (!node_name)
                return false;

        len = strchrnul(node_name, '@') - node_name;

        return str_has_prefix(node_name, name) == len;
}
EXPORT_SYMBOL_GPL(fwnode_name_eq);

/**
 * fwnode_get_parent - Return parent firwmare node
 * @fwnode: Firmware whose parent is retrieved
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: parent firmware node of the given node if possible or %NULL if no
 * parent was available.
 */
struct fwnode_handle *fwnode_get_parent(const struct fwnode_handle *fwnode)
{
        return fwnode_call_ptr_op(fwnode, get_parent);
}
EXPORT_SYMBOL_GPL(fwnode_get_parent);

/**
 * fwnode_get_next_parent - Iterate to the node's parent
 * @fwnode: Firmware whose parent is retrieved
 *
 * This is like fwnode_get_parent() except that it drops the refcount
 * on the passed node, making it suitable for iterating through a
 * node's parents.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer. Note that this function also puts a reference to @fwnode
 * unconditionally.
 *
 * Return: parent firmware node of the given node if possible or %NULL if no
 * parent was available.
 */
struct fwnode_handle *fwnode_get_next_parent(struct fwnode_handle *fwnode)
{
        struct fwnode_handle *parent = fwnode_get_parent(fwnode);

        fwnode_handle_put(fwnode);

        return parent;
}
EXPORT_SYMBOL_GPL(fwnode_get_next_parent);

/**
 * fwnode_count_parents - Return the number of parents a node has
 * @fwnode: The node the parents of which are to be counted
 *
 * Return: the number of parents a node has.
 */
unsigned int fwnode_count_parents(const struct fwnode_handle *fwnode)
{
        struct fwnode_handle *parent;
        unsigned int count = 0;

        fwnode_for_each_parent_node(fwnode, parent)
                count++;

        return count;
}
EXPORT_SYMBOL_GPL(fwnode_count_parents);

/**
 * fwnode_get_nth_parent - Return an nth parent of a node
 * @fwnode: The node the parent of which is requested
 * @depth: Distance of the parent from the node
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: the nth parent of a node. If there is no parent at the requested
 * @depth, %NULL is returned. If @depth is 0, the functionality is equivalent to
 * fwnode_handle_get(). For @depth == 1, it is fwnode_get_parent() and so on.
 */
struct fwnode_handle *fwnode_get_nth_parent(struct fwnode_handle *fwnode,
                                            unsigned int depth)
{
        struct fwnode_handle *parent;

        if (depth == 0)
                return fwnode_handle_get(fwnode);

        fwnode_for_each_parent_node(fwnode, parent) {
                if (--depth == 0)
                        return parent;
        }
        return NULL;
}
EXPORT_SYMBOL_GPL(fwnode_get_nth_parent);

/**
 * fwnode_get_next_child_node - Return the next child node handle for a node
 * @fwnode: Firmware node to find the next child node for.
 * @child: Handle to one of the node's child nodes or a %NULL handle.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer. Note that this function also puts a reference to @child
 * unconditionally.
 */
struct fwnode_handle *
fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
                           struct fwnode_handle *child)
{
        return fwnode_call_ptr_op(fwnode, get_next_child_node, child);
}
EXPORT_SYMBOL_GPL(fwnode_get_next_child_node);

/**
 * fwnode_get_next_available_child_node - Return the next available child node handle for a node
 * @fwnode: Firmware node to find the next child node for.
 * @child: Handle to one of the node's child nodes or a %NULL handle.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer. Note that this function also puts a reference to @child
 * unconditionally.
 */
struct fwnode_handle *
fwnode_get_next_available_child_node(const struct fwnode_handle *fwnode,
                                     struct fwnode_handle *child)
{
        struct fwnode_handle *next_child = child;

        if (IS_ERR_OR_NULL(fwnode))
                return NULL;

        do {
                next_child = fwnode_get_next_child_node(fwnode, next_child);
                if (!next_child)
                        return NULL;
        } while (!fwnode_device_is_available(next_child));

        return next_child;
}
EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node);

/**
 * device_get_next_child_node - Return the next child node handle for a device
 * @dev: Device to find the next child node for.
 * @child: Handle to one of the device's child nodes or a %NULL handle.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer. Note that this function also puts a reference to @child
 * unconditionally.
 */
struct fwnode_handle *device_get_next_child_node(const struct device *dev,
                                                 struct fwnode_handle *child)
{
        const struct fwnode_handle *fwnode = dev_fwnode(dev);
        struct fwnode_handle *next;

        if (IS_ERR_OR_NULL(fwnode))
                return NULL;

        /* Try to find a child in primary fwnode */
        next = fwnode_get_next_child_node(fwnode, child);
        if (next)
                return next;

        /* When no more children in primary, continue with secondary */
        return fwnode_get_next_child_node(fwnode->secondary, child);
}
EXPORT_SYMBOL_GPL(device_get_next_child_node);

/**
 * fwnode_get_named_child_node - Return first matching named child node handle
 * @fwnode: Firmware node to find the named child node for.
 * @childname: String to match child node name against.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 */
struct fwnode_handle *
fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
                            const char *childname)
{
        return fwnode_call_ptr_op(fwnode, get_named_child_node, childname);
}
EXPORT_SYMBOL_GPL(fwnode_get_named_child_node);

/**
 * device_get_named_child_node - Return first matching named child node handle
 * @dev: Device to find the named child node for.
 * @childname: String to match child node name against.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 */
struct fwnode_handle *device_get_named_child_node(const struct device *dev,
                                                  const char *childname)
{
        return fwnode_get_named_child_node(dev_fwnode(dev), childname);
}
EXPORT_SYMBOL_GPL(device_get_named_child_node);

/**
 * fwnode_handle_get - Obtain a reference to a device node
 * @fwnode: Pointer to the device node to obtain the reference to.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: the fwnode handle.
 */
struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *fwnode)
{
        if (!fwnode_has_op(fwnode, get))
                return fwnode;

        return fwnode_call_ptr_op(fwnode, get);
}
EXPORT_SYMBOL_GPL(fwnode_handle_get);

/**
 * fwnode_device_is_available - check if a device is available for use
 * @fwnode: Pointer to the fwnode of the device.
 *
 * Return: true if device is available for use. Otherwise, returns false.
 *
 * For fwnode node types that don't implement the .device_is_available()
 * operation, this function returns true.
 */
bool fwnode_device_is_available(const struct fwnode_handle *fwnode)
{
        if (IS_ERR_OR_NULL(fwnode))
                return false;

        if (!fwnode_has_op(fwnode, device_is_available))
                return true;

        return fwnode_call_bool_op(fwnode, device_is_available);
}
EXPORT_SYMBOL_GPL(fwnode_device_is_available);

/**
 * fwnode_get_child_node_count - return the number of child nodes for a given firmware node
 * @fwnode: Pointer to the parent firmware node
 *
 * Return: the number of child nodes for a given firmware node.
 */
unsigned int fwnode_get_child_node_count(const struct fwnode_handle *fwnode)
{
        struct fwnode_handle *child;
        unsigned int count = 0;

        fwnode_for_each_child_node(fwnode, child)
                count++;

        return count;
}
EXPORT_SYMBOL_GPL(fwnode_get_child_node_count);

/**
 * fwnode_get_named_child_node_count - number of child nodes with given name
 * @fwnode: Node which child nodes are counted.
 * @name: String to match child node name against.
 *
 * Scan child nodes and count all the nodes with a specific name. Potential
 * 'number' -ending after the 'at sign' for scanned names is ignored.
 * E.g.::
 *   fwnode_get_named_child_node_count(fwnode, "channel");
 * would match all the nodes::
 *   channel { }, channel@0 {}, channel@0xabba {}...
 *
 * Return: the number of child nodes with a matching name for a given device.
 */
unsigned int fwnode_get_named_child_node_count(const struct fwnode_handle *fwnode,
                                               const char *name)
{
        struct fwnode_handle *child;
        unsigned int count = 0;

        fwnode_for_each_named_child_node(fwnode, child, name)
                count++;

        return count;
}
EXPORT_SYMBOL_GPL(fwnode_get_named_child_node_count);

bool device_dma_supported(const struct device *dev)
{
        return fwnode_call_bool_op(dev_fwnode(dev), device_dma_supported);
}
EXPORT_SYMBOL_GPL(device_dma_supported);

enum dev_dma_attr device_get_dma_attr(const struct device *dev)
{
        if (!fwnode_has_op(dev_fwnode(dev), device_get_dma_attr))
                return DEV_DMA_NOT_SUPPORTED;

        return fwnode_call_int_op(dev_fwnode(dev), device_get_dma_attr);
}
EXPORT_SYMBOL_GPL(device_get_dma_attr);

/**
 * fwnode_get_phy_mode - Get phy mode for given firmware node
 * @fwnode:        Pointer to the given node
 *
 * The function gets phy interface string from property 'phy-mode' or
 * 'phy-connection-type', and return its index in phy_modes table, or errno in
 * error case.
 */
int fwnode_get_phy_mode(const struct fwnode_handle *fwnode)
{
        const char *pm;
        int err, i;

        err = fwnode_property_read_string(fwnode, "phy-mode", &pm);
        if (err < 0)
                err = fwnode_property_read_string(fwnode,
                                                  "phy-connection-type", &pm);
        if (err < 0)
                return err;

        for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++)
                if (!strcasecmp(pm, phy_modes(i)))
                        return i;

        return -ENODEV;
}
EXPORT_SYMBOL_GPL(fwnode_get_phy_mode);

/**
 * device_get_phy_mode - Get phy mode for given device
 * @dev:        Pointer to the given device
 *
 * The function gets phy interface string from property 'phy-mode' or
 * 'phy-connection-type', and return its index in phy_modes table, or errno in
 * error case.
 */
int device_get_phy_mode(struct device *dev)
{
        return fwnode_get_phy_mode(dev_fwnode(dev));
}
EXPORT_SYMBOL_GPL(device_get_phy_mode);

/**
 * fwnode_iomap - Maps the memory mapped IO for a given fwnode
 * @fwnode:        Pointer to the firmware node
 * @index:        Index of the IO range
 *
 * Return: a pointer to the mapped memory.
 */
void __iomem *fwnode_iomap(struct fwnode_handle *fwnode, int index)
{
        return fwnode_call_ptr_op(fwnode, iomap, index);
}
EXPORT_SYMBOL(fwnode_iomap);

/**
 * fwnode_irq_get - Get IRQ directly from a fwnode
 * @fwnode:        Pointer to the firmware node
 * @index:        Zero-based index of the IRQ
 *
 * Return: Linux IRQ number on success. Negative errno on failure.
 */
int fwnode_irq_get(const struct fwnode_handle *fwnode, unsigned int index)
{
        int ret;

        ret = fwnode_call_int_op(fwnode, irq_get, index);
        /* We treat mapping errors as invalid case */
        if (ret == 0)
                return -EINVAL;

        return ret;
}
EXPORT_SYMBOL(fwnode_irq_get);

/**
 * fwnode_irq_get_byname - Get IRQ from a fwnode using its name
 * @fwnode:        Pointer to the firmware node
 * @name:        IRQ name
 *
 * Description:
 * Find a match to the string @name in the 'interrupt-names' string array
 * in _DSD for ACPI, or of_node for Device Tree. Then get the Linux IRQ
 * number of the IRQ resource corresponding to the index of the matched
 * string.
 *
 * Return: Linux IRQ number on success, or negative errno otherwise.
 */
int fwnode_irq_get_byname(const struct fwnode_handle *fwnode, const char *name)
{
        int index;

        if (!name)
                return -EINVAL;

        index = fwnode_property_match_string(fwnode, "interrupt-names",  name);
        if (index < 0)
                return index;

        return fwnode_irq_get(fwnode, index);
}
EXPORT_SYMBOL(fwnode_irq_get_byname);

/**
 * fwnode_graph_get_next_endpoint - Get next endpoint firmware node
 * @fwnode: Pointer to the parent firmware node
 * @prev: Previous endpoint node or %NULL to get the first
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer. Note that this function also puts a reference to @prev
 * unconditionally.
 *
 * Return: an endpoint firmware node pointer or %NULL if no more endpoints
 * are available.
 */
struct fwnode_handle *
fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
                               struct fwnode_handle *prev)
{
        struct fwnode_handle *ep, *port_parent = NULL;
        const struct fwnode_handle *parent;

        /*
         * If this function is in a loop and the previous iteration returned
         * an endpoint from fwnode->secondary, then we need to use the secondary
         * as parent rather than @fwnode.
         */
        if (prev) {
                port_parent = fwnode_graph_get_port_parent(prev);
                parent = port_parent;
        } else {
                parent = fwnode;
        }
        if (IS_ERR_OR_NULL(parent))
                return NULL;

        ep = fwnode_call_ptr_op(parent, graph_get_next_endpoint, prev);
        if (ep)
                goto out_put_port_parent;

        ep = fwnode_graph_get_next_endpoint(parent->secondary, NULL);

out_put_port_parent:
        fwnode_handle_put(port_parent);
        return ep;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint);

/**
 * fwnode_graph_get_port_parent - Return the device fwnode of a port endpoint
 * @endpoint: Endpoint firmware node of the port
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: the firmware node of the device the @endpoint belongs to.
 */
struct fwnode_handle *
fwnode_graph_get_port_parent(const struct fwnode_handle *endpoint)
{
        struct fwnode_handle *port, *parent;

        port = fwnode_get_parent(endpoint);
        parent = fwnode_call_ptr_op(port, graph_get_port_parent);

        fwnode_handle_put(port);

        return parent;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_port_parent);

/**
 * fwnode_graph_get_remote_port_parent - Return fwnode of a remote device
 * @fwnode: Endpoint firmware node pointing to the remote endpoint
 *
 * Extracts firmware node of a remote device the @fwnode points to.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 */
struct fwnode_handle *
fwnode_graph_get_remote_port_parent(const struct fwnode_handle *fwnode)
{
        struct fwnode_handle *endpoint, *parent;

        endpoint = fwnode_graph_get_remote_endpoint(fwnode);
        parent = fwnode_graph_get_port_parent(endpoint);

        fwnode_handle_put(endpoint);

        return parent;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port_parent);

/**
 * fwnode_graph_get_remote_port - Return fwnode of a remote port
 * @fwnode: Endpoint firmware node pointing to the remote endpoint
 *
 * Extracts firmware node of a remote port the @fwnode points to.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 */
struct fwnode_handle *
fwnode_graph_get_remote_port(const struct fwnode_handle *fwnode)
{
        return fwnode_get_next_parent(fwnode_graph_get_remote_endpoint(fwnode));
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port);

/**
 * fwnode_graph_get_remote_endpoint - Return fwnode of a remote endpoint
 * @fwnode: Endpoint firmware node pointing to the remote endpoint
 *
 * Extracts firmware node of a remote endpoint the @fwnode points to.
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 */
struct fwnode_handle *
fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
{
        return fwnode_call_ptr_op(fwnode, graph_get_remote_endpoint);
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_endpoint);

static bool fwnode_graph_remote_available(struct fwnode_handle *ep)
{
        struct fwnode_handle *dev_node;
        bool available;

        dev_node = fwnode_graph_get_remote_port_parent(ep);
        available = fwnode_device_is_available(dev_node);
        fwnode_handle_put(dev_node);

        return available;
}

/**
 * fwnode_graph_get_endpoint_by_id - get endpoint by port and endpoint numbers
 * @fwnode: parent fwnode_handle containing the graph
 * @port: identifier of the port node
 * @endpoint: identifier of the endpoint node under the port node
 * @flags: fwnode lookup flags
 *
 * The caller is responsible for calling fwnode_handle_put() on the returned
 * fwnode pointer.
 *
 * Return: the fwnode handle of the local endpoint corresponding the port and
 * endpoint IDs or %NULL if not found.
 *
 * If FWNODE_GRAPH_ENDPOINT_NEXT is passed in @flags and the specified endpoint
 * has not been found, look for the closest endpoint ID greater than the
 * specified one and return the endpoint that corresponds to it, if present.
 *
 * Does not return endpoints that belong to disabled devices or endpoints that
 * are unconnected, unless FWNODE_GRAPH_DEVICE_DISABLED is passed in @flags.
 */
struct fwnode_handle *
fwnode_graph_get_endpoint_by_id(const struct fwnode_handle *fwnode,
                                u32 port, u32 endpoint, unsigned long flags)
{
        struct fwnode_handle *ep, *best_ep = NULL;
        unsigned int best_ep_id = 0;
        bool endpoint_next = flags & FWNODE_GRAPH_ENDPOINT_NEXT;
        bool enabled_only = !(flags & FWNODE_GRAPH_DEVICE_DISABLED);

        fwnode_graph_for_each_endpoint(fwnode, ep) {
                struct fwnode_endpoint fwnode_ep = { 0 };
                int ret;

                if (enabled_only && !fwnode_graph_remote_available(ep))
                        continue;

                ret = fwnode_graph_parse_endpoint(ep, &fwnode_ep);
                if (ret < 0)
                        continue;

                if (fwnode_ep.port != port)
                        continue;

                if (fwnode_ep.id == endpoint)
                        return ep;

                if (!endpoint_next)
                        continue;

                /*
                 * If the endpoint that has just been found is not the first
                 * matching one and the ID of the one found previously is closer
                 * to the requested endpoint ID, skip it.
                 */
                if (fwnode_ep.id < endpoint ||
                    (best_ep && best_ep_id < fwnode_ep.id))
                        continue;

                fwnode_handle_put(best_ep);
                best_ep = fwnode_handle_get(ep);
                best_ep_id = fwnode_ep.id;
        }

        return best_ep;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_by_id);

/**
 * fwnode_graph_get_endpoint_count - Count endpoints on a device node
 * @fwnode: The node related to a device
 * @flags: fwnode lookup flags
 * Count endpoints in a device node.
 *
 * If FWNODE_GRAPH_DEVICE_DISABLED flag is specified, also unconnected endpoints
 * and endpoints connected to disabled devices are counted.
 */
unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode,
                                             unsigned long flags)
{
        struct fwnode_handle *ep;
        unsigned int count = 0;

        fwnode_graph_for_each_endpoint(fwnode, ep) {
                if (flags & FWNODE_GRAPH_DEVICE_DISABLED ||
                    fwnode_graph_remote_available(ep))
                        count++;
        }

        return count;
}
EXPORT_SYMBOL_GPL(fwnode_graph_get_endpoint_count);

/**
 * fwnode_graph_parse_endpoint - parse common endpoint node properties
 * @fwnode: pointer to endpoint fwnode_handle
 * @endpoint: pointer to the fwnode endpoint data structure
 *
 * Parse @fwnode representing a graph endpoint node and store the
 * information in @endpoint. The caller must hold a reference to
 * @fwnode.
 */
int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
                                struct fwnode_endpoint *endpoint)
{
        memset(endpoint, 0, sizeof(*endpoint));

        return fwnode_call_int_op(fwnode, graph_parse_endpoint, endpoint);
}
EXPORT_SYMBOL(fwnode_graph_parse_endpoint);

const void *device_get_match_data(const struct device *dev)
{
        return fwnode_call_ptr_op(dev_fwnode(dev), device_get_match_data, dev);
}
EXPORT_SYMBOL_GPL(device_get_match_data);

static unsigned int fwnode_graph_devcon_matches(const struct fwnode_handle *fwnode,
                                                const char *con_id, void *data,
                                                devcon_match_fn_t match,
                                                void **matches,
                                                unsigned int matches_len)
{
        struct fwnode_handle *node;
        struct fwnode_handle *ep;
        unsigned int count = 0;
        void *ret;

        fwnode_graph_for_each_endpoint(fwnode, ep) {
                if (matches && count >= matches_len) {
                        fwnode_handle_put(ep);
                        break;
                }

                node = fwnode_graph_get_remote_port_parent(ep);
                if (!fwnode_device_is_available(node)) {
                        fwnode_handle_put(node);
                        continue;
                }

                ret = match(node, con_id, data);
                fwnode_handle_put(node);
                if (ret) {
                        if (matches)
                                matches[count] = ret;
                        count++;
                }
        }
        return count;
}

static unsigned int fwnode_devcon_matches(const struct fwnode_handle *fwnode,
                                          const char *con_id, void *data,
                                          devcon_match_fn_t match,
                                          void **matches,
                                          unsigned int matches_len)
{
        struct fwnode_handle *node;
        unsigned int count = 0;
        unsigned int i;
        void *ret;

        for (i = 0; ; i++) {
                if (matches && count >= matches_len)
                        break;

                node = fwnode_find_reference(fwnode, con_id, i);
                if (IS_ERR(node))
                        break;

                ret = match(node, NULL, data);
                fwnode_handle_put(node);
                if (ret) {
                        if (matches)
                                matches[count] = ret;
                        count++;
                }
        }

        return count;
}

/**
 * fwnode_connection_find_match - Find connection from a device node
 * @fwnode: Device node with the connection
 * @con_id: Identifier for the connection
 * @data: Data for the match function
 * @match: Function to check and convert the connection description
 *
 * Find a connection with unique identifier @con_id between @fwnode and another
 * device node. @match will be used to convert the connection description to
 * data the caller is expecting to be returned.
 */
void *fwnode_connection_find_match(const struct fwnode_handle *fwnode,
                                   const char *con_id, void *data,
                                   devcon_match_fn_t match)
{
        unsigned int count;
        void *ret;

        if (!fwnode || !match)
                return NULL;

        count = fwnode_graph_devcon_matches(fwnode, con_id, data, match, &ret, 1);
        if (count)
                return ret;

        count = fwnode_devcon_matches(fwnode, con_id, data, match, &ret, 1);
        return count ? ret : NULL;
}
EXPORT_SYMBOL_GPL(fwnode_connection_find_match);

/**
 * fwnode_connection_find_matches - Find connections from a device node
 * @fwnode: Device node with the connection
 * @con_id: Identifier for the connection
 * @data: Data for the match function
 * @match: Function to check and convert the connection description
 * @matches: (Optional) array of pointers to fill with matches
 * @matches_len: Length of @matches
 *
 * Find up to @matches_len connections with unique identifier @con_id between
 * @fwnode and other device nodes. @match will be used to convert the
 * connection description to data the caller is expecting to be returned
 * through the @matches array.
 *
 * If @matches is %NULL @matches_len is ignored and the total number of resolved
 * matches is returned.
 *
 * Return: Number of matches resolved, or negative errno.
 */
int fwnode_connection_find_matches(const struct fwnode_handle *fwnode,
                                   const char *con_id, void *data,
                                   devcon_match_fn_t match,
                                   void **matches, unsigned int matches_len)
{
        unsigned int count_graph;
        unsigned int count_ref;

        if (!fwnode || !match)
                return -EINVAL;

        count_graph = fwnode_graph_devcon_matches(fwnode, con_id, data, match,
                                                  matches, matches_len);

        if (matches) {
                matches += count_graph;
                matches_len -= count_graph;
        }

        count_ref = fwnode_devcon_matches(fwnode, con_id, data, match,
                                          matches, matches_len);

        return count_graph + count_ref;
}
EXPORT_SYMBOL_GPL(fwnode_connection_find_matches);




































   18 




   18 


    2 


    3 





    2 


   13 


























































    1 
    2 







    1 














    2 















    2 













    4 







    2 



    1 






















    1 



    2 












    2 
    1 

    1 














    1 





    6 








    2 


    4 



    1 






    1 
    1 











    5 







    2 



    1 








    1 



    1 












   13 









    2 





    1 




    1 










    7 



    1 
    1 










    1 


    1 




    1 






































   34 






    1 





   32 








    2 

    2 













    1 

    1 





   30 



   34 






    5 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Crypto user configuration API.
 *
 * Copyright (C) 2011 secunet Security Networks AG
 * Copyright (C) 2011 Steffen Klassert <steffen.klassert@secunet.com>
 */

#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/cryptouser.h>
#include <linux/sched.h>
#include <linux/security.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/rng.h>
#include <crypto/akcipher.h>
#include <crypto/kpp.h>

#include "internal.h"

#define null_terminated(x)        (strnlen(x, sizeof(x)) < sizeof(x))

static DEFINE_MUTEX(crypto_cfg_mutex);

struct crypto_dump_info {
        struct sk_buff *in_skb;
        struct sk_buff *out_skb;
        u32 nlmsg_seq;
        u16 nlmsg_flags;
};

static struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
{
        struct crypto_alg *q, *alg = NULL;

        down_read(&crypto_alg_sem);

        list_for_each_entry(q, &crypto_alg_list, cra_list) {
                int match = 0;

                if (crypto_is_larval(q))
                        continue;

                if ((q->cra_flags ^ p->cru_type) & p->cru_mask)
                        continue;

                if (strlen(p->cru_driver_name))
                        match = !strcmp(q->cra_driver_name,
                                        p->cru_driver_name);
                else if (!exact)
                        match = !strcmp(q->cra_name, p->cru_name);

                if (!match)
                        continue;

                if (unlikely(!crypto_mod_get(q)))
                        continue;

                alg = q;
                break;
        }

        up_read(&crypto_alg_sem);

        return alg;
}

static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
{
        struct crypto_report_cipher rcipher;

        memset(&rcipher, 0, sizeof(rcipher));

        strscpy(rcipher.type, "cipher", sizeof(rcipher.type));

        rcipher.blocksize = alg->cra_blocksize;
        rcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
        rcipher.max_keysize = alg->cra_cipher.cia_max_keysize;

        return nla_put(skb, CRYPTOCFGA_REPORT_CIPHER,
                       sizeof(rcipher), &rcipher);
}

static int crypto_report_one(struct crypto_alg *alg,
                             struct crypto_user_alg *ualg, struct sk_buff *skb)
{
        memset(ualg, 0, sizeof(*ualg));

        strscpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name));
        strscpy(ualg->cru_driver_name, alg->cra_driver_name,
                sizeof(ualg->cru_driver_name));
        strscpy(ualg->cru_module_name, module_name(alg->cra_module),
                sizeof(ualg->cru_module_name));

        ualg->cru_type = 0;
        ualg->cru_mask = 0;
        ualg->cru_flags = alg->cra_flags;
        ualg->cru_refcnt = refcount_read(&alg->cra_refcnt);

        if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority))
                goto nla_put_failure;
        if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
                struct crypto_report_larval rl;

                memset(&rl, 0, sizeof(rl));
                strscpy(rl.type, "larval", sizeof(rl.type));
                if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, sizeof(rl), &rl))
                        goto nla_put_failure;
                goto out;
        }

        if (alg->cra_type && alg->cra_type->report) {
                if (alg->cra_type->report(skb, alg))
                        goto nla_put_failure;

                goto out;
        }

        switch (alg->cra_flags & (CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_LARVAL)) {
        case CRYPTO_ALG_TYPE_CIPHER:
                if (crypto_report_cipher(skb, alg))
                        goto nla_put_failure;

                break;
        }

out:
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int crypto_report_alg(struct crypto_alg *alg,
                             struct crypto_dump_info *info)
{
        struct sk_buff *in_skb = info->in_skb;
        struct sk_buff *skb = info->out_skb;
        struct nlmsghdr *nlh;
        struct crypto_user_alg *ualg;
        int err = 0;

        nlh = nlmsg_put(skb, NETLINK_CB(in_skb).portid, info->nlmsg_seq,
                        CRYPTO_MSG_GETALG, sizeof(*ualg), info->nlmsg_flags);
        if (!nlh) {
                err = -EMSGSIZE;
                goto out;
        }

        ualg = nlmsg_data(nlh);

        err = crypto_report_one(alg, ualg, skb);
        if (err) {
                nlmsg_cancel(skb, nlh);
                goto out;
        }

        nlmsg_end(skb, nlh);

out:
        return err;
}

static int crypto_report(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
                         struct nlattr **attrs)
{
        struct net *net = sock_net(in_skb->sk);
        struct crypto_user_alg *p = nlmsg_data(in_nlh);
        struct crypto_alg *alg;
        struct sk_buff *skb;
        struct crypto_dump_info info;
        int err;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 0);
        if (!alg)
                return -ENOENT;

        err = -ENOMEM;
        skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                goto drop_alg;

        info.in_skb = in_skb;
        info.out_skb = skb;
        info.nlmsg_seq = in_nlh->nlmsg_seq;
        info.nlmsg_flags = 0;

        err = crypto_report_alg(alg, &info);

drop_alg:
        crypto_mod_put(alg);

        if (err) {
                kfree_skb(skb);
                return err;
        }

        return nlmsg_unicast(net->crypto_nlsk, skb, NETLINK_CB(in_skb).portid);
}

static int crypto_dump_report(struct sk_buff *skb, struct netlink_callback *cb)
{
        const size_t start_pos = cb->args[0];
        size_t pos = 0;
        struct crypto_dump_info info;
        struct crypto_alg *alg;
        int res;

        info.in_skb = cb->skb;
        info.out_skb = skb;
        info.nlmsg_seq = cb->nlh->nlmsg_seq;
        info.nlmsg_flags = NLM_F_MULTI;

        down_read(&crypto_alg_sem);
        list_for_each_entry(alg, &crypto_alg_list, cra_list) {
                if (pos >= start_pos) {
                        res = crypto_report_alg(alg, &info);
                        if (res == -EMSGSIZE)
                                break;
                        if (res)
                                goto out;
                }
                pos++;
        }
        cb->args[0] = pos;
        res = skb->len;
out:
        up_read(&crypto_alg_sem);
        return res;
}

static int crypto_dump_report_done(struct netlink_callback *cb)
{
        return 0;
}

static int crypto_update_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct nlattr **attrs)
{
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
        LIST_HEAD(list);

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        if (priority && !strlen(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 1);
        if (!alg)
                return -ENOENT;

        down_write(&crypto_alg_sem);

        crypto_remove_spawns(alg, &list, NULL);

        if (priority)
                alg->cra_priority = nla_get_u32(priority);

        up_write(&crypto_alg_sem);

        crypto_mod_put(alg);
        crypto_remove_final(&list);

        return 0;
}

static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        int err;

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        alg = crypto_alg_match(p, 1);
        if (!alg)
                return -ENOENT;

        /* We can not unregister core algorithms such as aes-generic.
         * We would loose the reference in the crypto_alg_list to this algorithm
         * if we try to unregister. Unregistering such an algorithm without
         * removing the module is not possible, so we restrict to crypto
         * instances that are build from templates. */
        err = -EINVAL;
        if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE))
                goto drop_alg;

        err = -EBUSY;
        if (refcount_read(&alg->cra_refcnt) > 2)
                goto drop_alg;

        crypto_unregister_instance((struct crypto_instance *)alg);
        err = 0;

drop_alg:
        crypto_mod_put(alg);
        return err;
}

static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        int exact = 0;
        const char *name;
        struct crypto_alg *alg;
        struct crypto_user_alg *p = nlmsg_data(nlh);
        struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
                return -EINVAL;

        if (strlen(p->cru_driver_name))
                exact = 1;

        if (priority && !exact)
                return -EINVAL;

        alg = crypto_alg_match(p, exact);
        if (alg) {
                crypto_mod_put(alg);
                return -EEXIST;
        }

        if (strlen(p->cru_driver_name))
                name = p->cru_driver_name;
        else
                name = p->cru_name;

        alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask);
        if (IS_ERR(alg))
                return PTR_ERR(alg);

        down_write(&crypto_alg_sem);

        if (priority)
                alg->cra_priority = nla_get_u32(priority);

        up_write(&crypto_alg_sem);

        crypto_mod_put(alg);

        return 0;
}

static int crypto_del_rng(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct nlattr **attrs)
{
        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;
        return crypto_del_default_rng();
}

static int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
                             struct nlattr **attrs)
{
        /* No longer supported */
        return -ENOTSUPP;
}

#define MSGSIZE(type) sizeof(struct type)

static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = {
        [CRYPTO_MSG_NEWALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_DELALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_UPDATEALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_GETALG        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
        [CRYPTO_MSG_DELRNG        - CRYPTO_MSG_BASE] = 0,
        [CRYPTO_MSG_GETSTAT        - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
};

static const struct nla_policy crypto_policy[CRYPTOCFGA_MAX+1] = {
        [CRYPTOCFGA_PRIORITY_VAL]   = { .type = NLA_U32},
};

#undef MSGSIZE

static const struct crypto_link {
        int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **);
        int (*dump)(struct sk_buff *, struct netlink_callback *);
        int (*done)(struct netlink_callback *);
} crypto_dispatch[CRYPTO_NR_MSGTYPES] = {
        [CRYPTO_MSG_NEWALG        - CRYPTO_MSG_BASE] = { .doit = crypto_add_alg},
        [CRYPTO_MSG_DELALG        - CRYPTO_MSG_BASE] = { .doit = crypto_del_alg},
        [CRYPTO_MSG_UPDATEALG        - CRYPTO_MSG_BASE] = { .doit = crypto_update_alg},
        [CRYPTO_MSG_GETALG        - CRYPTO_MSG_BASE] = { .doit = crypto_report,
                                                       .dump = crypto_dump_report,
                                                       .done = crypto_dump_report_done},
        [CRYPTO_MSG_DELRNG        - CRYPTO_MSG_BASE] = { .doit = crypto_del_rng },
        [CRYPTO_MSG_GETSTAT        - CRYPTO_MSG_BASE] = { .doit = crypto_reportstat},
};

static int crypto_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
                               struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *attrs[CRYPTOCFGA_MAX+1];
        const struct crypto_link *link;
        int type, err;

        type = nlh->nlmsg_type;
        if (type > CRYPTO_MSG_MAX)
                return -EINVAL;

        type -= CRYPTO_MSG_BASE;
        link = &crypto_dispatch[type];

        if ((type == (CRYPTO_MSG_GETALG - CRYPTO_MSG_BASE) &&
            (nlh->nlmsg_flags & NLM_F_DUMP))) {
                struct crypto_alg *alg;
                unsigned long dump_alloc = 0;

                if (link->dump == NULL)
                        return -EINVAL;

                down_read(&crypto_alg_sem);
                list_for_each_entry(alg, &crypto_alg_list, cra_list)
                        dump_alloc += CRYPTO_REPORT_MAXSIZE;
                up_read(&crypto_alg_sem);

                {
                        struct netlink_dump_control c = {
                                .dump = link->dump,
                                .done = link->done,
                                .min_dump_alloc = min(dump_alloc, 65535UL),
                        };
                        err = netlink_dump_start(net->crypto_nlsk, skb, nlh, &c);
                }

                return err;
        }

        err = nlmsg_parse_deprecated(nlh, crypto_msg_min[type], attrs,
                                     CRYPTOCFGA_MAX, crypto_policy, extack);
        if (err < 0)
                return err;

        if (link->doit == NULL)
                return -EINVAL;

        return link->doit(skb, nlh, attrs);
}

static void crypto_netlink_rcv(struct sk_buff *skb)
{
        mutex_lock(&crypto_cfg_mutex);
        netlink_rcv_skb(skb, &crypto_user_rcv_msg);
        mutex_unlock(&crypto_cfg_mutex);
}

static int __net_init crypto_netlink_init(struct net *net)
{
        struct netlink_kernel_cfg cfg = {
                .input        = crypto_netlink_rcv,
        };

        net->crypto_nlsk = netlink_kernel_create(net, NETLINK_CRYPTO, &cfg);
        return net->crypto_nlsk == NULL ? -ENOMEM : 0;
}

static void __net_exit crypto_netlink_exit(struct net *net)
{
        netlink_kernel_release(net->crypto_nlsk);
        net->crypto_nlsk = NULL;
}

static struct pernet_operations crypto_netlink_net_ops = {
        .init = crypto_netlink_init,
        .exit = crypto_netlink_exit,
};

static int __init crypto_user_init(void)
{
        return register_pernet_subsys(&crypto_netlink_net_ops);
}

static void __exit crypto_user_exit(void)
{
        unregister_pernet_subsys(&crypto_netlink_net_ops);
}

module_init(crypto_user_init);
module_exit(crypto_user_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
MODULE_DESCRIPTION("Crypto userspace configuration API");
MODULE_ALIAS("net-pf-16-proto-21");









































































































   87 






















  591 


  594 











  146 

  543 






  596 









  543 





  149 















  148 


  591 










   38 
   37 



    8 





   33 











   38 







   18 
   18 









   18 





   18 







  319 





















  320 








   88 

  284 





  309 



  276 

















   84 
   83 

















  309 
  308 





    1 
    1 






    9 
   10 

    7 


    4 























































  219 
  218 

























































    2 




































  216 
  219 

    2 


  219 






























































































































































































































































































































































































  220 











  220 

  217 





  220 

    1 







































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
// SPDX-License-Identifier: GPL-2.0-only
/*
 * jump label support
 *
 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
 * Copyright (C) 2011 Peter Zijlstra
 *
 */
#include <linux/memory.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
#include <linux/static_key.h>
#include <linux/jump_label_ratelimit.h>
#include <linux/bug.h>
#include <linux/cpu.h>
#include <asm/sections.h>

/* mutex to protect coming/going of the jump_label table */
static DEFINE_MUTEX(jump_label_mutex);

void jump_label_lock(void)
{
        mutex_lock(&jump_label_mutex);
}

void jump_label_unlock(void)
{
        mutex_unlock(&jump_label_mutex);
}

static int jump_label_cmp(const void *a, const void *b)
{
        const struct jump_entry *jea = a;
        const struct jump_entry *jeb = b;

        /*
         * Entrires are sorted by key.
         */
        if (jump_entry_key(jea) < jump_entry_key(jeb))
                return -1;

        if (jump_entry_key(jea) > jump_entry_key(jeb))
                return 1;

        /*
         * In the batching mode, entries should also be sorted by the code
         * inside the already sorted list of entries, enabling a bsearch in
         * the vector.
         */
        if (jump_entry_code(jea) < jump_entry_code(jeb))
                return -1;

        if (jump_entry_code(jea) > jump_entry_code(jeb))
                return 1;

        return 0;
}

static void jump_label_swap(void *a, void *b, int size)
{
        long delta = (unsigned long)a - (unsigned long)b;
        struct jump_entry *jea = a;
        struct jump_entry *jeb = b;
        struct jump_entry tmp = *jea;

        jea->code        = jeb->code - delta;
        jea->target        = jeb->target - delta;
        jea->key        = jeb->key - delta;

        jeb->code        = tmp.code + delta;
        jeb->target        = tmp.target + delta;
        jeb->key        = tmp.key + delta;
}

static void
jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
{
        unsigned long size;
        void *swapfn = NULL;

        if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
                swapfn = jump_label_swap;

        size = (((unsigned long)stop - (unsigned long)start)
                                        / sizeof(struct jump_entry));
        sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
}

static void jump_label_update(struct static_key *key);

/*
 * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
 * The use of 'atomic_read()' requires atomic.h and its problematic for some
 * kernel headers such as kernel.h and others. Since static_key_count() is not
 * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
 * to have it be a function here. Similarly, for 'static_key_enable()' and
 * 'static_key_disable()', which require bug.h. This should allow jump_label.h
 * to be included from most/all places for CONFIG_JUMP_LABEL.
 */
int static_key_count(struct static_key *key)
{
        /*
         * -1 means the first static_key_slow_inc() is in progress.
         *  static_key_enabled() must return true, so return 1 here.
         */
        int n = atomic_read(&key->enabled);

        return n >= 0 ? n : 1;
}
EXPORT_SYMBOL_GPL(static_key_count);

/*
 * static_key_fast_inc_not_disabled - adds a user for a static key
 * @key: static key that must be already enabled
 *
 * The caller must make sure that the static key can't get disabled while
 * in this function. It doesn't patch jump labels, only adds a user to
 * an already enabled static key.
 *
 * Returns true if the increment was done. Unlike refcount_t the ref counter
 * is not saturated, but will fail to increment on overflow.
 */
bool static_key_fast_inc_not_disabled(struct static_key *key)
{
        int v;

        STATIC_KEY_CHECK_USE(key);
        /*
         * Negative key->enabled has a special meaning: it sends
         * static_key_slow_inc/dec() down the slow path, and it is non-zero
         * so it counts as "enabled" in jump_label_update().
         *
         * The INT_MAX overflow condition is either used by the networking
         * code to reset or detected in the slow path of
         * static_key_slow_inc_cpuslocked().
         */
        v = atomic_read(&key->enabled);
        do {
                if (v <= 0 || v == INT_MAX)
                        return false;
        } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));

        return true;
}
EXPORT_SYMBOL_GPL(static_key_fast_inc_not_disabled);

bool static_key_slow_inc_cpuslocked(struct static_key *key)
{
        lockdep_assert_cpus_held();

        /*
         * Careful if we get concurrent static_key_slow_inc/dec() calls;
         * later calls must wait for the first one to _finish_ the
         * jump_label_update() process.  At the same time, however,
         * the jump_label_update() call below wants to see
         * static_key_enabled(&key) for jumps to be updated properly.
         */
        if (static_key_fast_inc_not_disabled(key))
                return true;

        guard(mutex)(&jump_label_mutex);
        /* Try to mark it as 'enabling in progress. */
        if (!atomic_cmpxchg(&key->enabled, 0, -1)) {
                jump_label_update(key);
                /*
                 * Ensure that when static_key_fast_inc_not_disabled() or
                 * static_key_dec_not_one() observe the positive value,
                 * they must also observe all the text changes.
                 */
                atomic_set_release(&key->enabled, 1);
        } else {
                /*
                 * While holding the mutex this should never observe
                 * anything else than a value >= 1 and succeed
                 */
                if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key)))
                        return false;
        }
        return true;
}

bool static_key_slow_inc(struct static_key *key)
{
        bool ret;

        cpus_read_lock();
        ret = static_key_slow_inc_cpuslocked(key);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(static_key_slow_inc);

void static_key_enable_cpuslocked(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);
        lockdep_assert_cpus_held();

        if (atomic_read(&key->enabled) > 0) {
                WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
                return;
        }

        jump_label_lock();
        if (atomic_read(&key->enabled) == 0) {
                atomic_set(&key->enabled, -1);
                jump_label_update(key);
                /*
                 * See static_key_slow_inc().
                 */
                atomic_set_release(&key->enabled, 1);
        }
        jump_label_unlock();
}
EXPORT_SYMBOL_GPL(static_key_enable_cpuslocked);

void static_key_enable(struct static_key *key)
{
        cpus_read_lock();
        static_key_enable_cpuslocked(key);
        cpus_read_unlock();
}
EXPORT_SYMBOL_GPL(static_key_enable);

void static_key_disable_cpuslocked(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);
        lockdep_assert_cpus_held();

        if (atomic_read(&key->enabled) != 1) {
                WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
                return;
        }

        jump_label_lock();
        if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)
                jump_label_update(key);
        jump_label_unlock();
}
EXPORT_SYMBOL_GPL(static_key_disable_cpuslocked);

void static_key_disable(struct static_key *key)
{
        cpus_read_lock();
        static_key_disable_cpuslocked(key);
        cpus_read_unlock();
}
EXPORT_SYMBOL_GPL(static_key_disable);

static bool static_key_dec_not_one(struct static_key *key)
{
        int v;

        /*
         * Go into the slow path if key::enabled is less than or equal than
         * one. One is valid to shut down the key, anything less than one
         * is an imbalance, which is handled at the call site.
         *
         * That includes the special case of '-1' which is set in
         * static_key_slow_inc_cpuslocked(), but that's harmless as it is
         * fully serialized in the slow path below. By the time this task
         * acquires the jump label lock the value is back to one and the
         * retry under the lock must succeed.
         */
        v = atomic_read(&key->enabled);
        do {
                /*
                 * Warn about the '-1' case though; since that means a
                 * decrement is concurrent with a first (0->1) increment. IOW
                 * people are trying to disable something that wasn't yet fully
                 * enabled. This suggests an ordering problem on the user side.
                 */
                WARN_ON_ONCE(v < 0);

                /*
                 * Warn about underflow, and lie about success in an attempt to
                 * not make things worse.
                 */
                if (WARN_ON_ONCE(v == 0))
                        return true;

                if (v <= 1)
                        return false;
        } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1)));

        return true;
}

static void __static_key_slow_dec_cpuslocked(struct static_key *key)
{
        lockdep_assert_cpus_held();
        int val;

        if (static_key_dec_not_one(key))
                return;

        guard(mutex)(&jump_label_mutex);
        val = atomic_read(&key->enabled);
        /*
         * It should be impossible to observe -1 with jump_label_mutex held,
         * see static_key_slow_inc_cpuslocked().
         */
        if (WARN_ON_ONCE(val == -1))
                return;
        /*
         * Cannot already be 0, something went sideways.
         */
        if (WARN_ON_ONCE(val == 0))
                return;

        if (atomic_dec_and_test(&key->enabled))
                jump_label_update(key);
}

static void __static_key_slow_dec(struct static_key *key)
{
        cpus_read_lock();
        __static_key_slow_dec_cpuslocked(key);
        cpus_read_unlock();
}

void jump_label_update_timeout(struct work_struct *work)
{
        struct static_key_deferred *key =
                container_of(work, struct static_key_deferred, work.work);
        __static_key_slow_dec(&key->key);
}
EXPORT_SYMBOL_GPL(jump_label_update_timeout);

void static_key_slow_dec(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);
        __static_key_slow_dec(key);
}
EXPORT_SYMBOL_GPL(static_key_slow_dec);

void static_key_slow_dec_cpuslocked(struct static_key *key)
{
        STATIC_KEY_CHECK_USE(key);
        __static_key_slow_dec_cpuslocked(key);
}

void __static_key_slow_dec_deferred(struct static_key *key,
                                    struct delayed_work *work,
                                    unsigned long timeout)
{
        STATIC_KEY_CHECK_USE(key);

        if (static_key_dec_not_one(key))
                return;

        schedule_delayed_work(work, timeout);
}
EXPORT_SYMBOL_GPL(__static_key_slow_dec_deferred);

void __static_key_deferred_flush(void *key, struct delayed_work *work)
{
        STATIC_KEY_CHECK_USE(key);
        flush_delayed_work(work);
}
EXPORT_SYMBOL_GPL(__static_key_deferred_flush);

void jump_label_rate_limit(struct static_key_deferred *key,
                unsigned long rl)
{
        STATIC_KEY_CHECK_USE(key);
        key->timeout = rl;
        INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
}
EXPORT_SYMBOL_GPL(jump_label_rate_limit);

static int addr_conflict(struct jump_entry *entry, void *start, void *end)
{
        if (jump_entry_code(entry) <= (unsigned long)end &&
            jump_entry_code(entry) + jump_entry_size(entry) > (unsigned long)start)
                return 1;

        return 0;
}

static int __jump_label_text_reserved(struct jump_entry *iter_start,
                struct jump_entry *iter_stop, void *start, void *end, bool init)
{
        struct jump_entry *iter;

        iter = iter_start;
        while (iter < iter_stop) {
                if (init || !jump_entry_is_init(iter)) {
                        if (addr_conflict(iter, start, end))
                                return 1;
                }
                iter++;
        }

        return 0;
}

#ifndef arch_jump_label_transform_static
static void arch_jump_label_transform_static(struct jump_entry *entry,
                                             enum jump_label_type type)
{
        /* nothing to do on most architectures */
}
#endif

static inline struct jump_entry *static_key_entries(struct static_key *key)
{
        WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);
        return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);
}

static inline bool static_key_type(struct static_key *key)
{
        return key->type & JUMP_TYPE_TRUE;
}

static inline bool static_key_linked(struct static_key *key)
{
        return key->type & JUMP_TYPE_LINKED;
}

static inline void static_key_clear_linked(struct static_key *key)
{
        key->type &= ~JUMP_TYPE_LINKED;
}

static inline void static_key_set_linked(struct static_key *key)
{
        key->type |= JUMP_TYPE_LINKED;
}

/***
 * A 'struct static_key' uses a union such that it either points directly
 * to a table of 'struct jump_entry' or to a linked list of modules which in
 * turn point to 'struct jump_entry' tables.
 *
 * The two lower bits of the pointer are used to keep track of which pointer
 * type is in use and to store the initial branch direction, we use an access
 * function which preserves these bits.
 */
static void static_key_set_entries(struct static_key *key,
                                   struct jump_entry *entries)
{
        unsigned long type;

        WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
        type = key->type & JUMP_TYPE_MASK;
        key->entries = entries;
        key->type |= type;
}

static enum jump_label_type jump_label_type(struct jump_entry *entry)
{
        struct static_key *key = jump_entry_key(entry);
        bool enabled = static_key_enabled(key);
        bool branch = jump_entry_is_branch(entry);

        /* See the comment in linux/jump_label.h */
        return enabled ^ branch;
}

static bool jump_label_can_update(struct jump_entry *entry, bool init)
{
        /*
         * Cannot update code that was in an init text area.
         */
        if (!init && jump_entry_is_init(entry))
                return false;

        if (!kernel_text_address(jump_entry_code(entry))) {
                /*
                 * This skips patching built-in __exit, which
                 * is part of init_section_contains() but is
                 * not part of kernel_text_address().
                 *
                 * Skipping built-in __exit is fine since it
                 * will never be executed.
                 */
                WARN_ONCE(!jump_entry_is_init(entry),
                          "can't patch jump_label at %pS",
                          (void *)jump_entry_code(entry));
                return false;
        }

        return true;
}

#ifndef HAVE_JUMP_LABEL_BATCH
static void __jump_label_update(struct static_key *key,
                                struct jump_entry *entry,
                                struct jump_entry *stop,
                                bool init)
{
        for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
                if (jump_label_can_update(entry, init))
                        arch_jump_label_transform(entry, jump_label_type(entry));
        }
}
#else
static void __jump_label_update(struct static_key *key,
                                struct jump_entry *entry,
                                struct jump_entry *stop,
                                bool init)
{
        for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {

                if (!jump_label_can_update(entry, init))
                        continue;

                if (!arch_jump_label_transform_queue(entry, jump_label_type(entry))) {
                        /*
                         * Queue is full: Apply the current queue and try again.
                         */
                        arch_jump_label_transform_apply();
                        BUG_ON(!arch_jump_label_transform_queue(entry, jump_label_type(entry)));
                }
        }
        arch_jump_label_transform_apply();
}
#endif

void __init jump_label_init(void)
{
        struct jump_entry *iter_start = __start___jump_table;
        struct jump_entry *iter_stop = __stop___jump_table;
        struct static_key *key = NULL;
        struct jump_entry *iter;

        /*
         * Since we are initializing the static_key.enabled field with
         * with the 'raw' int values (to avoid pulling in atomic.h) in
         * jump_label.h, let's make sure that is safe. There are only two
         * cases to check since we initialize to 0 or 1.
         */
        BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
        BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);

        if (static_key_initialized)
                return;

        cpus_read_lock();
        jump_label_lock();
        jump_label_sort_entries(iter_start, iter_stop);

        for (iter = iter_start; iter < iter_stop; iter++) {
                struct static_key *iterk;
                bool in_init;

                /* rewrite NOPs */
                if (jump_label_type(iter) == JUMP_LABEL_NOP)
                        arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);

                in_init = init_section_contains((void *)jump_entry_code(iter), 1);
                jump_entry_set_init(iter, in_init);

                iterk = jump_entry_key(iter);
                if (iterk == key)
                        continue;

                key = iterk;
                static_key_set_entries(key, iter);
        }
        static_key_initialized = true;
        jump_label_unlock();
        cpus_read_unlock();
}

static inline bool static_key_sealed(struct static_key *key)
{
        return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK);
}

static inline void static_key_seal(struct static_key *key)
{
        unsigned long type = key->type & JUMP_TYPE_TRUE;
        key->type = JUMP_TYPE_LINKED | type;
}

void jump_label_init_ro(void)
{
        struct jump_entry *iter_start = __start___jump_table;
        struct jump_entry *iter_stop = __stop___jump_table;
        struct jump_entry *iter;

        if (WARN_ON_ONCE(!static_key_initialized))
                return;

        cpus_read_lock();
        jump_label_lock();

        for (iter = iter_start; iter < iter_stop; iter++) {
                struct static_key *iterk = jump_entry_key(iter);

                if (!is_kernel_ro_after_init((unsigned long)iterk))
                        continue;

                if (static_key_sealed(iterk))
                        continue;

                static_key_seal(iterk);
        }

        jump_label_unlock();
        cpus_read_unlock();
}

#ifdef CONFIG_MODULES

enum jump_label_type jump_label_init_type(struct jump_entry *entry)
{
        struct static_key *key = jump_entry_key(entry);
        bool type = static_key_type(key);
        bool branch = jump_entry_is_branch(entry);

        /* See the comment in linux/jump_label.h */
        return type ^ branch;
}

struct static_key_mod {
        struct static_key_mod *next;
        struct jump_entry *entries;
        struct module *mod;
};

static inline struct static_key_mod *static_key_mod(struct static_key *key)
{
        WARN_ON_ONCE(!static_key_linked(key));
        return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
}

/***
 * key->type and key->next are the same via union.
 * This sets key->next and preserves the type bits.
 *
 * See additional comments above static_key_set_entries().
 */
static void static_key_set_mod(struct static_key *key,
                               struct static_key_mod *mod)
{
        unsigned long type;

        WARN_ON_ONCE((unsigned long)mod & JUMP_TYPE_MASK);
        type = key->type & JUMP_TYPE_MASK;
        key->next = mod;
        key->type |= type;
}

static int __jump_label_mod_text_reserved(void *start, void *end)
{
        struct module *mod;
        int ret;

        scoped_guard(rcu) {
                mod = __module_text_address((unsigned long)start);
                WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
                if (!try_module_get(mod))
                        mod = NULL;
        }
        if (!mod)
                return 0;

        ret = __jump_label_text_reserved(mod->jump_entries,
                                mod->jump_entries + mod->num_jump_entries,
                                start, end, mod->state == MODULE_STATE_COMING);

        module_put(mod);

        return ret;
}

static void __jump_label_mod_update(struct static_key *key)
{
        struct static_key_mod *mod;

        for (mod = static_key_mod(key); mod; mod = mod->next) {
                struct jump_entry *stop;
                struct module *m;

                /*
                 * NULL if the static_key is defined in a module
                 * that does not use it
                 */
                if (!mod->entries)
                        continue;

                m = mod->mod;
                if (!m)
                        stop = __stop___jump_table;
                else
                        stop = m->jump_entries + m->num_jump_entries;
                __jump_label_update(key, mod->entries, stop,
                                    m && m->state == MODULE_STATE_COMING);
        }
}

static int jump_label_add_module(struct module *mod)
{
        struct jump_entry *iter_start = mod->jump_entries;
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
        struct jump_entry *iter;
        struct static_key *key = NULL;
        struct static_key_mod *jlm, *jlm2;

        /* if the module doesn't have jump label entries, just return */
        if (iter_start == iter_stop)
                return 0;

        jump_label_sort_entries(iter_start, iter_stop);

        for (iter = iter_start; iter < iter_stop; iter++) {
                struct static_key *iterk;
                bool in_init;

                in_init = within_module_init(jump_entry_code(iter), mod);
                jump_entry_set_init(iter, in_init);

                iterk = jump_entry_key(iter);
                if (iterk == key)
                        continue;

                key = iterk;
                if (within_module((unsigned long)key, mod)) {
                        static_key_set_entries(key, iter);
                        continue;
                }

                /*
                 * If the key was sealed at init, then there's no need to keep a
                 * reference to its module entries - just patch them now and be
                 * done with it.
                 */
                if (static_key_sealed(key))
                        goto do_poke;

                jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
                if (!jlm)
                        return -ENOMEM;
                if (!static_key_linked(key)) {
                        jlm2 = kzalloc(sizeof(struct static_key_mod),
                                       GFP_KERNEL);
                        if (!jlm2) {
                                kfree(jlm);
                                return -ENOMEM;
                        }
                        scoped_guard(rcu)
                                jlm2->mod = __module_address((unsigned long)key);

                        jlm2->entries = static_key_entries(key);
                        jlm2->next = NULL;
                        static_key_set_mod(key, jlm2);
                        static_key_set_linked(key);
                }
                jlm->mod = mod;
                jlm->entries = iter;
                jlm->next = static_key_mod(key);
                static_key_set_mod(key, jlm);
                static_key_set_linked(key);

                /* Only update if we've changed from our initial state */
do_poke:
                if (jump_label_type(iter) != jump_label_init_type(iter))
                        __jump_label_update(key, iter, iter_stop, true);
        }

        return 0;
}

static void jump_label_del_module(struct module *mod)
{
        struct jump_entry *iter_start = mod->jump_entries;
        struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
        struct jump_entry *iter;
        struct static_key *key = NULL;
        struct static_key_mod *jlm, **prev;

        for (iter = iter_start; iter < iter_stop; iter++) {
                if (jump_entry_key(iter) == key)
                        continue;

                key = jump_entry_key(iter);

                if (within_module((unsigned long)key, mod))
                        continue;

                /* No @jlm allocated because key was sealed at init. */
                if (static_key_sealed(key))
                        continue;

                /* No memory during module load */
                if (WARN_ON(!static_key_linked(key)))
                        continue;

                prev = &key->next;
                jlm = static_key_mod(key);

                while (jlm && jlm->mod != mod) {
                        prev = &jlm->next;
                        jlm = jlm->next;
                }

                /* No memory during module load */
                if (WARN_ON(!jlm))
                        continue;

                if (prev == &key->next)
                        static_key_set_mod(key, jlm->next);
                else
                        *prev = jlm->next;

                kfree(jlm);

                jlm = static_key_mod(key);
                /* if only one etry is left, fold it back into the static_key */
                if (jlm->next == NULL) {
                        static_key_set_entries(key, jlm->entries);
                        static_key_clear_linked(key);
                        kfree(jlm);
                }
        }
}

static int
jump_label_module_notify(struct notifier_block *self, unsigned long val,
                         void *data)
{
        struct module *mod = data;
        int ret = 0;

        cpus_read_lock();
        jump_label_lock();

        switch (val) {
        case MODULE_STATE_COMING:
                ret = jump_label_add_module(mod);
                if (ret) {
                        WARN(1, "Failed to allocate memory: jump_label may not work properly.\n");
                        jump_label_del_module(mod);
                }
                break;
        case MODULE_STATE_GOING:
                jump_label_del_module(mod);
                break;
        }

        jump_label_unlock();
        cpus_read_unlock();

        return notifier_from_errno(ret);
}

static struct notifier_block jump_label_module_nb = {
        .notifier_call = jump_label_module_notify,
        .priority = 1, /* higher than tracepoints */
};

static __init int jump_label_init_module(void)
{
        return register_module_notifier(&jump_label_module_nb);
}
early_initcall(jump_label_init_module);

#endif /* CONFIG_MODULES */

/***
 * jump_label_text_reserved - check if addr range is reserved
 * @start: start text addr
 * @end: end text addr
 *
 * checks if the text addr located between @start and @end
 * overlaps with any of the jump label patch addresses. Code
 * that wants to modify kernel text should first verify that
 * it does not overlap with any of the jump label addresses.
 * Caller must hold jump_label_mutex.
 *
 * returns 1 if there is an overlap, 0 otherwise
 */
int jump_label_text_reserved(void *start, void *end)
{
        bool init = system_state < SYSTEM_RUNNING;
        int ret = __jump_label_text_reserved(__start___jump_table,
                        __stop___jump_table, start, end, init);

        if (ret)
                return ret;

#ifdef CONFIG_MODULES
        ret = __jump_label_mod_text_reserved(start, end);
#endif
        return ret;
}

static void jump_label_update(struct static_key *key)
{
        struct jump_entry *stop = __stop___jump_table;
        bool init = system_state < SYSTEM_RUNNING;
        struct jump_entry *entry;
#ifdef CONFIG_MODULES
        struct module *mod;

        if (static_key_linked(key)) {
                __jump_label_mod_update(key);
                return;
        }

        scoped_guard(rcu) {
                mod = __module_address((unsigned long)key);
                if (mod) {
                        stop = mod->jump_entries + mod->num_jump_entries;
                        init = mod->state == MODULE_STATE_COMING;
                }
        }
#endif
        entry = static_key_entries(key);
        /* if there are no users, entry can be NULL */
        if (entry)
                __jump_label_update(key, entry, stop, init);
}

#ifdef CONFIG_STATIC_KEYS_SELFTEST
static DEFINE_STATIC_KEY_TRUE(sk_true);
static DEFINE_STATIC_KEY_FALSE(sk_false);

static __init int jump_label_test(void)
{
        int i;

        for (i = 0; i < 2; i++) {
                WARN_ON(static_key_enabled(&sk_true.key) != true);
                WARN_ON(static_key_enabled(&sk_false.key) != false);

                WARN_ON(!static_branch_likely(&sk_true));
                WARN_ON(!static_branch_unlikely(&sk_true));
                WARN_ON(static_branch_likely(&sk_false));
                WARN_ON(static_branch_unlikely(&sk_false));

                static_branch_disable(&sk_true);
                static_branch_enable(&sk_false);

                WARN_ON(static_key_enabled(&sk_true.key) == true);
                WARN_ON(static_key_enabled(&sk_false.key) == false);

                WARN_ON(static_branch_likely(&sk_true));
                WARN_ON(static_branch_unlikely(&sk_true));
                WARN_ON(!static_branch_likely(&sk_false));
                WARN_ON(!static_branch_unlikely(&sk_false));

                static_branch_enable(&sk_true);
                static_branch_disable(&sk_false);
        }

        return 0;
}
early_initcall(jump_label_test);
#endif /* STATIC_KEYS_SELFTEST */




















































































  123 
















 1487 





























































































  262 
  265 
























































    5 
    5 




























































































































































































































































  235 


















  234 








  238 














































  236 





















   86 







    3 



   86 
   84 
   86 
   86 












    3 
    3 
    3 
    3 
















































 6204 
 1288 



 5975 











   70 
   70 

   71 









































































   70 



   71 

   71 
   70 






   71 

   71 




























































































































































































































































































































































 1983 







 1987 
 1995 

 1991 






 2000 
 2009 

 2009 

 1993 

   79 
 1990 
















 1997 




 1983 
 1991 








































































































































































































































































































































































































































































































































  940 


  938 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
// SPDX-License-Identifier: GPL-2.0-only
/* Kernel thread helper functions.
 *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
 *   Copyright (C) 2009 Red Hat, Inc.
 *
 * Creation is done via kthreadd, so that we get a clean environment
 * even if we're invoked from userspace (think modprobe, hotplug cpu,
 * etc.).
 */
#include <uapi/linux/sched/types.h>
#include <linux/mm.h>
#include <linux/mmu_context.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/kthread.h>
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/cgroup.h>
#include <linux/cpuset.h>
#include <linux/unistd.h>
#include <linux/file.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/freezer.h>
#include <linux/ptrace.h>
#include <linux/uaccess.h>
#include <linux/numa.h>
#include <linux/sched/isolation.h>
#include <trace/events/sched.h>


static DEFINE_SPINLOCK(kthread_create_lock);
static LIST_HEAD(kthread_create_list);
struct task_struct *kthreadd_task;

static LIST_HEAD(kthreads_hotplug);
static DEFINE_MUTEX(kthreads_hotplug_lock);

struct kthread_create_info
{
        /* Information passed to kthread() from kthreadd. */
        char *full_name;
        int (*threadfn)(void *data);
        void *data;
        int node;

        /* Result passed back to kthread_create() from kthreadd. */
        struct task_struct *result;
        struct completion *done;

        struct list_head list;
};

struct kthread {
        unsigned long flags;
        unsigned int cpu;
        unsigned int node;
        int started;
        int result;
        int (*threadfn)(void *);
        void *data;
        struct completion parked;
        struct completion exited;
#ifdef CONFIG_BLK_CGROUP
        struct cgroup_subsys_state *blkcg_css;
#endif
        /* To store the full name if task comm is truncated. */
        char *full_name;
        struct task_struct *task;
        struct list_head hotplug_node;
        struct cpumask *preferred_affinity;
};

enum KTHREAD_BITS {
        KTHREAD_IS_PER_CPU = 0,
        KTHREAD_SHOULD_STOP,
        KTHREAD_SHOULD_PARK,
};

static inline struct kthread *to_kthread(struct task_struct *k)
{
        WARN_ON(!(k->flags & PF_KTHREAD));
        return k->worker_private;
}

/*
 * Variant of to_kthread() that doesn't assume @p is a kthread.
 *
 * When "(p->flags & PF_KTHREAD)" is set the task is a kthread and will
 * always remain a kthread.  For kthreads p->worker_private always
 * points to a struct kthread.  For tasks that are not kthreads
 * p->worker_private is used to point to other things.
 *
 * Return NULL for any task that is not a kthread.
 */
static inline struct kthread *__to_kthread(struct task_struct *p)
{
        void *kthread = p->worker_private;
        if (kthread && !(p->flags & PF_KTHREAD))
                kthread = NULL;
        return kthread;
}

void get_kthread_comm(char *buf, size_t buf_size, struct task_struct *tsk)
{
        struct kthread *kthread = to_kthread(tsk);

        if (!kthread || !kthread->full_name) {
                strscpy(buf, tsk->comm, buf_size);
                return;
        }

        strscpy_pad(buf, kthread->full_name, buf_size);
}

bool set_kthread_struct(struct task_struct *p)
{
        struct kthread *kthread;

        if (WARN_ON_ONCE(to_kthread(p)))
                return false;

        kthread = kzalloc(sizeof(*kthread), GFP_KERNEL);
        if (!kthread)
                return false;

        init_completion(&kthread->exited);
        init_completion(&kthread->parked);
        INIT_LIST_HEAD(&kthread->hotplug_node);
        p->vfork_done = &kthread->exited;

        kthread->task = p;
        kthread->node = tsk_fork_get_node(current);
        p->worker_private = kthread;
        return true;
}

void free_kthread_struct(struct task_struct *k)
{
        struct kthread *kthread;

        /*
         * Can be NULL if kmalloc() in set_kthread_struct() failed.
         */
        kthread = to_kthread(k);
        if (!kthread)
                return;

#ifdef CONFIG_BLK_CGROUP
        WARN_ON_ONCE(kthread->blkcg_css);
#endif
        k->worker_private = NULL;
        kfree(kthread->full_name);
        kfree(kthread);
}

/**
 * kthread_should_stop - should this kthread return now?
 *
 * When someone calls kthread_stop() on your kthread, it will be woken
 * and this will return true.  You should then return, and your return
 * value will be passed through to kthread_stop().
 */
bool kthread_should_stop(void)
{
        return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
}
EXPORT_SYMBOL(kthread_should_stop);

static bool __kthread_should_park(struct task_struct *k)
{
        return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(k)->flags);
}

/**
 * kthread_should_park - should this kthread park now?
 *
 * When someone calls kthread_park() on your kthread, it will be woken
 * and this will return true.  You should then do the necessary
 * cleanup and call kthread_parkme()
 *
 * Similar to kthread_should_stop(), but this keeps the thread alive
 * and in a park position. kthread_unpark() "restarts" the thread and
 * calls the thread function again.
 */
bool kthread_should_park(void)
{
        return __kthread_should_park(current);
}
EXPORT_SYMBOL_GPL(kthread_should_park);

bool kthread_should_stop_or_park(void)
{
        struct kthread *kthread = __to_kthread(current);

        if (!kthread)
                return false;

        return kthread->flags & (BIT(KTHREAD_SHOULD_STOP) | BIT(KTHREAD_SHOULD_PARK));
}

/**
 * kthread_freezable_should_stop - should this freezable kthread return now?
 * @was_frozen: optional out parameter, indicates whether %current was frozen
 *
 * kthread_should_stop() for freezable kthreads, which will enter
 * refrigerator if necessary.  This function is safe from kthread_stop() /
 * freezer deadlock and freezable kthreads should use this function instead
 * of calling try_to_freeze() directly.
 */
bool kthread_freezable_should_stop(bool *was_frozen)
{
        bool frozen = false;

        might_sleep();

        if (unlikely(freezing(current)))
                frozen = __refrigerator(true);

        if (was_frozen)
                *was_frozen = frozen;

        return kthread_should_stop();
}
EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);

/**
 * kthread_func - return the function specified on kthread creation
 * @task: kthread task in question
 *
 * Returns NULL if the task is not a kthread.
 */
void *kthread_func(struct task_struct *task)
{
        struct kthread *kthread = __to_kthread(task);
        if (kthread)
                return kthread->threadfn;
        return NULL;
}
EXPORT_SYMBOL_GPL(kthread_func);

/**
 * kthread_data - return data value specified on kthread creation
 * @task: kthread task in question
 *
 * Return the data value specified when kthread @task was created.
 * The caller is responsible for ensuring the validity of @task when
 * calling this function.
 */
void *kthread_data(struct task_struct *task)
{
        return to_kthread(task)->data;
}
EXPORT_SYMBOL_GPL(kthread_data);

/**
 * kthread_probe_data - speculative version of kthread_data()
 * @task: possible kthread task in question
 *
 * @task could be a kthread task.  Return the data value specified when it
 * was created if accessible.  If @task isn't a kthread task or its data is
 * inaccessible for any reason, %NULL is returned.  This function requires
 * that @task itself is safe to dereference.
 */
void *kthread_probe_data(struct task_struct *task)
{
        struct kthread *kthread = __to_kthread(task);
        void *data = NULL;

        if (kthread)
                copy_from_kernel_nofault(&data, &kthread->data, sizeof(data));
        return data;
}

static void __kthread_parkme(struct kthread *self)
{
        for (;;) {
                /*
                 * TASK_PARKED is a special state; we must serialize against
                 * possible pending wakeups to avoid store-store collisions on
                 * task->state.
                 *
                 * Such a collision might possibly result in the task state
                 * changin from TASK_PARKED and us failing the
                 * wait_task_inactive() in kthread_park().
                 */
                set_special_state(TASK_PARKED);
                if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
                        break;

                /*
                 * Thread is going to call schedule(), do not preempt it,
                 * or the caller of kthread_park() may spend more time in
                 * wait_task_inactive().
                 */
                preempt_disable();
                complete(&self->parked);
                schedule_preempt_disabled();
                preempt_enable();
        }
        __set_current_state(TASK_RUNNING);
}

void kthread_parkme(void)
{
        __kthread_parkme(to_kthread(current));
}
EXPORT_SYMBOL_GPL(kthread_parkme);

/**
 * kthread_exit - Cause the current kthread return @result to kthread_stop().
 * @result: The integer value to return to kthread_stop().
 *
 * While kthread_exit can be called directly, it exists so that
 * functions which do some additional work in non-modular code such as
 * module_put_and_kthread_exit can be implemented.
 *
 * Does not return.
 */
void __noreturn kthread_exit(long result)
{
        struct kthread *kthread = to_kthread(current);
        kthread->result = result;
        if (!list_empty(&kthread->hotplug_node)) {
                mutex_lock(&kthreads_hotplug_lock);
                list_del(&kthread->hotplug_node);
                mutex_unlock(&kthreads_hotplug_lock);

                if (kthread->preferred_affinity) {
                        kfree(kthread->preferred_affinity);
                        kthread->preferred_affinity = NULL;
                }
        }
        do_exit(0);
}
EXPORT_SYMBOL(kthread_exit);

/**
 * kthread_complete_and_exit - Exit the current kthread.
 * @comp: Completion to complete
 * @code: The integer value to return to kthread_stop().
 *
 * If present, complete @comp and then return code to kthread_stop().
 *
 * A kernel thread whose module may be removed after the completion of
 * @comp can use this function to exit safely.
 *
 * Does not return.
 */
void __noreturn kthread_complete_and_exit(struct completion *comp, long code)
{
        if (comp)
                complete(comp);

        kthread_exit(code);
}
EXPORT_SYMBOL(kthread_complete_and_exit);

static void kthread_fetch_affinity(struct kthread *kthread, struct cpumask *cpumask)
{
        const struct cpumask *pref;

        if (kthread->preferred_affinity) {
                pref = kthread->preferred_affinity;
        } else {
                if (WARN_ON_ONCE(kthread->node == NUMA_NO_NODE))
                        return;
                pref = cpumask_of_node(kthread->node);
        }

        cpumask_and(cpumask, pref, housekeeping_cpumask(HK_TYPE_KTHREAD));
        if (cpumask_empty(cpumask))
                cpumask_copy(cpumask, housekeeping_cpumask(HK_TYPE_KTHREAD));
}

static void kthread_affine_node(void)
{
        struct kthread *kthread = to_kthread(current);
        cpumask_var_t affinity;

        WARN_ON_ONCE(kthread_is_per_cpu(current));

        if (kthread->node == NUMA_NO_NODE) {
                housekeeping_affine(current, HK_TYPE_KTHREAD);
        } else {
                if (!zalloc_cpumask_var(&affinity, GFP_KERNEL)) {
                        WARN_ON_ONCE(1);
                        return;
                }

                mutex_lock(&kthreads_hotplug_lock);
                WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
                list_add_tail(&kthread->hotplug_node, &kthreads_hotplug);
                /*
                 * The node cpumask is racy when read from kthread() but:
                 * - a racing CPU going down will either fail on the subsequent
                 *   call to set_cpus_allowed_ptr() or be migrated to housekeepers
                 *   afterwards by the scheduler.
                 * - a racing CPU going up will be handled by kthreads_online_cpu()
                 */
                kthread_fetch_affinity(kthread, affinity);
                set_cpus_allowed_ptr(current, affinity);
                mutex_unlock(&kthreads_hotplug_lock);

                free_cpumask_var(affinity);
        }
}

static int kthread(void *_create)
{
        static const struct sched_param param = { .sched_priority = 0 };
        /* Copy data: it's on kthread's stack */
        struct kthread_create_info *create = _create;
        int (*threadfn)(void *data) = create->threadfn;
        void *data = create->data;
        struct completion *done;
        struct kthread *self;
        int ret;

        self = to_kthread(current);

        /* Release the structure when caller killed by a fatal signal. */
        done = xchg(&create->done, NULL);
        if (!done) {
                kfree(create->full_name);
                kfree(create);
                kthread_exit(-EINTR);
        }

        self->full_name = create->full_name;
        self->threadfn = threadfn;
        self->data = data;

        /*
         * The new thread inherited kthreadd's priority and CPU mask. Reset
         * back to default in case they have been changed.
         */
        sched_setscheduler_nocheck(current, SCHED_NORMAL, &param);

        /* OK, tell user we're spawned, wait for stop or wakeup */
        __set_current_state(TASK_UNINTERRUPTIBLE);
        create->result = current;
        /*
         * Thread is going to call schedule(), do not preempt it,
         * or the creator may spend more time in wait_task_inactive().
         */
        preempt_disable();
        complete(done);
        schedule_preempt_disabled();
        preempt_enable();

        self->started = 1;

        if (!(current->flags & PF_NO_SETAFFINITY) && !self->preferred_affinity)
                kthread_affine_node();

        ret = -EINTR;
        if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
                cgroup_kthread_ready();
                __kthread_parkme(self);
                ret = threadfn(data);
        }
        kthread_exit(ret);
}

/* called from kernel_clone() to get node information for about to be created task */
int tsk_fork_get_node(struct task_struct *tsk)
{
#ifdef CONFIG_NUMA
        if (tsk == kthreadd_task)
                return tsk->pref_node_fork;
#endif
        return NUMA_NO_NODE;
}

static void create_kthread(struct kthread_create_info *create)
{
        int pid;

#ifdef CONFIG_NUMA
        current->pref_node_fork = create->node;
#endif
        /* We want our own signal handler (we take no signals by default). */
        pid = kernel_thread(kthread, create, create->full_name,
                            CLONE_FS | CLONE_FILES | SIGCHLD);
        if (pid < 0) {
                /* Release the structure when caller killed by a fatal signal. */
                struct completion *done = xchg(&create->done, NULL);

                kfree(create->full_name);
                if (!done) {
                        kfree(create);
                        return;
                }
                create->result = ERR_PTR(pid);
                complete(done);
        }
}

static __printf(4, 0)
struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
                                                    void *data, int node,
                                                    const char namefmt[],
                                                    va_list args)
{
        DECLARE_COMPLETION_ONSTACK(done);
        struct task_struct *task;
        struct kthread_create_info *create = kmalloc(sizeof(*create),
                                                     GFP_KERNEL);

        if (!create)
                return ERR_PTR(-ENOMEM);
        create->threadfn = threadfn;
        create->data = data;
        create->node = node;
        create->done = &done;
        create->full_name = kvasprintf(GFP_KERNEL, namefmt, args);
        if (!create->full_name) {
                task = ERR_PTR(-ENOMEM);
                goto free_create;
        }

        spin_lock(&kthread_create_lock);
        list_add_tail(&create->list, &kthread_create_list);
        spin_unlock(&kthread_create_lock);

        wake_up_process(kthreadd_task);
        /*
         * Wait for completion in killable state, for I might be chosen by
         * the OOM killer while kthreadd is trying to allocate memory for
         * new kernel thread.
         */
        if (unlikely(wait_for_completion_killable(&done))) {
                /*
                 * If I was killed by a fatal signal before kthreadd (or new
                 * kernel thread) calls complete(), leave the cleanup of this
                 * structure to that thread.
                 */
                if (xchg(&create->done, NULL))
                        return ERR_PTR(-EINTR);
                /*
                 * kthreadd (or new kernel thread) will call complete()
                 * shortly.
                 */
                wait_for_completion(&done);
        }
        task = create->result;
free_create:
        kfree(create);
        return task;
}

/**
 * kthread_create_on_node - create a kthread.
 * @threadfn: the function to run until signal_pending(current).
 * @data: data ptr for @threadfn.
 * @node: task and thread structures for the thread are allocated on this node
 * @namefmt: printf-style name for the thread.
 *
 * Description: This helper function creates and names a kernel
 * thread.  The thread will be stopped: use wake_up_process() to start
 * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and
 * is affine to all CPUs.
 *
 * If thread is going to be bound on a particular cpu, give its node
 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
 * When woken, the thread will run @threadfn() with @data as its
 * argument. @threadfn() can either return directly if it is a
 * standalone thread for which no one will call kthread_stop(), or
 * return when 'kthread_should_stop()' is true (which means
 * kthread_stop() has been called).  The return value should be zero
 * or a negative error number; it will be passed to kthread_stop().
 *
 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
 */
struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
                                           void *data, int node,
                                           const char namefmt[],
                                           ...)
{
        struct task_struct *task;
        va_list args;

        va_start(args, namefmt);
        task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
        va_end(args);

        return task;
}
EXPORT_SYMBOL(kthread_create_on_node);

static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, unsigned int state)
{
        unsigned long flags;

        if (!wait_task_inactive(p, state)) {
                WARN_ON(1);
                return;
        }

        /* It's safe because the task is inactive. */
        raw_spin_lock_irqsave(&p->pi_lock, flags);
        do_set_cpus_allowed(p, mask);
        p->flags |= PF_NO_SETAFFINITY;
        raw_spin_unlock_irqrestore(&p->pi_lock, flags);
}

static void __kthread_bind(struct task_struct *p, unsigned int cpu, unsigned int state)
{
        __kthread_bind_mask(p, cpumask_of(cpu), state);
}

void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
{
        struct kthread *kthread = to_kthread(p);
        __kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
        WARN_ON_ONCE(kthread->started);
}

/**
 * kthread_bind - bind a just-created kthread to a cpu.
 * @p: thread created by kthread_create().
 * @cpu: cpu (might not be online, must be possible) for @k to run on.
 *
 * Description: This function is equivalent to set_cpus_allowed(),
 * except that @cpu doesn't need to be online, and the thread must be
 * stopped (i.e., just returned from kthread_create()).
 */
void kthread_bind(struct task_struct *p, unsigned int cpu)
{
        struct kthread *kthread = to_kthread(p);
        __kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
        WARN_ON_ONCE(kthread->started);
}
EXPORT_SYMBOL(kthread_bind);

/**
 * kthread_create_on_cpu - Create a cpu bound kthread
 * @threadfn: the function to run until signal_pending(current).
 * @data: data ptr for @threadfn.
 * @cpu: The cpu on which the thread should be bound,
 * @namefmt: printf-style name for the thread. Format is restricted
 *             to "name.*%u". Code fills in cpu number.
 *
 * Description: This helper function creates and names a kernel thread
 */
struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
                                          void *data, unsigned int cpu,
                                          const char *namefmt)
{
        struct task_struct *p;

        p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
                                   cpu);
        if (IS_ERR(p))
                return p;
        kthread_bind(p, cpu);
        /* CPU hotplug need to bind once again when unparking the thread. */
        to_kthread(p)->cpu = cpu;
        return p;
}
EXPORT_SYMBOL(kthread_create_on_cpu);

void kthread_set_per_cpu(struct task_struct *k, int cpu)
{
        struct kthread *kthread = to_kthread(k);
        if (!kthread)
                return;

        WARN_ON_ONCE(!(k->flags & PF_NO_SETAFFINITY));

        if (cpu < 0) {
                clear_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
                return;
        }

        kthread->cpu = cpu;
        set_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
}

bool kthread_is_per_cpu(struct task_struct *p)
{
        struct kthread *kthread = __to_kthread(p);
        if (!kthread)
                return false;

        return test_bit(KTHREAD_IS_PER_CPU, &kthread->flags);
}

/**
 * kthread_unpark - unpark a thread created by kthread_create().
 * @k:                thread created by kthread_create().
 *
 * Sets kthread_should_park() for @k to return false, wakes it, and
 * waits for it to return. If the thread is marked percpu then its
 * bound to the cpu again.
 */
void kthread_unpark(struct task_struct *k)
{
        struct kthread *kthread = to_kthread(k);

        if (!test_bit(KTHREAD_SHOULD_PARK, &kthread->flags))
                return;
        /*
         * Newly created kthread was parked when the CPU was offline.
         * The binding was lost and we need to set it again.
         */
        if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
                __kthread_bind(k, kthread->cpu, TASK_PARKED);

        clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
        /*
         * __kthread_parkme() will either see !SHOULD_PARK or get the wakeup.
         */
        wake_up_state(k, TASK_PARKED);
}
EXPORT_SYMBOL_GPL(kthread_unpark);

/**
 * kthread_park - park a thread created by kthread_create().
 * @k: thread created by kthread_create().
 *
 * Sets kthread_should_park() for @k to return true, wakes it, and
 * waits for it to return. This can also be called after kthread_create()
 * instead of calling wake_up_process(): the thread will park without
 * calling threadfn().
 *
 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
 * If called by the kthread itself just the park bit is set.
 */
int kthread_park(struct task_struct *k)
{
        struct kthread *kthread = to_kthread(k);

        if (WARN_ON(k->flags & PF_EXITING))
                return -ENOSYS;

        if (WARN_ON_ONCE(test_bit(KTHREAD_SHOULD_PARK, &kthread->flags)))
                return -EBUSY;

        set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
        if (k != current) {
                wake_up_process(k);
                /*
                 * Wait for __kthread_parkme() to complete(), this means we
                 * _will_ have TASK_PARKED and are about to call schedule().
                 */
                wait_for_completion(&kthread->parked);
                /*
                 * Now wait for that schedule() to complete and the task to
                 * get scheduled out.
                 */
                WARN_ON_ONCE(!wait_task_inactive(k, TASK_PARKED));
        }

        return 0;
}
EXPORT_SYMBOL_GPL(kthread_park);

/**
 * kthread_stop - stop a thread created by kthread_create().
 * @k: thread created by kthread_create().
 *
 * Sets kthread_should_stop() for @k to return true, wakes it, and
 * waits for it to exit. This can also be called after kthread_create()
 * instead of calling wake_up_process(): the thread will exit without
 * calling threadfn().
 *
 * If threadfn() may call kthread_exit() itself, the caller must ensure
 * task_struct can't go away.
 *
 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
 * was never called.
 */
int kthread_stop(struct task_struct *k)
{
        struct kthread *kthread;
        int ret;

        trace_sched_kthread_stop(k);

        get_task_struct(k);
        kthread = to_kthread(k);
        set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
        kthread_unpark(k);
        set_tsk_thread_flag(k, TIF_NOTIFY_SIGNAL);
        wake_up_process(k);
        wait_for_completion(&kthread->exited);
        ret = kthread->result;
        put_task_struct(k);

        trace_sched_kthread_stop_ret(ret);
        return ret;
}
EXPORT_SYMBOL(kthread_stop);

/**
 * kthread_stop_put - stop a thread and put its task struct
 * @k: thread created by kthread_create().
 *
 * Stops a thread created by kthread_create() and put its task_struct.
 * Only use when holding an extra task struct reference obtained by
 * calling get_task_struct().
 */
int kthread_stop_put(struct task_struct *k)
{
        int ret;

        ret = kthread_stop(k);
        put_task_struct(k);
        return ret;
}
EXPORT_SYMBOL(kthread_stop_put);

int kthreadd(void *unused)
{
        static const char comm[TASK_COMM_LEN] = "kthreadd";
        struct task_struct *tsk = current;

        /* Setup a clean context for our children to inherit. */
        set_task_comm(tsk, comm);
        ignore_signals(tsk);
        set_cpus_allowed_ptr(tsk, housekeeping_cpumask(HK_TYPE_KTHREAD));
        set_mems_allowed(node_states[N_MEMORY]);

        current->flags |= PF_NOFREEZE;
        cgroup_init_kthreadd();

        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);
                if (list_empty(&kthread_create_list))
                        schedule();
                __set_current_state(TASK_RUNNING);

                spin_lock(&kthread_create_lock);
                while (!list_empty(&kthread_create_list)) {
                        struct kthread_create_info *create;

                        create = list_entry(kthread_create_list.next,
                                            struct kthread_create_info, list);
                        list_del_init(&create->list);
                        spin_unlock(&kthread_create_lock);

                        create_kthread(create);

                        spin_lock(&kthread_create_lock);
                }
                spin_unlock(&kthread_create_lock);
        }

        return 0;
}

int kthread_affine_preferred(struct task_struct *p, const struct cpumask *mask)
{
        struct kthread *kthread = to_kthread(p);
        cpumask_var_t affinity;
        unsigned long flags;
        int ret = 0;

        if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE) || kthread->started) {
                WARN_ON(1);
                return -EINVAL;
        }

        WARN_ON_ONCE(kthread->preferred_affinity);

        if (!zalloc_cpumask_var(&affinity, GFP_KERNEL))
                return -ENOMEM;

        kthread->preferred_affinity = kzalloc(sizeof(struct cpumask), GFP_KERNEL);
        if (!kthread->preferred_affinity) {
                ret = -ENOMEM;
                goto out;
        }

        mutex_lock(&kthreads_hotplug_lock);
        cpumask_copy(kthread->preferred_affinity, mask);
        WARN_ON_ONCE(!list_empty(&kthread->hotplug_node));
        list_add_tail(&kthread->hotplug_node, &kthreads_hotplug);
        kthread_fetch_affinity(kthread, affinity);

        /* It's safe because the task is inactive. */
        raw_spin_lock_irqsave(&p->pi_lock, flags);
        do_set_cpus_allowed(p, affinity);
        raw_spin_unlock_irqrestore(&p->pi_lock, flags);

        mutex_unlock(&kthreads_hotplug_lock);
out:
        free_cpumask_var(affinity);

        return ret;
}
EXPORT_SYMBOL_GPL(kthread_affine_preferred);

/*
 * Re-affine kthreads according to their preferences
 * and the newly online CPU. The CPU down part is handled
 * by select_fallback_rq() which default re-affines to
 * housekeepers from other nodes in case the preferred
 * affinity doesn't apply anymore.
 */
static int kthreads_online_cpu(unsigned int cpu)
{
        cpumask_var_t affinity;
        struct kthread *k;
        int ret;

        guard(mutex)(&kthreads_hotplug_lock);

        if (list_empty(&kthreads_hotplug))
                return 0;

        if (!zalloc_cpumask_var(&affinity, GFP_KERNEL))
                return -ENOMEM;

        ret = 0;

        list_for_each_entry(k, &kthreads_hotplug, hotplug_node) {
                if (WARN_ON_ONCE((k->task->flags & PF_NO_SETAFFINITY) ||
                                 kthread_is_per_cpu(k->task))) {
                        ret = -EINVAL;
                        continue;
                }
                kthread_fetch_affinity(k, affinity);
                set_cpus_allowed_ptr(k->task, affinity);
        }

        free_cpumask_var(affinity);

        return ret;
}

static int kthreads_init(void)
{
        return cpuhp_setup_state(CPUHP_AP_KTHREADS_ONLINE, "kthreads:online",
                                kthreads_online_cpu, NULL);
}
early_initcall(kthreads_init);

void __kthread_init_worker(struct kthread_worker *worker,
                                const char *name,
                                struct lock_class_key *key)
{
        memset(worker, 0, sizeof(struct kthread_worker));
        raw_spin_lock_init(&worker->lock);
        lockdep_set_class_and_name(&worker->lock, key, name);
        INIT_LIST_HEAD(&worker->work_list);
        INIT_LIST_HEAD(&worker->delayed_work_list);
}
EXPORT_SYMBOL_GPL(__kthread_init_worker);

/**
 * kthread_worker_fn - kthread function to process kthread_worker
 * @worker_ptr: pointer to initialized kthread_worker
 *
 * This function implements the main cycle of kthread worker. It processes
 * work_list until it is stopped with kthread_stop(). It sleeps when the queue
 * is empty.
 *
 * The works are not allowed to keep any locks, disable preemption or interrupts
 * when they finish. There is defined a safe point for freezing when one work
 * finishes and before a new one is started.
 *
 * Also the works must not be handled by more than one worker at the same time,
 * see also kthread_queue_work().
 */
int kthread_worker_fn(void *worker_ptr)
{
        struct kthread_worker *worker = worker_ptr;
        struct kthread_work *work;

        /*
         * FIXME: Update the check and remove the assignment when all kthread
         * worker users are created using kthread_create_worker*() functions.
         */
        WARN_ON(worker->task && worker->task != current);
        worker->task = current;

        if (worker->flags & KTW_FREEZABLE)
                set_freezable();

repeat:
        set_current_state(TASK_INTERRUPTIBLE);        /* mb paired w/ kthread_stop */

        if (kthread_should_stop()) {
                __set_current_state(TASK_RUNNING);
                raw_spin_lock_irq(&worker->lock);
                worker->task = NULL;
                raw_spin_unlock_irq(&worker->lock);
                return 0;
        }

        work = NULL;
        raw_spin_lock_irq(&worker->lock);
        if (!list_empty(&worker->work_list)) {
                work = list_first_entry(&worker->work_list,
                                        struct kthread_work, node);
                list_del_init(&work->node);
        }
        worker->current_work = work;
        raw_spin_unlock_irq(&worker->lock);

        if (work) {
                kthread_work_func_t func = work->func;
                __set_current_state(TASK_RUNNING);
                trace_sched_kthread_work_execute_start(work);
                work->func(work);
                /*
                 * Avoid dereferencing work after this point.  The trace
                 * event only cares about the address.
                 */
                trace_sched_kthread_work_execute_end(work, func);
        } else if (!freezing(current)) {
                schedule();
        } else {
                /*
                 * Handle the case where the current remains
                 * TASK_INTERRUPTIBLE. try_to_freeze() expects
                 * the current to be TASK_RUNNING.
                 */
                __set_current_state(TASK_RUNNING);
        }

        try_to_freeze();
        cond_resched();
        goto repeat;
}
EXPORT_SYMBOL_GPL(kthread_worker_fn);

static __printf(3, 0) struct kthread_worker *
__kthread_create_worker_on_node(unsigned int flags, int node,
                                const char namefmt[], va_list args)
{
        struct kthread_worker *worker;
        struct task_struct *task;

        worker = kzalloc(sizeof(*worker), GFP_KERNEL);
        if (!worker)
                return ERR_PTR(-ENOMEM);

        kthread_init_worker(worker);

        task = __kthread_create_on_node(kthread_worker_fn, worker,
                                        node, namefmt, args);
        if (IS_ERR(task))
                goto fail_task;

        worker->flags = flags;
        worker->task = task;

        return worker;

fail_task:
        kfree(worker);
        return ERR_CAST(task);
}

/**
 * kthread_create_worker_on_node - create a kthread worker
 * @flags: flags modifying the default behavior of the worker
 * @node: task structure for the thread is allocated on this node
 * @namefmt: printf-style name for the kthread worker (task).
 *
 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
 * when the caller was killed by a fatal signal.
 */
struct kthread_worker *
kthread_create_worker_on_node(unsigned int flags, int node, const char namefmt[], ...)
{
        struct kthread_worker *worker;
        va_list args;

        va_start(args, namefmt);
        worker = __kthread_create_worker_on_node(flags, node, namefmt, args);
        va_end(args);

        return worker;
}
EXPORT_SYMBOL(kthread_create_worker_on_node);

/**
 * kthread_create_worker_on_cpu - create a kthread worker and bind it
 *        to a given CPU and the associated NUMA node.
 * @cpu: CPU number
 * @flags: flags modifying the default behavior of the worker
 * @namefmt: printf-style name for the thread. Format is restricted
 *             to "name.*%u". Code fills in cpu number.
 *
 * Use a valid CPU number if you want to bind the kthread worker
 * to the given CPU and the associated NUMA node.
 *
 * A good practice is to add the cpu number also into the worker name.
 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
 *
 * CPU hotplug:
 * The kthread worker API is simple and generic. It just provides a way
 * to create, use, and destroy workers.
 *
 * It is up to the API user how to handle CPU hotplug. They have to decide
 * how to handle pending work items, prevent queuing new ones, and
 * restore the functionality when the CPU goes off and on. There are a
 * few catches:
 *
 *    - CPU affinity gets lost when it is scheduled on an offline CPU.
 *
 *    - The worker might not exist when the CPU was off when the user
 *      created the workers.
 *
 * Good practice is to implement two CPU hotplug callbacks and to
 * destroy/create the worker when the CPU goes down/up.
 *
 * Return:
 * The pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
 * when the caller was killed by a fatal signal.
 */
struct kthread_worker *
kthread_create_worker_on_cpu(int cpu, unsigned int flags,
                             const char namefmt[])
{
        struct kthread_worker *worker;

        worker = kthread_create_worker_on_node(flags, cpu_to_node(cpu), namefmt, cpu);
        if (!IS_ERR(worker))
                kthread_bind(worker->task, cpu);

        return worker;
}
EXPORT_SYMBOL(kthread_create_worker_on_cpu);

/*
 * Returns true when the work could not be queued at the moment.
 * It happens when it is already pending in a worker list
 * or when it is being cancelled.
 */
static inline bool queuing_blocked(struct kthread_worker *worker,
                                   struct kthread_work *work)
{
        lockdep_assert_held(&worker->lock);

        return !list_empty(&work->node) || work->canceling;
}

static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
                                             struct kthread_work *work)
{
        lockdep_assert_held(&worker->lock);
        WARN_ON_ONCE(!list_empty(&work->node));
        /* Do not use a work with >1 worker, see kthread_queue_work() */
        WARN_ON_ONCE(work->worker && work->worker != worker);
}

/* insert @work before @pos in @worker */
static void kthread_insert_work(struct kthread_worker *worker,
                                struct kthread_work *work,
                                struct list_head *pos)
{
        kthread_insert_work_sanity_check(worker, work);

        trace_sched_kthread_work_queue_work(worker, work);

        list_add_tail(&work->node, pos);
        work->worker = worker;
        if (!worker->current_work && likely(worker->task))
                wake_up_process(worker->task);
}

/**
 * kthread_queue_work - queue a kthread_work
 * @worker: target kthread_worker
 * @work: kthread_work to queue
 *
 * Queue @work to work processor @task for async execution.  @task
 * must have been created with kthread_create_worker().  Returns %true
 * if @work was successfully queued, %false if it was already pending.
 *
 * Reinitialize the work if it needs to be used by another worker.
 * For example, when the worker was stopped and started again.
 */
bool kthread_queue_work(struct kthread_worker *worker,
                        struct kthread_work *work)
{
        bool ret = false;
        unsigned long flags;

        raw_spin_lock_irqsave(&worker->lock, flags);
        if (!queuing_blocked(worker, work)) {
                kthread_insert_work(worker, work, &worker->work_list);
                ret = true;
        }
        raw_spin_unlock_irqrestore(&worker->lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(kthread_queue_work);

/**
 * kthread_delayed_work_timer_fn - callback that queues the associated kthread
 *        delayed work when the timer expires.
 * @t: pointer to the expired timer
 *
 * The format of the function is defined by struct timer_list.
 * It should have been called from irqsafe timer with irq already off.
 */
void kthread_delayed_work_timer_fn(struct timer_list *t)
{
        struct kthread_delayed_work *dwork = timer_container_of(dwork, t,
                                                                timer);
        struct kthread_work *work = &dwork->work;
        struct kthread_worker *worker = work->worker;
        unsigned long flags;

        /*
         * This might happen when a pending work is reinitialized.
         * It means that it is used a wrong way.
         */
        if (WARN_ON_ONCE(!worker))
                return;

        raw_spin_lock_irqsave(&worker->lock, flags);
        /* Work must not be used with >1 worker, see kthread_queue_work(). */
        WARN_ON_ONCE(work->worker != worker);

        /* Move the work from worker->delayed_work_list. */
        WARN_ON_ONCE(list_empty(&work->node));
        list_del_init(&work->node);
        if (!work->canceling)
                kthread_insert_work(worker, work, &worker->work_list);

        raw_spin_unlock_irqrestore(&worker->lock, flags);
}
EXPORT_SYMBOL(kthread_delayed_work_timer_fn);

static void __kthread_queue_delayed_work(struct kthread_worker *worker,
                                         struct kthread_delayed_work *dwork,
                                         unsigned long delay)
{
        struct timer_list *timer = &dwork->timer;
        struct kthread_work *work = &dwork->work;

        WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);

        /*
         * If @delay is 0, queue @dwork->work immediately.  This is for
         * both optimization and correctness.  The earliest @timer can
         * expire is on the closest next tick and delayed_work users depend
         * on that there's no such delay when @delay is 0.
         */
        if (!delay) {
                kthread_insert_work(worker, work, &worker->work_list);
                return;
        }

        /* Be paranoid and try to detect possible races already now. */
        kthread_insert_work_sanity_check(worker, work);

        list_add(&work->node, &worker->delayed_work_list);
        work->worker = worker;
        timer->expires = jiffies + delay;
        add_timer(timer);
}

/**
 * kthread_queue_delayed_work - queue the associated kthread work
 *        after a delay.
 * @worker: target kthread_worker
 * @dwork: kthread_delayed_work to queue
 * @delay: number of jiffies to wait before queuing
 *
 * If the work has not been pending it starts a timer that will queue
 * the work after the given @delay. If @delay is zero, it queues the
 * work immediately.
 *
 * Return: %false if the @work has already been pending. It means that
 * either the timer was running or the work was queued. It returns %true
 * otherwise.
 */
bool kthread_queue_delayed_work(struct kthread_worker *worker,
                                struct kthread_delayed_work *dwork,
                                unsigned long delay)
{
        struct kthread_work *work = &dwork->work;
        unsigned long flags;
        bool ret = false;

        raw_spin_lock_irqsave(&worker->lock, flags);

        if (!queuing_blocked(worker, work)) {
                __kthread_queue_delayed_work(worker, dwork, delay);
                ret = true;
        }

        raw_spin_unlock_irqrestore(&worker->lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);

struct kthread_flush_work {
        struct kthread_work        work;
        struct completion        done;
};

static void kthread_flush_work_fn(struct kthread_work *work)
{
        struct kthread_flush_work *fwork =
                container_of(work, struct kthread_flush_work, work);
        complete(&fwork->done);
}

/**
 * kthread_flush_work - flush a kthread_work
 * @work: work to flush
 *
 * If @work is queued or executing, wait for it to finish execution.
 */
void kthread_flush_work(struct kthread_work *work)
{
        struct kthread_flush_work fwork = {
                KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
                COMPLETION_INITIALIZER_ONSTACK(fwork.done),
        };
        struct kthread_worker *worker;
        bool noop = false;

        worker = work->worker;
        if (!worker)
                return;

        raw_spin_lock_irq(&worker->lock);
        /* Work must not be used with >1 worker, see kthread_queue_work(). */
        WARN_ON_ONCE(work->worker != worker);

        if (!list_empty(&work->node))
                kthread_insert_work(worker, &fwork.work, work->node.next);
        else if (worker->current_work == work)
                kthread_insert_work(worker, &fwork.work,
                                    worker->work_list.next);
        else
                noop = true;

        raw_spin_unlock_irq(&worker->lock);

        if (!noop)
                wait_for_completion(&fwork.done);
}
EXPORT_SYMBOL_GPL(kthread_flush_work);

/*
 * Make sure that the timer is neither set nor running and could
 * not manipulate the work list_head any longer.
 *
 * The function is called under worker->lock. The lock is temporary
 * released but the timer can't be set again in the meantime.
 */
static void kthread_cancel_delayed_work_timer(struct kthread_work *work,
                                              unsigned long *flags)
{
        struct kthread_delayed_work *dwork =
                container_of(work, struct kthread_delayed_work, work);
        struct kthread_worker *worker = work->worker;

        /*
         * timer_delete_sync() must be called to make sure that the timer
         * callback is not running. The lock must be temporary released
         * to avoid a deadlock with the callback. In the meantime,
         * any queuing is blocked by setting the canceling counter.
         */
        work->canceling++;
        raw_spin_unlock_irqrestore(&worker->lock, *flags);
        timer_delete_sync(&dwork->timer);
        raw_spin_lock_irqsave(&worker->lock, *flags);
        work->canceling--;
}

/*
 * This function removes the work from the worker queue.
 *
 * It is called under worker->lock. The caller must make sure that
 * the timer used by delayed work is not running, e.g. by calling
 * kthread_cancel_delayed_work_timer().
 *
 * The work might still be in use when this function finishes. See the
 * current_work proceed by the worker.
 *
 * Return: %true if @work was pending and successfully canceled,
 *        %false if @work was not pending
 */
static bool __kthread_cancel_work(struct kthread_work *work)
{
        /*
         * Try to remove the work from a worker list. It might either
         * be from worker->work_list or from worker->delayed_work_list.
         */
        if (!list_empty(&work->node)) {
                list_del_init(&work->node);
                return true;
        }

        return false;
}

/**
 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
 * @worker: kthread worker to use
 * @dwork: kthread delayed work to queue
 * @delay: number of jiffies to wait before queuing
 *
 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
 * modify @dwork's timer so that it expires after @delay. If @delay is zero,
 * @work is guaranteed to be queued immediately.
 *
 * Return: %false if @dwork was idle and queued, %true otherwise.
 *
 * A special case is when the work is being canceled in parallel.
 * It might be caused either by the real kthread_cancel_delayed_work_sync()
 * or yet another kthread_mod_delayed_work() call. We let the other command
 * win and return %true here. The return value can be used for reference
 * counting and the number of queued works stays the same. Anyway, the caller
 * is supposed to synchronize these operations a reasonable way.
 *
 * This function is safe to call from any context including IRQ handler.
 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
 * for details.
 */
bool kthread_mod_delayed_work(struct kthread_worker *worker,
                              struct kthread_delayed_work *dwork,
                              unsigned long delay)
{
        struct kthread_work *work = &dwork->work;
        unsigned long flags;
        int ret;

        raw_spin_lock_irqsave(&worker->lock, flags);

        /* Do not bother with canceling when never queued. */
        if (!work->worker) {
                ret = false;
                goto fast_queue;
        }

        /* Work must not be used with >1 worker, see kthread_queue_work() */
        WARN_ON_ONCE(work->worker != worker);

        /*
         * Temporary cancel the work but do not fight with another command
         * that is canceling the work as well.
         *
         * It is a bit tricky because of possible races with another
         * mod_delayed_work() and cancel_delayed_work() callers.
         *
         * The timer must be canceled first because worker->lock is released
         * when doing so. But the work can be removed from the queue (list)
         * only when it can be queued again so that the return value can
         * be used for reference counting.
         */
        kthread_cancel_delayed_work_timer(work, &flags);
        if (work->canceling) {
                /* The number of works in the queue does not change. */
                ret = true;
                goto out;
        }
        ret = __kthread_cancel_work(work);

fast_queue:
        __kthread_queue_delayed_work(worker, dwork, delay);
out:
        raw_spin_unlock_irqrestore(&worker->lock, flags);
        return ret;
}
EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);

static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
{
        struct kthread_worker *worker = work->worker;
        unsigned long flags;
        int ret = false;

        if (!worker)
                goto out;

        raw_spin_lock_irqsave(&worker->lock, flags);
        /* Work must not be used with >1 worker, see kthread_queue_work(). */
        WARN_ON_ONCE(work->worker != worker);

        if (is_dwork)
                kthread_cancel_delayed_work_timer(work, &flags);

        ret = __kthread_cancel_work(work);

        if (worker->current_work != work)
                goto out_fast;

        /*
         * The work is in progress and we need to wait with the lock released.
         * In the meantime, block any queuing by setting the canceling counter.
         */
        work->canceling++;
        raw_spin_unlock_irqrestore(&worker->lock, flags);
        kthread_flush_work(work);
        raw_spin_lock_irqsave(&worker->lock, flags);
        work->canceling--;

out_fast:
        raw_spin_unlock_irqrestore(&worker->lock, flags);
out:
        return ret;
}

/**
 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
 * @work: the kthread work to cancel
 *
 * Cancel @work and wait for its execution to finish.  This function
 * can be used even if the work re-queues itself. On return from this
 * function, @work is guaranteed to be not pending or executing on any CPU.
 *
 * kthread_cancel_work_sync(&delayed_work->work) must not be used for
 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
 *
 * The caller must ensure that the worker on which @work was last
 * queued can't be destroyed before this function returns.
 *
 * Return: %true if @work was pending, %false otherwise.
 */
bool kthread_cancel_work_sync(struct kthread_work *work)
{
        return __kthread_cancel_work_sync(work, false);
}
EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);

/**
 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
 *        wait for it to finish.
 * @dwork: the kthread delayed work to cancel
 *
 * This is kthread_cancel_work_sync() for delayed works.
 *
 * Return: %true if @dwork was pending, %false otherwise.
 */
bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
{
        return __kthread_cancel_work_sync(&dwork->work, true);
}
EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);

/**
 * kthread_flush_worker - flush all current works on a kthread_worker
 * @worker: worker to flush
 *
 * Wait until all currently executing or pending works on @worker are
 * finished.
 */
void kthread_flush_worker(struct kthread_worker *worker)
{
        struct kthread_flush_work fwork = {
                KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
                COMPLETION_INITIALIZER_ONSTACK(fwork.done),
        };

        kthread_queue_work(worker, &fwork.work);
        wait_for_completion(&fwork.done);
}
EXPORT_SYMBOL_GPL(kthread_flush_worker);

/**
 * kthread_destroy_worker - destroy a kthread worker
 * @worker: worker to be destroyed
 *
 * Flush and destroy @worker.  The simple flush is enough because the kthread
 * worker API is used only in trivial scenarios.  There are no multi-step state
 * machines needed.
 *
 * Note that this function is not responsible for handling delayed work, so
 * caller should be responsible for queuing or canceling all delayed work items
 * before invoke this function.
 */
void kthread_destroy_worker(struct kthread_worker *worker)
{
        struct task_struct *task;

        task = worker->task;
        if (WARN_ON(!task))
                return;

        kthread_flush_worker(worker);
        kthread_stop(task);
        WARN_ON(!list_empty(&worker->delayed_work_list));
        WARN_ON(!list_empty(&worker->work_list));
        kfree(worker);
}
EXPORT_SYMBOL(kthread_destroy_worker);

/**
 * kthread_use_mm - make the calling kthread operate on an address space
 * @mm: address space to operate on
 */
void kthread_use_mm(struct mm_struct *mm)
{
        struct mm_struct *active_mm;
        struct task_struct *tsk = current;

        WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
        WARN_ON_ONCE(tsk->mm);

        /*
         * It is possible for mm to be the same as tsk->active_mm, but
         * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
         * because these references are not equivalent.
         */
        mmgrab(mm);

        task_lock(tsk);
        /* Hold off tlb flush IPIs while switching mm's */
        local_irq_disable();
        active_mm = tsk->active_mm;
        tsk->active_mm = mm;
        tsk->mm = mm;
        membarrier_update_current_mm(mm);
        switch_mm_irqs_off(active_mm, mm, tsk);
        local_irq_enable();
        task_unlock(tsk);
#ifdef finish_arch_post_lock_switch
        finish_arch_post_lock_switch();
#endif

        /*
         * When a kthread starts operating on an address space, the loop
         * in membarrier_{private,global}_expedited() may not observe
         * that tsk->mm, and not issue an IPI. Membarrier requires a
         * memory barrier after storing to tsk->mm, before accessing
         * user-space memory. A full memory barrier for membarrier
         * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
         * mmdrop_lazy_tlb().
         */
        mmdrop_lazy_tlb(active_mm);
}
EXPORT_SYMBOL_GPL(kthread_use_mm);

/**
 * kthread_unuse_mm - reverse the effect of kthread_use_mm()
 * @mm: address space to operate on
 */
void kthread_unuse_mm(struct mm_struct *mm)
{
        struct task_struct *tsk = current;

        WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
        WARN_ON_ONCE(!tsk->mm);

        task_lock(tsk);
        /*
         * When a kthread stops operating on an address space, the loop
         * in membarrier_{private,global}_expedited() may not observe
         * that tsk->mm, and not issue an IPI. Membarrier requires a
         * memory barrier after accessing user-space memory, before
         * clearing tsk->mm.
         */
        smp_mb__after_spinlock();
        local_irq_disable();
        tsk->mm = NULL;
        membarrier_update_current_mm(NULL);
        mmgrab_lazy_tlb(mm);
        /* active_mm is still 'mm' */
        enter_lazy_tlb(mm, tsk);
        local_irq_enable();
        task_unlock(tsk);

        mmdrop(mm);
}
EXPORT_SYMBOL_GPL(kthread_unuse_mm);

#ifdef CONFIG_BLK_CGROUP
/**
 * kthread_associate_blkcg - associate blkcg to current kthread
 * @css: the cgroup info
 *
 * Current thread must be a kthread. The thread is running jobs on behalf of
 * other threads. In some cases, we expect the jobs attach cgroup info of
 * original threads instead of that of current thread. This function stores
 * original thread's cgroup info in current kthread context for later
 * retrieval.
 */
void kthread_associate_blkcg(struct cgroup_subsys_state *css)
{
        struct kthread *kthread;

        if (!(current->flags & PF_KTHREAD))
                return;
        kthread = to_kthread(current);
        if (!kthread)
                return;

        if (kthread->blkcg_css) {
                css_put(kthread->blkcg_css);
                kthread->blkcg_css = NULL;
        }
        if (css) {
                css_get(css);
                kthread->blkcg_css = css;
        }
}
EXPORT_SYMBOL(kthread_associate_blkcg);

/**
 * kthread_blkcg - get associated blkcg css of current kthread
 *
 * Current thread must be a kthread.
 */
struct cgroup_subsys_state *kthread_blkcg(void)
{
        struct kthread *kthread;

        if (current->flags & PF_KTHREAD) {
                kthread = to_kthread(current);
                if (kthread)
                        return kthread->blkcg_css;
        }
        return NULL;
}
#endif




























































   13 







    2 



   11 






    1 




    1 





    1 

































    5 








    3 

















    5 







    2 



    3 







    1 





    1 










    1 
















    3 



    3 



    3 








    1 










   12 





   12 

    1 




    1 


    2 





    3 




    1 




    1 





    2 





    3 




    1 




    1 










   10 







   10 







   10 

    1 


    1 



    1 






    1 

    2 







    1 








    1 







    1 





    1 














































    3 





    3 


































    4 





    4 



    1 




    3 



    1 
    3 

    1 




    1 








    2 


    2 














    2 




    1 


    1 
















    1 













    1 












    5 



    5 

    5 


    1 

    4 















    4 


    4 

    4 





    9 





    3 


    6 


    1 







    5 


    4 



    1 
















    1 
    4 





    6 






    5 







    5 

    2 


    3 


    2 














    1 











    1 
















































































   10 




   10 

    9 
    1 


    1 





    1 





    1 



    5 



    1 









    1 











    3 





    3 



    2 

    1 





    1 




    1 





    1 
































































































    6 


    6 

    6 




    6 

    6 






    3 





























    3 











    6 








    4 


    4 

    1 




    2 
    1 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2011  Intel Corporation. All rights reserved.
 */

#define pr_fmt(fmt) "llcp: %s: " fmt, __func__

#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/nfc.h>
#include <linux/sched/signal.h>

#include "nfc.h"
#include "llcp.h"

static int sock_wait_state(struct sock *sk, int state, unsigned long timeo)
{
        DECLARE_WAITQUEUE(wait, current);
        int err = 0;

        pr_debug("sk %p", sk);

        add_wait_queue(sk_sleep(sk), &wait);
        set_current_state(TASK_INTERRUPTIBLE);

        while (sk->sk_state != state) {
                if (!timeo) {
                        err = -EINPROGRESS;
                        break;
                }

                if (signal_pending(current)) {
                        err = sock_intr_errno(timeo);
                        break;
                }

                release_sock(sk);
                timeo = schedule_timeout(timeo);
                lock_sock(sk);
                set_current_state(TASK_INTERRUPTIBLE);

                err = sock_error(sk);
                if (err)
                        break;
        }

        __set_current_state(TASK_RUNNING);
        remove_wait_queue(sk_sleep(sk), &wait);
        return err;
}

static struct proto llcp_sock_proto = {
        .name     = "NFC_LLCP",
        .owner    = THIS_MODULE,
        .obj_size = sizeof(struct nfc_llcp_sock),
};

static int llcp_sock_bind(struct socket *sock, struct sockaddr *addr, int alen)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        struct nfc_llcp_local *local;
        struct nfc_dev *dev;
        struct sockaddr_nfc_llcp llcp_addr;
        int len, ret = 0;

        if (!addr || alen < offsetofend(struct sockaddr, sa_family) ||
            addr->sa_family != AF_NFC)
                return -EINVAL;

        pr_debug("sk %p addr %p family %d\n", sk, addr, addr->sa_family);

        memset(&llcp_addr, 0, sizeof(llcp_addr));
        len = min_t(unsigned int, sizeof(llcp_addr), alen);
        memcpy(&llcp_addr, addr, len);

        /* This is going to be a listening socket, dsap must be 0 */
        if (llcp_addr.dsap != 0)
                return -EINVAL;

        lock_sock(sk);

        if (sk->sk_state != LLCP_CLOSED) {
                ret = -EBADFD;
                goto error;
        }

        dev = nfc_get_device(llcp_addr.dev_idx);
        if (dev == NULL) {
                ret = -ENODEV;
                goto error;
        }

        local = nfc_llcp_find_local(dev);
        if (local == NULL) {
                ret = -ENODEV;
                goto put_dev;
        }

        llcp_sock->dev = dev;
        llcp_sock->local = local;
        llcp_sock->nfc_protocol = llcp_addr.nfc_protocol;
        llcp_sock->service_name_len = min_t(unsigned int,
                                            llcp_addr.service_name_len,
                                            NFC_LLCP_MAX_SERVICE_NAME);
        llcp_sock->service_name = kmemdup(llcp_addr.service_name,
                                          llcp_sock->service_name_len,
                                          GFP_KERNEL);
        if (!llcp_sock->service_name) {
                ret = -ENOMEM;
                goto sock_llcp_put_local;
        }
        llcp_sock->ssap = nfc_llcp_get_sdp_ssap(local, llcp_sock);
        if (llcp_sock->ssap == LLCP_SAP_MAX) {
                ret = -EADDRINUSE;
                goto free_service_name;
        }

        llcp_sock->reserved_ssap = llcp_sock->ssap;

        nfc_llcp_sock_link(&local->sockets, sk);

        pr_debug("Socket bound to SAP %d\n", llcp_sock->ssap);

        sk->sk_state = LLCP_BOUND;
        nfc_put_device(dev);
        release_sock(sk);

        return 0;

free_service_name:
        kfree(llcp_sock->service_name);
        llcp_sock->service_name = NULL;

sock_llcp_put_local:
        nfc_llcp_local_put(llcp_sock->local);
        llcp_sock->local = NULL;
        llcp_sock->dev = NULL;

put_dev:
        nfc_put_device(dev);

error:
        release_sock(sk);
        return ret;
}

static int llcp_raw_sock_bind(struct socket *sock, struct sockaddr *addr,
                              int alen)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        struct nfc_llcp_local *local;
        struct nfc_dev *dev;
        struct sockaddr_nfc_llcp llcp_addr;
        int len, ret = 0;

        if (!addr || alen < offsetofend(struct sockaddr, sa_family) ||
            addr->sa_family != AF_NFC)
                return -EINVAL;

        pr_debug("sk %p addr %p family %d\n", sk, addr, addr->sa_family);

        memset(&llcp_addr, 0, sizeof(llcp_addr));
        len = min_t(unsigned int, sizeof(llcp_addr), alen);
        memcpy(&llcp_addr, addr, len);

        lock_sock(sk);

        if (sk->sk_state != LLCP_CLOSED) {
                ret = -EBADFD;
                goto error;
        }

        dev = nfc_get_device(llcp_addr.dev_idx);
        if (dev == NULL) {
                ret = -ENODEV;
                goto error;
        }

        local = nfc_llcp_find_local(dev);
        if (local == NULL) {
                ret = -ENODEV;
                goto put_dev;
        }

        llcp_sock->dev = dev;
        llcp_sock->local = local;
        llcp_sock->nfc_protocol = llcp_addr.nfc_protocol;

        nfc_llcp_sock_link(&local->raw_sockets, sk);

        sk->sk_state = LLCP_BOUND;

put_dev:
        nfc_put_device(dev);

error:
        release_sock(sk);
        return ret;
}

static int llcp_sock_listen(struct socket *sock, int backlog)
{
        struct sock *sk = sock->sk;
        int ret = 0;

        pr_debug("sk %p backlog %d\n", sk, backlog);

        lock_sock(sk);

        if ((sock->type != SOCK_SEQPACKET && sock->type != SOCK_STREAM) ||
            sk->sk_state != LLCP_BOUND) {
                ret = -EBADFD;
                goto error;
        }

        sk->sk_max_ack_backlog = backlog;
        sk->sk_ack_backlog = 0;

        pr_debug("Socket listening\n");
        sk->sk_state = LLCP_LISTEN;

error:
        release_sock(sk);

        return ret;
}

static int nfc_llcp_setsockopt(struct socket *sock, int level, int optname,
                               sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        u32 opt;
        int err = 0;

        pr_debug("%p optname %d\n", sk, optname);

        if (level != SOL_NFC)
                return -ENOPROTOOPT;

        lock_sock(sk);

        switch (optname) {
        case NFC_LLCP_RW:
                if (sk->sk_state == LLCP_CONNECTED ||
                    sk->sk_state == LLCP_BOUND ||
                    sk->sk_state == LLCP_LISTEN) {
                        err = -EINVAL;
                        break;
                }

                err = copy_safe_from_sockptr(&opt, sizeof(opt),
                                             optval, optlen);
                if (err)
                        break;

                if (opt > LLCP_MAX_RW) {
                        err = -EINVAL;
                        break;
                }

                llcp_sock->rw = (u8) opt;

                break;

        case NFC_LLCP_MIUX:
                if (sk->sk_state == LLCP_CONNECTED ||
                    sk->sk_state == LLCP_BOUND ||
                    sk->sk_state == LLCP_LISTEN) {
                        err = -EINVAL;
                        break;
                }

                err = copy_safe_from_sockptr(&opt, sizeof(opt),
                                             optval, optlen);
                if (err)
                        break;

                if (opt > LLCP_MAX_MIUX) {
                        err = -EINVAL;
                        break;
                }

                llcp_sock->miux = cpu_to_be16((u16) opt);

                break;

        default:
                err = -ENOPROTOOPT;
                break;
        }

        release_sock(sk);

        pr_debug("%p rw %d miux %d\n", llcp_sock,
                 llcp_sock->rw, llcp_sock->miux);

        return err;
}

static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname,
                               char __user *optval, int __user *optlen)
{
        struct nfc_llcp_local *local;
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        int len, err = 0;
        u16 miux, remote_miu;
        u8 rw;

        pr_debug("%p optname %d\n", sk, optname);

        if (level != SOL_NFC)
                return -ENOPROTOOPT;

        if (get_user(len, optlen))
                return -EFAULT;

        local = llcp_sock->local;
        if (!local)
                return -ENODEV;

        len = min_t(u32, len, sizeof(u32));

        lock_sock(sk);

        switch (optname) {
        case NFC_LLCP_RW:
                rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw;
                if (put_user(rw, (u32 __user *) optval))
                        err = -EFAULT;

                break;

        case NFC_LLCP_MIUX:
                miux = be16_to_cpu(llcp_sock->miux) > LLCP_MAX_MIUX ?
                        be16_to_cpu(local->miux) : be16_to_cpu(llcp_sock->miux);

                if (put_user(miux, (u32 __user *) optval))
                        err = -EFAULT;

                break;

        case NFC_LLCP_REMOTE_MIU:
                remote_miu = llcp_sock->remote_miu > LLCP_MAX_MIU ?
                                local->remote_miu : llcp_sock->remote_miu;

                if (put_user(remote_miu, (u32 __user *) optval))
                        err = -EFAULT;

                break;

        case NFC_LLCP_REMOTE_LTO:
                if (put_user(local->remote_lto / 10, (u32 __user *) optval))
                        err = -EFAULT;

                break;

        case NFC_LLCP_REMOTE_RW:
                if (put_user(llcp_sock->remote_rw, (u32 __user *) optval))
                        err = -EFAULT;

                break;

        default:
                err = -ENOPROTOOPT;
                break;
        }

        release_sock(sk);

        if (put_user(len, optlen))
                return -EFAULT;

        return err;
}

void nfc_llcp_accept_unlink(struct sock *sk)
{
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);

        pr_debug("state %d\n", sk->sk_state);

        list_del_init(&llcp_sock->accept_queue);
        sk_acceptq_removed(llcp_sock->parent);
        llcp_sock->parent = NULL;

        sock_put(sk);
}

void nfc_llcp_accept_enqueue(struct sock *parent, struct sock *sk)
{
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        struct nfc_llcp_sock *llcp_sock_parent = nfc_llcp_sock(parent);

        /* Lock will be free from unlink */
        sock_hold(sk);

        list_add_tail(&llcp_sock->accept_queue,
                      &llcp_sock_parent->accept_queue);
        llcp_sock->parent = parent;
        sk_acceptq_added(parent);
}

struct sock *nfc_llcp_accept_dequeue(struct sock *parent,
                                     struct socket *newsock)
{
        struct nfc_llcp_sock *lsk, *n, *llcp_parent;
        struct sock *sk;

        llcp_parent = nfc_llcp_sock(parent);

        list_for_each_entry_safe(lsk, n, &llcp_parent->accept_queue,
                                 accept_queue) {
                sk = &lsk->sk;
                lock_sock(sk);

                if (sk->sk_state == LLCP_CLOSED) {
                        release_sock(sk);
                        nfc_llcp_accept_unlink(sk);
                        continue;
                }

                if (sk->sk_state == LLCP_CONNECTED || !newsock) {
                        list_del_init(&lsk->accept_queue);
                        sock_put(sk);

                        if (newsock)
                                sock_graft(sk, newsock);

                        release_sock(sk);

                        pr_debug("Returning sk state %d\n", sk->sk_state);

                        sk_acceptq_removed(parent);

                        return sk;
                }

                release_sock(sk);
        }

        return NULL;
}

static int llcp_sock_accept(struct socket *sock, struct socket *newsock,
                            struct proto_accept_arg *arg)
{
        DECLARE_WAITQUEUE(wait, current);
        struct sock *sk = sock->sk, *new_sk;
        long timeo;
        int ret = 0;

        pr_debug("parent %p\n", sk);

        lock_sock_nested(sk, SINGLE_DEPTH_NESTING);

        if (sk->sk_state != LLCP_LISTEN) {
                ret = -EBADFD;
                goto error;
        }

        timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK);

        /* Wait for an incoming connection. */
        add_wait_queue_exclusive(sk_sleep(sk), &wait);
        while (!(new_sk = nfc_llcp_accept_dequeue(sk, newsock))) {
                set_current_state(TASK_INTERRUPTIBLE);

                if (!timeo) {
                        ret = -EAGAIN;
                        break;
                }

                if (signal_pending(current)) {
                        ret = sock_intr_errno(timeo);
                        break;
                }

                release_sock(sk);
                timeo = schedule_timeout(timeo);
                lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
        }
        __set_current_state(TASK_RUNNING);
        remove_wait_queue(sk_sleep(sk), &wait);

        if (ret)
                goto error;

        newsock->state = SS_CONNECTED;

        pr_debug("new socket %p\n", new_sk);

error:
        release_sock(sk);

        return ret;
}

static int llcp_sock_getname(struct socket *sock, struct sockaddr *uaddr,
                             int peer)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, llcp_addr, uaddr);

        if (llcp_sock == NULL || llcp_sock->dev == NULL)
                return -EBADFD;

        pr_debug("%p %d %d %d\n", sk, llcp_sock->target_idx,
                 llcp_sock->dsap, llcp_sock->ssap);

        memset(llcp_addr, 0, sizeof(*llcp_addr));

        lock_sock(sk);
        if (!llcp_sock->dev) {
                release_sock(sk);
                return -EBADFD;
        }
        llcp_addr->sa_family = AF_NFC;
        llcp_addr->dev_idx = llcp_sock->dev->idx;
        llcp_addr->target_idx = llcp_sock->target_idx;
        llcp_addr->nfc_protocol = llcp_sock->nfc_protocol;
        llcp_addr->dsap = llcp_sock->dsap;
        llcp_addr->ssap = llcp_sock->ssap;
        llcp_addr->service_name_len = llcp_sock->service_name_len;
        memcpy(llcp_addr->service_name, llcp_sock->service_name,
               llcp_addr->service_name_len);
        release_sock(sk);

        return sizeof(struct sockaddr_nfc_llcp);
}

static inline __poll_t llcp_accept_poll(struct sock *parent)
{
        struct nfc_llcp_sock *llcp_sock, *parent_sock;
        struct sock *sk;

        parent_sock = nfc_llcp_sock(parent);

        list_for_each_entry(llcp_sock, &parent_sock->accept_queue,
                            accept_queue) {
                sk = &llcp_sock->sk;

                if (sk->sk_state == LLCP_CONNECTED)
                        return EPOLLIN | EPOLLRDNORM;
        }

        return 0;
}

static __poll_t llcp_sock_poll(struct file *file, struct socket *sock,
                                   poll_table *wait)
{
        struct sock *sk = sock->sk;
        __poll_t mask = 0;

        pr_debug("%p\n", sk);

        sock_poll_wait(file, sock, wait);

        if (sk->sk_state == LLCP_LISTEN)
                return llcp_accept_poll(sk);

        if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue))
                mask |= EPOLLERR |
                        (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? EPOLLPRI : 0);

        if (!skb_queue_empty_lockless(&sk->sk_receive_queue))
                mask |= EPOLLIN | EPOLLRDNORM;

        if (sk->sk_state == LLCP_CLOSED)
                mask |= EPOLLHUP;

        if (sk->sk_shutdown & RCV_SHUTDOWN)
                mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM;

        if (sk->sk_shutdown == SHUTDOWN_MASK)
                mask |= EPOLLHUP;

        if (sock_writeable(sk) && sk->sk_state == LLCP_CONNECTED)
                mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
        else
                sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);

        pr_debug("mask 0x%x\n", mask);

        return mask;
}

static int llcp_sock_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_local *local;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        int err = 0;

        if (!sk)
                return 0;

        pr_debug("%p\n", sk);

        local = llcp_sock->local;
        if (local == NULL) {
                err = -ENODEV;
                goto out;
        }

        lock_sock(sk);

        /* Send a DISC */
        if (sk->sk_state == LLCP_CONNECTED)
                nfc_llcp_send_disconnect(llcp_sock);

        if (sk->sk_state == LLCP_LISTEN) {
                struct nfc_llcp_sock *lsk, *n;
                struct sock *accept_sk;

                list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue,
                                         accept_queue) {
                        accept_sk = &lsk->sk;
                        lock_sock(accept_sk);

                        nfc_llcp_send_disconnect(lsk);
                        nfc_llcp_accept_unlink(accept_sk);

                        release_sock(accept_sk);
                }
        }

        if (sock->type == SOCK_RAW)
                nfc_llcp_sock_unlink(&local->raw_sockets, sk);
        else
                nfc_llcp_sock_unlink(&local->sockets, sk);

        if (llcp_sock->reserved_ssap < LLCP_SAP_MAX)
                nfc_llcp_put_ssap(llcp_sock->local, llcp_sock->ssap);

        release_sock(sk);

out:
        sock_orphan(sk);
        sock_put(sk);

        return err;
}

static int llcp_sock_connect(struct socket *sock, struct sockaddr *_addr,
                             int len, int flags)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        struct sockaddr_nfc_llcp *addr = (struct sockaddr_nfc_llcp *)_addr;
        struct nfc_dev *dev;
        struct nfc_llcp_local *local;
        int ret = 0;

        pr_debug("sock %p sk %p flags 0x%x\n", sock, sk, flags);

        if (!addr || len < sizeof(*addr) || addr->sa_family != AF_NFC)
                return -EINVAL;

        if (addr->service_name_len == 0 && addr->dsap == 0)
                return -EINVAL;

        pr_debug("addr dev_idx=%u target_idx=%u protocol=%u\n", addr->dev_idx,
                 addr->target_idx, addr->nfc_protocol);

        lock_sock(sk);

        if (sk->sk_state == LLCP_CONNECTED) {
                ret = -EISCONN;
                goto error;
        }
        if (sk->sk_state == LLCP_CONNECTING) {
                ret = -EINPROGRESS;
                goto error;
        }

        dev = nfc_get_device(addr->dev_idx);
        if (dev == NULL) {
                ret = -ENODEV;
                goto error;
        }

        local = nfc_llcp_find_local(dev);
        if (local == NULL) {
                ret = -ENODEV;
                goto put_dev;
        }

        device_lock(&dev->dev);
        if (dev->dep_link_up == false) {
                ret = -ENOLINK;
                device_unlock(&dev->dev);
                goto sock_llcp_put_local;
        }
        device_unlock(&dev->dev);

        if (local->rf_mode == NFC_RF_INITIATOR &&
            addr->target_idx != local->target_idx) {
                ret = -ENOLINK;
                goto sock_llcp_put_local;
        }

        llcp_sock->dev = dev;
        llcp_sock->local = local;
        llcp_sock->ssap = nfc_llcp_get_local_ssap(local);
        if (llcp_sock->ssap == LLCP_SAP_MAX) {
                ret = -ENOMEM;
                goto sock_llcp_nullify;
        }

        llcp_sock->reserved_ssap = llcp_sock->ssap;

        if (addr->service_name_len == 0)
                llcp_sock->dsap = addr->dsap;
        else
                llcp_sock->dsap = LLCP_SAP_SDP;
        llcp_sock->nfc_protocol = addr->nfc_protocol;
        llcp_sock->service_name_len = min_t(unsigned int,
                                            addr->service_name_len,
                                            NFC_LLCP_MAX_SERVICE_NAME);
        llcp_sock->service_name = kmemdup(addr->service_name,
                                          llcp_sock->service_name_len,
                                          GFP_KERNEL);
        if (!llcp_sock->service_name) {
                ret = -ENOMEM;
                goto sock_llcp_release;
        }

        nfc_llcp_sock_link(&local->connecting_sockets, sk);

        ret = nfc_llcp_send_connect(llcp_sock);
        if (ret)
                goto sock_unlink;

        sk->sk_state = LLCP_CONNECTING;

        ret = sock_wait_state(sk, LLCP_CONNECTED,
                              sock_sndtimeo(sk, flags & O_NONBLOCK));
        if (ret && ret != -EINPROGRESS)
                goto sock_unlink;

        release_sock(sk);

        return ret;

sock_unlink:
        nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
        kfree(llcp_sock->service_name);
        llcp_sock->service_name = NULL;

sock_llcp_release:
        nfc_llcp_put_ssap(local, llcp_sock->ssap);

sock_llcp_nullify:
        llcp_sock->local = NULL;
        llcp_sock->dev = NULL;

sock_llcp_put_local:
        nfc_llcp_local_put(local);

put_dev:
        nfc_put_device(dev);

error:
        release_sock(sk);
        return ret;
}

static int llcp_sock_sendmsg(struct socket *sock, struct msghdr *msg,
                             size_t len)
{
        struct sock *sk = sock->sk;
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);
        int ret;

        pr_debug("sock %p sk %p", sock, sk);

        ret = sock_error(sk);
        if (ret)
                return ret;

        if (msg->msg_flags & MSG_OOB)
                return -EOPNOTSUPP;

        lock_sock(sk);

        if (!llcp_sock->local) {
                release_sock(sk);
                return -ENODEV;
        }

        if (sk->sk_type == SOCK_DGRAM) {
                if (sk->sk_state != LLCP_BOUND) {
                        release_sock(sk);
                        return -ENOTCONN;
                }

                DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, addr,
                                 msg->msg_name);

                if (msg->msg_namelen < sizeof(*addr)) {
                        release_sock(sk);
                        return -EINVAL;
                }

                release_sock(sk);

                return nfc_llcp_send_ui_frame(llcp_sock, addr->dsap, addr->ssap,
                                              msg, len);
        }

        if (sk->sk_state != LLCP_CONNECTED) {
                release_sock(sk);
                return -ENOTCONN;
        }

        release_sock(sk);

        return nfc_llcp_send_i_frame(llcp_sock, msg, len);
}

static int llcp_sock_recvmsg(struct socket *sock, struct msghdr *msg,
                             size_t len, int flags)
{
        struct sock *sk = sock->sk;
        unsigned int copied, rlen;
        struct sk_buff *skb, *cskb;
        int err = 0;

        pr_debug("%p %zu\n", sk, len);

        lock_sock(sk);

        if (sk->sk_state == LLCP_CLOSED &&
            skb_queue_empty(&sk->sk_receive_queue)) {
                release_sock(sk);
                return 0;
        }

        release_sock(sk);

        if (flags & (MSG_OOB))
                return -EOPNOTSUPP;

        skb = skb_recv_datagram(sk, flags, &err);
        if (!skb) {
                pr_err("Recv datagram failed state %d %d %d",
                       sk->sk_state, err, sock_error(sk));

                if (sk->sk_shutdown & RCV_SHUTDOWN)
                        return 0;

                return err;
        }

        rlen = skb->len;                /* real length of skb */
        copied = min_t(unsigned int, rlen, len);

        cskb = skb;
        if (skb_copy_datagram_msg(cskb, 0, msg, copied)) {
                if (!(flags & MSG_PEEK))
                        skb_queue_head(&sk->sk_receive_queue, skb);
                return -EFAULT;
        }

        sock_recv_timestamp(msg, sk, skb);

        if (sk->sk_type == SOCK_DGRAM && msg->msg_name) {
                struct nfc_llcp_ui_cb *ui_cb = nfc_llcp_ui_skb_cb(skb);
                DECLARE_SOCKADDR(struct sockaddr_nfc_llcp *, sockaddr,
                                 msg->msg_name);

                msg->msg_namelen = sizeof(struct sockaddr_nfc_llcp);

                pr_debug("Datagram socket %d %d\n", ui_cb->dsap, ui_cb->ssap);

                memset(sockaddr, 0, sizeof(*sockaddr));
                sockaddr->sa_family = AF_NFC;
                sockaddr->nfc_protocol = NFC_PROTO_NFC_DEP;
                sockaddr->dsap = ui_cb->dsap;
                sockaddr->ssap = ui_cb->ssap;
        }

        /* Mark read part of skb as used */
        if (!(flags & MSG_PEEK)) {

                /* SOCK_STREAM: re-queue skb if it contains unreceived data */
                if (sk->sk_type == SOCK_STREAM ||
                    sk->sk_type == SOCK_DGRAM ||
                    sk->sk_type == SOCK_RAW) {
                        skb_pull(skb, copied);
                        if (skb->len) {
                                skb_queue_head(&sk->sk_receive_queue, skb);
                                goto done;
                        }
                }

                kfree_skb(skb);
        }

        /* XXX Queue backlogged skbs */

done:
        /* SOCK_SEQPACKET: return real length if MSG_TRUNC is set */
        if (sk->sk_type == SOCK_SEQPACKET && (flags & MSG_TRUNC))
                copied = rlen;

        return copied;
}

static const struct proto_ops llcp_sock_ops = {
        .family         = PF_NFC,
        .owner          = THIS_MODULE,
        .bind           = llcp_sock_bind,
        .connect        = llcp_sock_connect,
        .release        = llcp_sock_release,
        .socketpair     = sock_no_socketpair,
        .accept         = llcp_sock_accept,
        .getname        = llcp_sock_getname,
        .poll           = llcp_sock_poll,
        .ioctl          = sock_no_ioctl,
        .listen         = llcp_sock_listen,
        .shutdown       = sock_no_shutdown,
        .setsockopt     = nfc_llcp_setsockopt,
        .getsockopt     = nfc_llcp_getsockopt,
        .sendmsg        = llcp_sock_sendmsg,
        .recvmsg        = llcp_sock_recvmsg,
        .mmap           = sock_no_mmap,
};

static const struct proto_ops llcp_rawsock_ops = {
        .family         = PF_NFC,
        .owner          = THIS_MODULE,
        .bind           = llcp_raw_sock_bind,
        .connect        = sock_no_connect,
        .release        = llcp_sock_release,
        .socketpair     = sock_no_socketpair,
        .accept         = sock_no_accept,
        .getname        = llcp_sock_getname,
        .poll           = llcp_sock_poll,
        .ioctl          = sock_no_ioctl,
        .listen         = sock_no_listen,
        .shutdown       = sock_no_shutdown,
        .sendmsg        = sock_no_sendmsg,
        .recvmsg        = llcp_sock_recvmsg,
        .mmap           = sock_no_mmap,
};

static void llcp_sock_destruct(struct sock *sk)
{
        struct nfc_llcp_sock *llcp_sock = nfc_llcp_sock(sk);

        pr_debug("%p\n", sk);

        if (sk->sk_state == LLCP_CONNECTED)
                nfc_put_device(llcp_sock->dev);

        skb_queue_purge(&sk->sk_receive_queue);

        nfc_llcp_sock_free(llcp_sock);

        if (!sock_flag(sk, SOCK_DEAD)) {
                pr_err("Freeing alive NFC LLCP socket %p\n", sk);
                return;
        }
}

struct sock *nfc_llcp_sock_alloc(struct socket *sock, int type, gfp_t gfp, int kern)
{
        struct sock *sk;
        struct nfc_llcp_sock *llcp_sock;

        sk = sk_alloc(&init_net, PF_NFC, gfp, &llcp_sock_proto, kern);
        if (!sk)
                return NULL;

        llcp_sock = nfc_llcp_sock(sk);

        sock_init_data(sock, sk);
        sk->sk_state = LLCP_CLOSED;
        sk->sk_protocol = NFC_SOCKPROTO_LLCP;
        sk->sk_type = type;
        sk->sk_destruct = llcp_sock_destruct;

        llcp_sock->ssap = 0;
        llcp_sock->dsap = LLCP_SAP_SDP;
        llcp_sock->rw = LLCP_MAX_RW + 1;
        llcp_sock->miux = cpu_to_be16(LLCP_MAX_MIUX + 1);
        llcp_sock->send_n = llcp_sock->send_ack_n = 0;
        llcp_sock->recv_n = llcp_sock->recv_ack_n = 0;
        llcp_sock->remote_ready = 1;
        llcp_sock->reserved_ssap = LLCP_SAP_MAX;
        nfc_llcp_socket_remote_param_init(llcp_sock);
        skb_queue_head_init(&llcp_sock->tx_queue);
        skb_queue_head_init(&llcp_sock->tx_pending_queue);
        INIT_LIST_HEAD(&llcp_sock->accept_queue);

        if (sock != NULL)
                sock->state = SS_UNCONNECTED;

        return sk;
}

void nfc_llcp_sock_free(struct nfc_llcp_sock *sock)
{
        kfree(sock->service_name);

        skb_queue_purge(&sock->tx_queue);
        skb_queue_purge(&sock->tx_pending_queue);

        list_del_init(&sock->accept_queue);

        sock->parent = NULL;

        nfc_llcp_local_put(sock->local);
}

static int llcp_sock_create(struct net *net, struct socket *sock,
                            const struct nfc_protocol *nfc_proto, int kern)
{
        struct sock *sk;

        pr_debug("%p\n", sock);

        if (sock->type != SOCK_STREAM &&
            sock->type != SOCK_DGRAM &&
            sock->type != SOCK_RAW)
                return -ESOCKTNOSUPPORT;

        if (sock->type == SOCK_RAW) {
                if (!capable(CAP_NET_RAW))
                        return -EPERM;
                sock->ops = &llcp_rawsock_ops;
        } else {
                sock->ops = &llcp_sock_ops;
        }

        sk = nfc_llcp_sock_alloc(sock, sock->type, GFP_ATOMIC, kern);
        if (sk == NULL)
                return -ENOMEM;

        return 0;
}

static const struct nfc_protocol llcp_nfc_proto = {
        .id          = NFC_SOCKPROTO_LLCP,
        .proto    = &llcp_sock_proto,
        .owner    = THIS_MODULE,
        .create   = llcp_sock_create
};

int __init nfc_llcp_sock_init(void)
{
        return nfc_proto_register(&llcp_nfc_proto);
}

void nfc_llcp_sock_exit(void)
{
        nfc_proto_unregister(&llcp_nfc_proto);
}






















































    3 


    2 






    1 





    1 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2006 Patrick McHardy <kaber@trash.net>
 *
 * Based on ipt_random and ipt_nth by Fabrice MARIE <fabrice@netfilter.org>.
 */

#include <linux/init.h>
#include <linux/spinlock.h>
#include <linux/skbuff.h>
#include <linux/net.h>
#include <linux/slab.h>

#include <linux/netfilter/xt_statistic.h>
#include <linux/netfilter/x_tables.h>
#include <linux/module.h>

struct xt_statistic_priv {
        atomic_t count;
} ____cacheline_aligned_in_smp;

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
MODULE_DESCRIPTION("Xtables: statistics-based matching (\"Nth\", random)");
MODULE_ALIAS("ipt_statistic");
MODULE_ALIAS("ip6t_statistic");

static bool
statistic_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_statistic_info *info = par->matchinfo;
        bool ret = info->flags & XT_STATISTIC_INVERT;
        int nval, oval;

        switch (info->mode) {
        case XT_STATISTIC_MODE_RANDOM:
                if ((get_random_u32() & 0x7FFFFFFF) < info->u.random.probability)
                        ret = !ret;
                break;
        case XT_STATISTIC_MODE_NTH:
                do {
                        oval = atomic_read(&info->master->count);
                        nval = (oval == info->u.nth.every) ? 0 : oval + 1;
                } while (atomic_cmpxchg(&info->master->count, oval, nval) != oval);
                if (nval == 0)
                        ret = !ret;
                break;
        }

        return ret;
}

static int statistic_mt_check(const struct xt_mtchk_param *par)
{
        struct xt_statistic_info *info = par->matchinfo;

        if (info->mode > XT_STATISTIC_MODE_MAX ||
            info->flags & ~XT_STATISTIC_MASK)
                return -EINVAL;

        info->master = kzalloc(sizeof(*info->master), GFP_KERNEL);
        if (info->master == NULL)
                return -ENOMEM;
        atomic_set(&info->master->count, info->u.nth.count);

        return 0;
}

static void statistic_mt_destroy(const struct xt_mtdtor_param *par)
{
        const struct xt_statistic_info *info = par->matchinfo;

        kfree(info->master);
}

static struct xt_match xt_statistic_mt_reg __read_mostly = {
        .name       = "statistic",
        .revision   = 0,
        .family     = NFPROTO_UNSPEC,
        .match      = statistic_mt,
        .checkentry = statistic_mt_check,
        .destroy    = statistic_mt_destroy,
        .matchsize  = sizeof(struct xt_statistic_info),
        .usersize   = offsetof(struct xt_statistic_info, master),
        .me         = THIS_MODULE,
};

static int __init statistic_mt_init(void)
{
        return xt_register_match(&xt_statistic_mt_reg);
}

static void __exit statistic_mt_exit(void)
{
        xt_unregister_match(&xt_statistic_mt_reg);
}

module_init(statistic_mt_init);
module_exit(statistic_mt_exit);







































   84 









    3 
   81 
   82 

















   37 




   39 






























































































































































































































































































































































































































































































































































    5 














    5 












































































    5 





    5 































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * ip_vs_proto_tcp.c:        TCP load balancing support for IPVS
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *              Julian Anastasov <ja@ssi.bg>
 *
 * Changes:     Hans Schillstrom <hans.schillstrom@ericsson.com>
 *
 *              Network name space (netns) aware.
 *              Global data moved to netns i.e struct netns_ipvs
 *              tcp_timeouts table has copy per netns in a hash table per
 *              protocol ip_vs_proto_data and is handled by netns
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/kernel.h>
#include <linux/ip.h>
#include <linux/tcp.h>                  /* for tcphdr */
#include <net/ip.h>
#include <net/tcp.h>                    /* for csum_tcpudp_magic */
#include <net/ip6_checksum.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <linux/indirect_call_wrapper.h>

#include <net/ip_vs.h>

static int
tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp);

static int
tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb,
                  struct ip_vs_proto_data *pd,
                  int *verdict, struct ip_vs_conn **cpp,
                  struct ip_vs_iphdr *iph)
{
        struct ip_vs_service *svc;
        struct tcphdr _tcph, *th;
        __be16 _ports[2], *ports = NULL;

        /* In the event of icmp, we're only guaranteed to have the first 8
         * bytes of the transport header, so we only check the rest of the
         * TCP packet for non-ICMP packets
         */
        if (likely(!ip_vs_iph_icmp(iph))) {
                th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph);
                if (th) {
                        if (th->rst || !(sysctl_sloppy_tcp(ipvs) || th->syn))
                                return 1;
                        ports = &th->source;
                }
        } else {
                ports = skb_header_pointer(
                        skb, iph->len, sizeof(_ports), &_ports);
        }

        if (!ports) {
                *verdict = NF_DROP;
                return 0;
        }

        /* No !th->ack check to allow scheduling on SYN+ACK for Active FTP */

        if (likely(!ip_vs_iph_inverse(iph)))
                svc = ip_vs_service_find(ipvs, af, skb->mark, iph->protocol,
                                         &iph->daddr, ports[1]);
        else
                svc = ip_vs_service_find(ipvs, af, skb->mark, iph->protocol,
                                         &iph->saddr, ports[0]);

        if (svc) {
                int ignored;

                if (ip_vs_todrop(ipvs)) {
                        /*
                         * It seems that we are very loaded.
                         * We have to drop this packet :(
                         */
                        *verdict = NF_DROP;
                        return 0;
                }

                /*
                 * Let the virtual server select a real server for the
                 * incoming connection, and create a connection entry.
                 */
                *cpp = ip_vs_schedule(svc, skb, pd, &ignored, iph);
                if (!*cpp && ignored <= 0) {
                        if (!ignored)
                                *verdict = ip_vs_leave(svc, skb, pd, iph);
                        else
                                *verdict = NF_DROP;
                        return 0;
                }
        }
        /* NF_ACCEPT */
        return 1;
}


static inline void
tcp_fast_csum_update(int af, struct tcphdr *tcph,
                     const union nf_inet_addr *oldip,
                     const union nf_inet_addr *newip,
                     __be16 oldport, __be16 newport)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                tcph->check =
                        csum_fold(ip_vs_check_diff16(oldip->ip6, newip->ip6,
                                         ip_vs_check_diff2(oldport, newport,
                                                ~csum_unfold(tcph->check))));
        else
#endif
        tcph->check =
                csum_fold(ip_vs_check_diff4(oldip->ip, newip->ip,
                                 ip_vs_check_diff2(oldport, newport,
                                                ~csum_unfold(tcph->check))));
}


static inline void
tcp_partial_csum_update(int af, struct tcphdr *tcph,
                     const union nf_inet_addr *oldip,
                     const union nf_inet_addr *newip,
                     __be16 oldlen, __be16 newlen)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                tcph->check =
                        ~csum_fold(ip_vs_check_diff16(oldip->ip6, newip->ip6,
                                         ip_vs_check_diff2(oldlen, newlen,
                                                csum_unfold(tcph->check))));
        else
#endif
        tcph->check =
                ~csum_fold(ip_vs_check_diff4(oldip->ip, newip->ip,
                                ip_vs_check_diff2(oldlen, newlen,
                                                csum_unfold(tcph->check))));
}


INDIRECT_CALLABLE_SCOPE int
tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)
{
        struct tcphdr *tcph;
        unsigned int tcphoff = iph->len;
        bool payload_csum = false;
        int oldlen;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6 && iph->fragoffs)
                return 1;
#endif
        oldlen = skb->len - tcphoff;

        /* csum_check requires unshared skb */
        if (skb_ensure_writable(skb, tcphoff + sizeof(*tcph)))
                return 0;

        if (unlikely(cp->app != NULL)) {
                int ret;

                /* Some checks before mangling */
                if (!tcp_csum_check(cp->af, skb, pp))
                        return 0;

                /* Call application helper if needed */
                if (!(ret = ip_vs_app_pkt_out(cp, skb, iph)))
                        return 0;
                /* ret=2: csum update is needed after payload mangling */
                if (ret == 1)
                        oldlen = skb->len - tcphoff;
                else
                        payload_csum = true;
        }

        tcph = (void *)skb_network_header(skb) + tcphoff;
        tcph->source = cp->vport;

        /* Adjust TCP checksums */
        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                tcp_partial_csum_update(cp->af, tcph, &cp->daddr, &cp->vaddr,
                                        htons(oldlen),
                                        htons(skb->len - tcphoff));
        } else if (!payload_csum) {
                /* Only port and addr are changed, do fast csum update */
                tcp_fast_csum_update(cp->af, tcph, &cp->daddr, &cp->vaddr,
                                     cp->dport, cp->vport);
                if (skb->ip_summed == CHECKSUM_COMPLETE)
                        skb->ip_summed = cp->app ?
                                         CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
        } else {
                /* full checksum calculation */
                tcph->check = 0;
                skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0);
#ifdef CONFIG_IP_VS_IPV6
                if (cp->af == AF_INET6)
                        tcph->check = csum_ipv6_magic(&cp->vaddr.in6,
                                                      &cp->caddr.in6,
                                                      skb->len - tcphoff,
                                                      cp->protocol, skb->csum);
                else
#endif
                        tcph->check = csum_tcpudp_magic(cp->vaddr.ip,
                                                        cp->caddr.ip,
                                                        skb->len - tcphoff,
                                                        cp->protocol,
                                                        skb->csum);
                skb->ip_summed = CHECKSUM_UNNECESSARY;

                IP_VS_DBG(11, "O-pkt: %s O-csum=%d (+%zd)\n",
                          pp->name, tcph->check,
                          (char*)&(tcph->check) - (char*)tcph);
        }
        return 1;
}


static int
tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp,
                 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)
{
        struct tcphdr *tcph;
        unsigned int tcphoff = iph->len;
        bool payload_csum = false;
        int oldlen;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6 && iph->fragoffs)
                return 1;
#endif
        oldlen = skb->len - tcphoff;

        /* csum_check requires unshared skb */
        if (skb_ensure_writable(skb, tcphoff + sizeof(*tcph)))
                return 0;

        if (unlikely(cp->app != NULL)) {
                int ret;

                /* Some checks before mangling */
                if (!tcp_csum_check(cp->af, skb, pp))
                        return 0;

                /*
                 *        Attempt ip_vs_app call.
                 *        It will fix ip_vs_conn and iph ack_seq stuff
                 */
                if (!(ret = ip_vs_app_pkt_in(cp, skb, iph)))
                        return 0;
                /* ret=2: csum update is needed after payload mangling */
                if (ret == 1)
                        oldlen = skb->len - tcphoff;
                else
                        payload_csum = true;
        }

        tcph = (void *)skb_network_header(skb) + tcphoff;
        tcph->dest = cp->dport;

        /*
         *        Adjust TCP checksums
         */
        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                tcp_partial_csum_update(cp->af, tcph, &cp->vaddr, &cp->daddr,
                                        htons(oldlen),
                                        htons(skb->len - tcphoff));
        } else if (!payload_csum) {
                /* Only port and addr are changed, do fast csum update */
                tcp_fast_csum_update(cp->af, tcph, &cp->vaddr, &cp->daddr,
                                     cp->vport, cp->dport);
                if (skb->ip_summed == CHECKSUM_COMPLETE)
                        skb->ip_summed = cp->app ?
                                         CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
        } else {
                /* full checksum calculation */
                tcph->check = 0;
                skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0);
#ifdef CONFIG_IP_VS_IPV6
                if (cp->af == AF_INET6)
                        tcph->check = csum_ipv6_magic(&cp->caddr.in6,
                                                      &cp->daddr.in6,
                                                      skb->len - tcphoff,
                                                      cp->protocol, skb->csum);
                else
#endif
                        tcph->check = csum_tcpudp_magic(cp->caddr.ip,
                                                        cp->daddr.ip,
                                                        skb->len - tcphoff,
                                                        cp->protocol,
                                                        skb->csum);
                skb->ip_summed = CHECKSUM_UNNECESSARY;
        }
        return 1;
}


static int
tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp)
{
        unsigned int tcphoff;

#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                tcphoff = sizeof(struct ipv6hdr);
        else
#endif
                tcphoff = ip_hdrlen(skb);

        switch (skb->ip_summed) {
        case CHECKSUM_NONE:
                skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0);
                fallthrough;
        case CHECKSUM_COMPLETE:
#ifdef CONFIG_IP_VS_IPV6
                if (af == AF_INET6) {
                        if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
                                            &ipv6_hdr(skb)->daddr,
                                            skb->len - tcphoff,
                                            ipv6_hdr(skb)->nexthdr,
                                            skb->csum)) {
                                IP_VS_DBG_RL_PKT(0, af, pp, skb, 0,
                                                 "Failed checksum for");
                                return 0;
                        }
                } else
#endif
                        if (csum_tcpudp_magic(ip_hdr(skb)->saddr,
                                              ip_hdr(skb)->daddr,
                                              skb->len - tcphoff,
                                              ip_hdr(skb)->protocol,
                                              skb->csum)) {
                                IP_VS_DBG_RL_PKT(0, af, pp, skb, 0,
                                                 "Failed checksum for");
                                return 0;
                        }
                break;
        default:
                /* No need to checksum. */
                break;
        }

        return 1;
}


#define TCP_DIR_INPUT                0
#define TCP_DIR_OUTPUT                4
#define TCP_DIR_INPUT_ONLY        8

static const int tcp_state_off[IP_VS_DIR_LAST] = {
        [IP_VS_DIR_INPUT]                =        TCP_DIR_INPUT,
        [IP_VS_DIR_OUTPUT]                =        TCP_DIR_OUTPUT,
        [IP_VS_DIR_INPUT_ONLY]                =        TCP_DIR_INPUT_ONLY,
};

/*
 *        Timeout table[state]
 */
static const int tcp_timeouts[IP_VS_TCP_S_LAST+1] = {
        [IP_VS_TCP_S_NONE]                =        2*HZ,
        [IP_VS_TCP_S_ESTABLISHED]        =        15*60*HZ,
        [IP_VS_TCP_S_SYN_SENT]                =        2*60*HZ,
        [IP_VS_TCP_S_SYN_RECV]                =        1*60*HZ,
        [IP_VS_TCP_S_FIN_WAIT]                =        2*60*HZ,
        [IP_VS_TCP_S_TIME_WAIT]                =        2*60*HZ,
        [IP_VS_TCP_S_CLOSE]                =        10*HZ,
        [IP_VS_TCP_S_CLOSE_WAIT]        =        60*HZ,
        [IP_VS_TCP_S_LAST_ACK]                =        30*HZ,
        [IP_VS_TCP_S_LISTEN]                =        2*60*HZ,
        [IP_VS_TCP_S_SYNACK]                =        120*HZ,
        [IP_VS_TCP_S_LAST]                =        2*HZ,
};

static const char *const tcp_state_name_table[IP_VS_TCP_S_LAST+1] = {
        [IP_VS_TCP_S_NONE]                =        "NONE",
        [IP_VS_TCP_S_ESTABLISHED]        =        "ESTABLISHED",
        [IP_VS_TCP_S_SYN_SENT]                =        "SYN_SENT",
        [IP_VS_TCP_S_SYN_RECV]                =        "SYN_RECV",
        [IP_VS_TCP_S_FIN_WAIT]                =        "FIN_WAIT",
        [IP_VS_TCP_S_TIME_WAIT]                =        "TIME_WAIT",
        [IP_VS_TCP_S_CLOSE]                =        "CLOSE",
        [IP_VS_TCP_S_CLOSE_WAIT]        =        "CLOSE_WAIT",
        [IP_VS_TCP_S_LAST_ACK]                =        "LAST_ACK",
        [IP_VS_TCP_S_LISTEN]                =        "LISTEN",
        [IP_VS_TCP_S_SYNACK]                =        "SYNACK",
        [IP_VS_TCP_S_LAST]                =        "BUG!",
};

static const bool tcp_state_active_table[IP_VS_TCP_S_LAST] = {
        [IP_VS_TCP_S_NONE]                =        false,
        [IP_VS_TCP_S_ESTABLISHED]        =        true,
        [IP_VS_TCP_S_SYN_SENT]                =        true,
        [IP_VS_TCP_S_SYN_RECV]                =        true,
        [IP_VS_TCP_S_FIN_WAIT]                =        false,
        [IP_VS_TCP_S_TIME_WAIT]                =        false,
        [IP_VS_TCP_S_CLOSE]                =        false,
        [IP_VS_TCP_S_CLOSE_WAIT]        =        false,
        [IP_VS_TCP_S_LAST_ACK]                =        false,
        [IP_VS_TCP_S_LISTEN]                =        false,
        [IP_VS_TCP_S_SYNACK]                =        true,
};

#define sNO IP_VS_TCP_S_NONE
#define sES IP_VS_TCP_S_ESTABLISHED
#define sSS IP_VS_TCP_S_SYN_SENT
#define sSR IP_VS_TCP_S_SYN_RECV
#define sFW IP_VS_TCP_S_FIN_WAIT
#define sTW IP_VS_TCP_S_TIME_WAIT
#define sCL IP_VS_TCP_S_CLOSE
#define sCW IP_VS_TCP_S_CLOSE_WAIT
#define sLA IP_VS_TCP_S_LAST_ACK
#define sLI IP_VS_TCP_S_LISTEN
#define sSA IP_VS_TCP_S_SYNACK

struct tcp_states_t {
        int next_state[IP_VS_TCP_S_LAST];
};

static const char * tcp_state_name(int state)
{
        if (state >= IP_VS_TCP_S_LAST)
                return "ERR!";
        return tcp_state_name_table[state] ? tcp_state_name_table[state] : "?";
}

static bool tcp_state_active(int state)
{
        if (state >= IP_VS_TCP_S_LAST)
                return false;
        return tcp_state_active_table[state];
}

static struct tcp_states_t tcp_states[] = {
/*        INPUT */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSR, sES, sES, sSR, sSR, sSR, sSR, sSR, sSR, sSR, sSR }},
/*fin*/ {{sCL, sCW, sSS, sTW, sTW, sTW, sCL, sCW, sLA, sLI, sTW }},
/*ack*/ {{sES, sES, sSS, sES, sFW, sTW, sCL, sCW, sCL, sLI, sES }},
/*rst*/ {{sCL, sCL, sCL, sSR, sCL, sCL, sCL, sCL, sLA, sLI, sSR }},

/*        OUTPUT */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSS, sES, sSS, sSR, sSS, sSS, sSS, sSS, sSS, sLI, sSR }},
/*fin*/ {{sTW, sFW, sSS, sTW, sFW, sTW, sCL, sTW, sLA, sLI, sTW }},
/*ack*/ {{sES, sES, sSS, sES, sFW, sTW, sCL, sCW, sLA, sES, sES }},
/*rst*/ {{sCL, sCL, sSS, sCL, sCL, sTW, sCL, sCL, sCL, sCL, sCL }},

/*        INPUT-ONLY */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSR, sES, sES, sSR, sSR, sSR, sSR, sSR, sSR, sSR, sSR }},
/*fin*/ {{sCL, sFW, sSS, sTW, sFW, sTW, sCL, sCW, sLA, sLI, sTW }},
/*ack*/ {{sES, sES, sSS, sES, sFW, sTW, sCL, sCW, sCL, sLI, sES }},
/*rst*/ {{sCL, sCL, sCL, sSR, sCL, sCL, sCL, sCL, sLA, sLI, sCL }},
};

static struct tcp_states_t tcp_states_dos[] = {
/*        INPUT */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSR, sES, sES, sSR, sSR, sSR, sSR, sSR, sSR, sSR, sSA }},
/*fin*/ {{sCL, sCW, sSS, sTW, sTW, sTW, sCL, sCW, sLA, sLI, sSA }},
/*ack*/ {{sES, sES, sSS, sSR, sFW, sTW, sCL, sCW, sCL, sLI, sSA }},
/*rst*/ {{sCL, sCL, sCL, sSR, sCL, sCL, sCL, sCL, sLA, sLI, sCL }},

/*        OUTPUT */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSS, sES, sSS, sSA, sSS, sSS, sSS, sSS, sSS, sLI, sSA }},
/*fin*/ {{sTW, sFW, sSS, sTW, sFW, sTW, sCL, sTW, sLA, sLI, sTW }},
/*ack*/ {{sES, sES, sSS, sES, sFW, sTW, sCL, sCW, sLA, sES, sES }},
/*rst*/ {{sCL, sCL, sSS, sCL, sCL, sTW, sCL, sCL, sCL, sCL, sCL }},

/*        INPUT-ONLY */
/*        sNO, sES, sSS, sSR, sFW, sTW, sCL, sCW, sLA, sLI, sSA        */
/*syn*/ {{sSA, sES, sES, sSR, sSA, sSA, sSA, sSA, sSA, sSA, sSA }},
/*fin*/ {{sCL, sFW, sSS, sTW, sFW, sTW, sCL, sCW, sLA, sLI, sTW }},
/*ack*/ {{sES, sES, sSS, sES, sFW, sTW, sCL, sCW, sCL, sLI, sES }},
/*rst*/ {{sCL, sCL, sCL, sSR, sCL, sCL, sCL, sCL, sLA, sLI, sCL }},
};

static void tcp_timeout_change(struct ip_vs_proto_data *pd, int flags)
{
        int on = (flags & 1);                /* secure_tcp */

        /*
        ** FIXME: change secure_tcp to independent sysctl var
        ** or make it per-service or per-app because it is valid
        ** for most if not for all of the applications. Something
        ** like "capabilities" (flags) for each object.
        */
        pd->tcp_state_table = (on ? tcp_states_dos : tcp_states);
}

static inline int tcp_state_idx(struct tcphdr *th)
{
        if (th->rst)
                return 3;
        if (th->syn)
                return 0;
        if (th->fin)
                return 1;
        if (th->ack)
                return 2;
        return -1;
}

static inline void
set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp,
              int direction, struct tcphdr *th)
{
        int state_idx;
        int new_state = IP_VS_TCP_S_CLOSE;
        int state_off = tcp_state_off[direction];

        /*
         *    Update state offset to INPUT_ONLY if necessary
         *    or delete NO_OUTPUT flag if output packet detected
         */
        if (cp->flags & IP_VS_CONN_F_NOOUTPUT) {
                if (state_off == TCP_DIR_OUTPUT)
                        cp->flags &= ~IP_VS_CONN_F_NOOUTPUT;
                else
                        state_off = TCP_DIR_INPUT_ONLY;
        }

        if ((state_idx = tcp_state_idx(th)) < 0) {
                IP_VS_DBG(8, "tcp_state_idx=%d!!!\n", state_idx);
                goto tcp_state_out;
        }

        new_state =
                pd->tcp_state_table[state_off+state_idx].next_state[cp->state];

  tcp_state_out:
        if (new_state != cp->state) {
                struct ip_vs_dest *dest = cp->dest;

                IP_VS_DBG_BUF(8, "%s %s [%c%c%c%c] c:%s:%d v:%s:%d "
                              "d:%s:%d state: %s->%s conn->refcnt:%d\n",
                              pd->pp->name,
                              ((state_off == TCP_DIR_OUTPUT) ?
                               "output " : "input "),
                              th->syn ? 'S' : '.',
                              th->fin ? 'F' : '.',
                              th->ack ? 'A' : '.',
                              th->rst ? 'R' : '.',
                              IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                              ntohs(cp->cport),
                              IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                              ntohs(cp->vport),
                              IP_VS_DBG_ADDR(cp->daf, &cp->daddr),
                              ntohs(cp->dport),
                              tcp_state_name(cp->state),
                              tcp_state_name(new_state),
                              refcount_read(&cp->refcnt));

                if (dest) {
                        if (!(cp->flags & IP_VS_CONN_F_INACTIVE) &&
                            !tcp_state_active(new_state)) {
                                atomic_dec(&dest->activeconns);
                                atomic_inc(&dest->inactconns);
                                cp->flags |= IP_VS_CONN_F_INACTIVE;
                        } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) &&
                                   tcp_state_active(new_state)) {
                                atomic_inc(&dest->activeconns);
                                atomic_dec(&dest->inactconns);
                                cp->flags &= ~IP_VS_CONN_F_INACTIVE;
                        }
                }
                if (new_state == IP_VS_TCP_S_ESTABLISHED)
                        ip_vs_control_assure_ct(cp);
        }

        if (likely(pd))
                cp->timeout = pd->timeout_table[cp->state = new_state];
        else        /* What to do ? */
                cp->timeout = tcp_timeouts[cp->state = new_state];
}

/*
 *        Handle state transitions
 */
static void
tcp_state_transition(struct ip_vs_conn *cp, int direction,
                     const struct sk_buff *skb,
                     struct ip_vs_proto_data *pd)
{
        struct tcphdr _tcph, *th;

#ifdef CONFIG_IP_VS_IPV6
        int ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr);
#else
        int ihl = ip_hdrlen(skb);
#endif

        th = skb_header_pointer(skb, ihl, sizeof(_tcph), &_tcph);
        if (th == NULL)
                return;

        spin_lock_bh(&cp->lock);
        set_tcp_state(pd, cp, direction, th);
        spin_unlock_bh(&cp->lock);
}

static inline __u16 tcp_app_hashkey(__be16 port)
{
        return (((__force u16)port >> TCP_APP_TAB_BITS) ^ (__force u16)port)
                & TCP_APP_TAB_MASK;
}


static int tcp_register_app(struct netns_ipvs *ipvs, struct ip_vs_app *inc)
{
        struct ip_vs_app *i;
        __u16 hash;
        __be16 port = inc->port;
        int ret = 0;
        struct ip_vs_proto_data *pd = ip_vs_proto_data_get(ipvs, IPPROTO_TCP);

        hash = tcp_app_hashkey(port);

        list_for_each_entry(i, &ipvs->tcp_apps[hash], p_list) {
                if (i->port == port) {
                        ret = -EEXIST;
                        goto out;
                }
        }
        list_add_rcu(&inc->p_list, &ipvs->tcp_apps[hash]);
        atomic_inc(&pd->appcnt);

  out:
        return ret;
}


static void
tcp_unregister_app(struct netns_ipvs *ipvs, struct ip_vs_app *inc)
{
        struct ip_vs_proto_data *pd = ip_vs_proto_data_get(ipvs, IPPROTO_TCP);

        atomic_dec(&pd->appcnt);
        list_del_rcu(&inc->p_list);
}


static int
tcp_app_conn_bind(struct ip_vs_conn *cp)
{
        struct netns_ipvs *ipvs = cp->ipvs;
        int hash;
        struct ip_vs_app *inc;
        int result = 0;

        /* Default binding: bind app only for NAT */
        if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)
                return 0;

        /* Lookup application incarnations and bind the right one */
        hash = tcp_app_hashkey(cp->vport);

        list_for_each_entry_rcu(inc, &ipvs->tcp_apps[hash], p_list) {
                if (inc->port == cp->vport) {
                        if (unlikely(!ip_vs_app_inc_get(inc)))
                                break;

                        IP_VS_DBG_BUF(9, "%s(): Binding conn %s:%u->"
                                      "%s:%u to app %s on port %u\n",
                                      __func__,
                                      IP_VS_DBG_ADDR(cp->af, &cp->caddr),
                                      ntohs(cp->cport),
                                      IP_VS_DBG_ADDR(cp->af, &cp->vaddr),
                                      ntohs(cp->vport),
                                      inc->name, ntohs(inc->port));

                        cp->app = inc;
                        if (inc->init_conn)
                                result = inc->init_conn(inc, cp);
                        break;
                }
        }

        return result;
}


/*
 *        Set LISTEN timeout. (ip_vs_conn_put will setup timer)
 */
void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp)
{
        struct ip_vs_proto_data *pd = ip_vs_proto_data_get(cp->ipvs, IPPROTO_TCP);

        spin_lock_bh(&cp->lock);
        cp->state = IP_VS_TCP_S_LISTEN;
        cp->timeout = (pd ? pd->timeout_table[IP_VS_TCP_S_LISTEN]
                           : tcp_timeouts[IP_VS_TCP_S_LISTEN]);
        spin_unlock_bh(&cp->lock);
}

/* ---------------------------------------------
 *   timeouts is netns related now.
 * ---------------------------------------------
 */
static int __ip_vs_tcp_init(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd)
{
        ip_vs_init_hash_table(ipvs->tcp_apps, TCP_APP_TAB_SIZE);
        pd->timeout_table = ip_vs_create_timeout_table((int *)tcp_timeouts,
                                                        sizeof(tcp_timeouts));
        if (!pd->timeout_table)
                return -ENOMEM;
        pd->tcp_state_table = tcp_states;
        return 0;
}

static void __ip_vs_tcp_exit(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd)
{
        kfree(pd->timeout_table);
}


struct ip_vs_protocol ip_vs_protocol_tcp = {
        .name =                        "TCP",
        .protocol =                IPPROTO_TCP,
        .num_states =                IP_VS_TCP_S_LAST,
        .dont_defrag =                0,
        .init =                        NULL,
        .exit =                        NULL,
        .init_netns =                __ip_vs_tcp_init,
        .exit_netns =                __ip_vs_tcp_exit,
        .register_app =                tcp_register_app,
        .unregister_app =        tcp_unregister_app,
        .conn_schedule =        tcp_conn_schedule,
        .conn_in_get =                ip_vs_conn_in_get_proto,
        .conn_out_get =                ip_vs_conn_out_get_proto,
        .snat_handler =                tcp_snat_handler,
        .dnat_handler =                tcp_dnat_handler,
        .state_name =                tcp_state_name,
        .state_transition =        tcp_state_transition,
        .app_conn_bind =        tcp_app_conn_bind,
        .debug_packet =                ip_vs_tcpudp_debug_packet,
        .timeout_change =        tcp_timeout_change,
};





































































































    1 









    1 






















    1 











    3 
   25 
















   25 
    2 



















    1 
   12 













   13 

















   25 





















































































































































































































































































































































   15 














   13 


    3 









   15 
    2 










   15 









   15 













   15 

    3 















   27 







   27 

























    2 


    2 











    2 






































   27 










    1 

    1 
    1 









































    2 









    2 





    2 







    2 








    2 








    2 











   13 

   19 

















    1 



    1 















  164 

   76 



   88 
   20 


   87 






   71 




  147 





   27 



  164 





   69 








   22 
   46 


  163 

    1 












   70 
































































































   15 










































































  110 


































  200 




  205 












  201 

  204 

  205 


  203 

























  205 





  202 

   25 






























































































  201 
  146 






  144 







  146 







   61 
   34 



   70 





  164 









    2 



    2 










   25 




   28 









































   28 




   28 
















   28 


























   25 









   64 








    1 
  174 








   14 


















    1 




    1 


    1 





    1 



    1 










  147 
  147 











  148 




   14 
   14 
    1 
   14 



   71 
   23 
   71 



  147 


















































   27 



    2 










   28 







    2 


















   15 







    2 



   27 








    1 








    1 




















   15 
    2 





   15 




    5 









    1 






















   29 













   28 







  109 


  110 






   24 
   24 











  201 
    1 














   42 

  202 


  203 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
// SPDX-License-Identifier: GPL-2.0-or-later
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 *
 * This file is part of the SCTP kernel implementation
 *
 * These functions work with the state functions in sctp_sm_statefuns.c
 * to implement that state operations.  These functions implement the
 * steps which require modifying existing data structures.
 *
 * Please send any bug reports or fixes you make to the
 * email address(es):
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson          <karl@athena.chicago.il.us>
 *    Jon Grimm             <jgrimm@austin.ibm.com>
 *    Hui Huang                    <hui.huang@nokia.com>
 *    Dajiang Zhang            <dajiang.zhang@nokia.com>
 *    Daisy Chang            <daisyc@us.ibm.com>
 *    Sridhar Samudrala            <sri@us.ibm.com>
 *    Ardelle Fan            <ardelle.fan@intel.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/skbuff.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/ip.h>
#include <linux/gfp.h>
#include <net/sock.h>
#include <net/sctp/sctp.h>
#include <net/sctp/sm.h>
#include <net/sctp/stream_sched.h>

static int sctp_cmd_interpreter(enum sctp_event_type event_type,
                                union sctp_subtype subtype,
                                enum sctp_state state,
                                struct sctp_endpoint *ep,
                                struct sctp_association *asoc,
                                void *event_arg,
                                enum sctp_disposition status,
                                struct sctp_cmd_seq *commands,
                                gfp_t gfp);
static int sctp_side_effects(enum sctp_event_type event_type,
                             union sctp_subtype subtype,
                             enum sctp_state state,
                             struct sctp_endpoint *ep,
                             struct sctp_association **asoc,
                             void *event_arg,
                             enum sctp_disposition status,
                             struct sctp_cmd_seq *commands,
                             gfp_t gfp);

/********************************************************************
 * Helper functions
 ********************************************************************/

/* A helper function for delayed processing of INET ECN CE bit. */
static void sctp_do_ecn_ce_work(struct sctp_association *asoc,
                                __u32 lowest_tsn)
{
        /* Save the TSN away for comparison when we receive CWR */

        asoc->last_ecne_tsn = lowest_tsn;
        asoc->need_ecne = 1;
}

/* Helper function for delayed processing of SCTP ECNE chunk.  */
/* RFC 2960 Appendix A
 *
 * RFC 2481 details a specific bit for a sender to send in
 * the header of its next outbound TCP segment to indicate to
 * its peer that it has reduced its congestion window.  This
 * is termed the CWR bit.  For SCTP the same indication is made
 * by including the CWR chunk.  This chunk contains one data
 * element, i.e. the TSN number that was sent in the ECNE chunk.
 * This element represents the lowest TSN number in the datagram
 * that was originally marked with the CE bit.
 */
static struct sctp_chunk *sctp_do_ecn_ecne_work(struct sctp_association *asoc,
                                                __u32 lowest_tsn,
                                                struct sctp_chunk *chunk)
{
        struct sctp_chunk *repl;

        /* Our previously transmitted packet ran into some congestion
         * so we should take action by reducing cwnd and ssthresh
         * and then ACK our peer that we we've done so by
         * sending a CWR.
         */

        /* First, try to determine if we want to actually lower
         * our cwnd variables.  Only lower them if the ECNE looks more
         * recent than the last response.
         */
        if (TSN_lt(asoc->last_cwr_tsn, lowest_tsn)) {
                struct sctp_transport *transport;

                /* Find which transport's congestion variables
                 * need to be adjusted.
                 */
                transport = sctp_assoc_lookup_tsn(asoc, lowest_tsn);

                /* Update the congestion variables. */
                if (transport)
                        sctp_transport_lower_cwnd(transport,
                                                  SCTP_LOWER_CWND_ECNE);
                asoc->last_cwr_tsn = lowest_tsn;
        }

        /* Always try to quiet the other end.  In case of lost CWR,
         * resend last_cwr_tsn.
         */
        repl = sctp_make_cwr(asoc, asoc->last_cwr_tsn, chunk);

        /* If we run out of memory, it will look like a lost CWR.  We'll
         * get back in sync eventually.
         */
        return repl;
}

/* Helper function to do delayed processing of ECN CWR chunk.  */
static void sctp_do_ecn_cwr_work(struct sctp_association *asoc,
                                 __u32 lowest_tsn)
{
        /* Turn off ECNE getting auto-prepended to every outgoing
         * packet
         */
        asoc->need_ecne = 0;
}

/* Generate SACK if necessary.  We call this at the end of a packet.  */
static int sctp_gen_sack(struct sctp_association *asoc, int force,
                         struct sctp_cmd_seq *commands)
{
        struct sctp_transport *trans = asoc->peer.last_data_from;
        __u32 ctsn, max_tsn_seen;
        struct sctp_chunk *sack;
        int error = 0;

        if (force ||
            (!trans && (asoc->param_flags & SPP_SACKDELAY_DISABLE)) ||
            (trans && (trans->param_flags & SPP_SACKDELAY_DISABLE)))
                asoc->peer.sack_needed = 1;

        ctsn = sctp_tsnmap_get_ctsn(&asoc->peer.tsn_map);
        max_tsn_seen = sctp_tsnmap_get_max_tsn_seen(&asoc->peer.tsn_map);

        /* From 12.2 Parameters necessary per association (i.e. the TCB):
         *
         * Ack State : This flag indicates if the next received packet
         *              : is to be responded to with a SACK. ...
         *             : When DATA chunks are out of order, SACK's
         *           : are not delayed (see Section 6).
         *
         * [This is actually not mentioned in Section 6, but we
         * implement it here anyway. --piggy]
         */
        if (max_tsn_seen != ctsn)
                asoc->peer.sack_needed = 1;

        /* From 6.2  Acknowledgement on Reception of DATA Chunks:
         *
         * Section 4.2 of [RFC2581] SHOULD be followed. Specifically,
         * an acknowledgement SHOULD be generated for at least every
         * second packet (not every second DATA chunk) received, and
         * SHOULD be generated within 200 ms of the arrival of any
         * unacknowledged DATA chunk. ...
         */
        if (!asoc->peer.sack_needed) {
                asoc->peer.sack_cnt++;

                /* Set the SACK delay timeout based on the
                 * SACK delay for the last transport
                 * data was received from, or the default
                 * for the association.
                 */
                if (trans) {
                        /* We will need a SACK for the next packet.  */
                        if (asoc->peer.sack_cnt >= trans->sackfreq - 1)
                                asoc->peer.sack_needed = 1;

                        asoc->timeouts[SCTP_EVENT_TIMEOUT_SACK] =
                                trans->sackdelay;
                } else {
                        /* We will need a SACK for the next packet.  */
                        if (asoc->peer.sack_cnt >= asoc->sackfreq - 1)
                                asoc->peer.sack_needed = 1;

                        asoc->timeouts[SCTP_EVENT_TIMEOUT_SACK] =
                                asoc->sackdelay;
                }

                /* Restart the SACK timer. */
                sctp_add_cmd_sf(commands, SCTP_CMD_TIMER_RESTART,
                                SCTP_TO(SCTP_EVENT_TIMEOUT_SACK));
        } else {
                __u32 old_a_rwnd = asoc->a_rwnd;

                asoc->a_rwnd = asoc->rwnd;
                sack = sctp_make_sack(asoc);
                if (!sack) {
                        asoc->a_rwnd = old_a_rwnd;
                        goto nomem;
                }

                asoc->peer.sack_needed = 0;
                asoc->peer.sack_cnt = 0;

                sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(sack));

                /* Stop the SACK timer.  */
                sctp_add_cmd_sf(commands, SCTP_CMD_TIMER_STOP,
                                SCTP_TO(SCTP_EVENT_TIMEOUT_SACK));
        }

        return error;
nomem:
        error = -ENOMEM;
        return error;
}

/* When the T3-RTX timer expires, it calls this function to create the
 * relevant state machine event.
 */
void sctp_generate_t3_rtx_event(struct timer_list *t)
{
        struct sctp_transport *transport =
                timer_container_of(transport, t, T3_rtx_timer);
        struct sctp_association *asoc = transport->asoc;
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);
        int error;

        /* Check whether a task is in the sock.  */

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy\n", __func__);

                /* Try again later.  */
                if (!mod_timer(&transport->T3_rtx_timer, jiffies + (HZ/20)))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        /* Run through the state machine.  */
        error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
                           SCTP_ST_TIMEOUT(SCTP_EVENT_TIMEOUT_T3_RTX),
                           asoc->state,
                           asoc->ep, asoc,
                           transport, GFP_ATOMIC);

        if (error)
                sk->sk_err = -error;

out_unlock:
        bh_unlock_sock(sk);
        sctp_transport_put(transport);
}

/* This is a sa interface for producing timeout events.  It works
 * for timeouts which use the association as their parameter.
 */
static void sctp_generate_timeout_event(struct sctp_association *asoc,
                                        enum sctp_event_timeout timeout_type)
{
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);
        int error = 0;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy: timer %d\n", __func__,
                         timeout_type);

                /* Try again later.  */
                if (!mod_timer(&asoc->timers[timeout_type], jiffies + (HZ/20)))
                        sctp_association_hold(asoc);
                goto out_unlock;
        }

        /* Is this association really dead and just waiting around for
         * the timer to let go of the reference?
         */
        if (asoc->base.dead)
                goto out_unlock;

        /* Run through the state machine.  */
        error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
                           SCTP_ST_TIMEOUT(timeout_type),
                           asoc->state, asoc->ep, asoc,
                           (void *)timeout_type, GFP_ATOMIC);

        if (error)
                sk->sk_err = -error;

out_unlock:
        bh_unlock_sock(sk);
        sctp_association_put(asoc);
}

static void sctp_generate_t1_cookie_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t,
                                   timers[SCTP_EVENT_TIMEOUT_T1_COOKIE]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_T1_COOKIE);
}

static void sctp_generate_t1_init_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t,
                                   timers[SCTP_EVENT_TIMEOUT_T1_INIT]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_T1_INIT);
}

static void sctp_generate_t2_shutdown_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t,
                                   timers[SCTP_EVENT_TIMEOUT_T2_SHUTDOWN]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_T2_SHUTDOWN);
}

static void sctp_generate_t4_rto_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t, timers[SCTP_EVENT_TIMEOUT_T4_RTO]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_T4_RTO);
}

static void sctp_generate_t5_shutdown_guard_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t,
                                   timers[SCTP_EVENT_TIMEOUT_T5_SHUTDOWN_GUARD]);

        sctp_generate_timeout_event(asoc,
                                    SCTP_EVENT_TIMEOUT_T5_SHUTDOWN_GUARD);

} /* sctp_generate_t5_shutdown_guard_event() */

static void sctp_generate_autoclose_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t,
                                   timers[SCTP_EVENT_TIMEOUT_AUTOCLOSE]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_AUTOCLOSE);
}

/* Generate a heart beat event.  If the sock is busy, reschedule.   Make
 * sure that the transport is still valid.
 */
void sctp_generate_heartbeat_event(struct timer_list *t)
{
        struct sctp_transport *transport = timer_container_of(transport, t,
                                                              hb_timer);
        struct sctp_association *asoc = transport->asoc;
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);
        u32 elapsed, timeout;
        int error = 0;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy\n", __func__);

                /* Try again later.  */
                if (!mod_timer(&transport->hb_timer, jiffies + (HZ/20)))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        /* Check if we should still send the heartbeat or reschedule */
        elapsed = jiffies - transport->last_time_sent;
        timeout = sctp_transport_timeout(transport);
        if (elapsed < timeout) {
                elapsed = timeout - elapsed;
                if (!mod_timer(&transport->hb_timer, jiffies + elapsed))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
                           SCTP_ST_TIMEOUT(SCTP_EVENT_TIMEOUT_HEARTBEAT),
                           asoc->state, asoc->ep, asoc,
                           transport, GFP_ATOMIC);

        if (error)
                sk->sk_err = -error;

out_unlock:
        bh_unlock_sock(sk);
        sctp_transport_put(transport);
}

/* Handle the timeout of the ICMP protocol unreachable timer.  Trigger
 * the correct state machine transition that will close the association.
 */
void sctp_generate_proto_unreach_event(struct timer_list *t)
{
        struct sctp_transport *transport =
                timer_container_of(transport, t, proto_unreach_timer);
        struct sctp_association *asoc = transport->asoc;
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy\n", __func__);

                /* Try again later.  */
                if (!mod_timer(&transport->proto_unreach_timer,
                                jiffies + (HZ/20)))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        /* Is this structure just waiting around for us to actually
         * get destroyed?
         */
        if (asoc->base.dead)
                goto out_unlock;

        sctp_do_sm(net, SCTP_EVENT_T_OTHER,
                   SCTP_ST_OTHER(SCTP_EVENT_ICMP_PROTO_UNREACH),
                   asoc->state, asoc->ep, asoc, transport, GFP_ATOMIC);

out_unlock:
        bh_unlock_sock(sk);
        sctp_transport_put(transport);
}

 /* Handle the timeout of the RE-CONFIG timer. */
void sctp_generate_reconf_event(struct timer_list *t)
{
        struct sctp_transport *transport =
                timer_container_of(transport, t, reconf_timer);
        struct sctp_association *asoc = transport->asoc;
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);
        int error = 0;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy\n", __func__);

                /* Try again later.  */
                if (!mod_timer(&transport->reconf_timer, jiffies + (HZ / 20)))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        /* This happens when the response arrives after the timer is triggered. */
        if (!asoc->strreset_chunk)
                goto out_unlock;

        error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
                           SCTP_ST_TIMEOUT(SCTP_EVENT_TIMEOUT_RECONF),
                           asoc->state, asoc->ep, asoc,
                           transport, GFP_ATOMIC);

        if (error)
                sk->sk_err = -error;

out_unlock:
        bh_unlock_sock(sk);
        sctp_transport_put(transport);
}

/* Handle the timeout of the probe timer. */
void sctp_generate_probe_event(struct timer_list *t)
{
        struct sctp_transport *transport = timer_container_of(transport, t,
                                                              probe_timer);
        struct sctp_association *asoc = transport->asoc;
        struct sock *sk = asoc->base.sk;
        struct net *net = sock_net(sk);
        int error = 0;

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk)) {
                pr_debug("%s: sock is busy\n", __func__);

                /* Try again later.  */
                if (!mod_timer(&transport->probe_timer, jiffies + (HZ / 20)))
                        sctp_transport_hold(transport);
                goto out_unlock;
        }

        error = sctp_do_sm(net, SCTP_EVENT_T_TIMEOUT,
                           SCTP_ST_TIMEOUT(SCTP_EVENT_TIMEOUT_PROBE),
                           asoc->state, asoc->ep, asoc,
                           transport, GFP_ATOMIC);

        if (error)
                sk->sk_err = -error;

out_unlock:
        bh_unlock_sock(sk);
        sctp_transport_put(transport);
}

/* Inject a SACK Timeout event into the state machine.  */
static void sctp_generate_sack_event(struct timer_list *t)
{
        struct sctp_association *asoc =
                timer_container_of(asoc, t, timers[SCTP_EVENT_TIMEOUT_SACK]);

        sctp_generate_timeout_event(asoc, SCTP_EVENT_TIMEOUT_SACK);
}

sctp_timer_event_t *sctp_timer_events[SCTP_NUM_TIMEOUT_TYPES] = {
        [SCTP_EVENT_TIMEOUT_NONE] =                NULL,
        [SCTP_EVENT_TIMEOUT_T1_COOKIE] =        sctp_generate_t1_cookie_event,
        [SCTP_EVENT_TIMEOUT_T1_INIT] =                sctp_generate_t1_init_event,
        [SCTP_EVENT_TIMEOUT_T2_SHUTDOWN] =        sctp_generate_t2_shutdown_event,
        [SCTP_EVENT_TIMEOUT_T3_RTX] =                NULL,
        [SCTP_EVENT_TIMEOUT_T4_RTO] =                sctp_generate_t4_rto_event,
        [SCTP_EVENT_TIMEOUT_T5_SHUTDOWN_GUARD] =
                                        sctp_generate_t5_shutdown_guard_event,
        [SCTP_EVENT_TIMEOUT_HEARTBEAT] =        NULL,
        [SCTP_EVENT_TIMEOUT_RECONF] =                NULL,
        [SCTP_EVENT_TIMEOUT_SACK] =                sctp_generate_sack_event,
        [SCTP_EVENT_TIMEOUT_AUTOCLOSE] =        sctp_generate_autoclose_event,
};


/* RFC 2960 8.2 Path Failure Detection
 *
 * When its peer endpoint is multi-homed, an endpoint should keep a
 * error counter for each of the destination transport addresses of the
 * peer endpoint.
 *
 * Each time the T3-rtx timer expires on any address, or when a
 * HEARTBEAT sent to an idle address is not acknowledged within a RTO,
 * the error counter of that destination address will be incremented.
 * When the value in the error counter exceeds the protocol parameter
 * 'Path.Max.Retrans' of that destination address, the endpoint should
 * mark the destination transport address as inactive, and a
 * notification SHOULD be sent to the upper layer.
 *
 */
static void sctp_do_8_2_transport_strike(struct sctp_cmd_seq *commands,
                                         struct sctp_association *asoc,
                                         struct sctp_transport *transport,
                                         int is_hb)
{
        /* The check for association's overall error counter exceeding the
         * threshold is done in the state function.
         */
        /* We are here due to a timer expiration.  If the timer was
         * not a HEARTBEAT, then normal error tracking is done.
         * If the timer was a heartbeat, we only increment error counts
         * when we already have an outstanding HEARTBEAT that has not
         * been acknowledged.
         * Additionally, some tranport states inhibit error increments.
         */
        if (!is_hb) {
                asoc->overall_error_count++;
                if (transport->state != SCTP_INACTIVE)
                        transport->error_count++;
         } else if (transport->hb_sent) {
                if (transport->state != SCTP_UNCONFIRMED)
                        asoc->overall_error_count++;
                if (transport->state != SCTP_INACTIVE)
                        transport->error_count++;
        }

        /* If the transport error count is greater than the pf_retrans
         * threshold, and less than pathmaxrtx, and if the current state
         * is SCTP_ACTIVE, then mark this transport as Partially Failed,
         * see SCTP Quick Failover Draft, section 5.1
         */
        if (asoc->base.net->sctp.pf_enable &&
            transport->state == SCTP_ACTIVE &&
            transport->error_count < transport->pathmaxrxt &&
            transport->error_count > transport->pf_retrans) {

                sctp_assoc_control_transport(asoc, transport,
                                             SCTP_TRANSPORT_PF,
                                             0);

                /* Update the hb timer to resend a heartbeat every rto */
                sctp_transport_reset_hb_timer(transport);
        }

        if (transport->state != SCTP_INACTIVE &&
            (transport->error_count > transport->pathmaxrxt)) {
                pr_debug("%s: association:%p transport addr:%pISpc failed\n",
                         __func__, asoc, &transport->ipaddr.sa);

                sctp_assoc_control_transport(asoc, transport,
                                             SCTP_TRANSPORT_DOWN,
                                             SCTP_FAILED_THRESHOLD);
        }

        if (transport->error_count > transport->ps_retrans &&
            asoc->peer.primary_path == transport &&
            asoc->peer.active_path != transport)
                sctp_assoc_set_primary(asoc, asoc->peer.active_path);

        /* E2) For the destination address for which the timer
         * expires, set RTO <- RTO * 2 ("back off the timer").  The
         * maximum value discussed in rule C7 above (RTO.max) may be
         * used to provide an upper bound to this doubling operation.
         *
         * Special Case:  the first HB doesn't trigger exponential backoff.
         * The first unacknowledged HB triggers it.  We do this with a flag
         * that indicates that we have an outstanding HB.
         */
        if (!is_hb || transport->hb_sent) {
                transport->rto = min((transport->rto * 2), transport->asoc->rto_max);
                sctp_max_rto(asoc, transport);
        }
}

/* Worker routine to handle INIT command failure.  */
static void sctp_cmd_init_failed(struct sctp_cmd_seq *commands,
                                 struct sctp_association *asoc,
                                 unsigned int error)
{
        struct sctp_ulpevent *event;

        event = sctp_ulpevent_make_assoc_change(asoc, 0, SCTP_CANT_STR_ASSOC,
                                                (__u16)error, 0, 0, NULL,
                                                GFP_ATOMIC);

        if (event)
                sctp_add_cmd_sf(commands, SCTP_CMD_EVENT_ULP,
                                SCTP_ULPEVENT(event));

        sctp_add_cmd_sf(commands, SCTP_CMD_NEW_STATE,
                        SCTP_STATE(SCTP_STATE_CLOSED));

        /* SEND_FAILED sent later when cleaning up the association. */
        asoc->outqueue.error = error;
        sctp_add_cmd_sf(commands, SCTP_CMD_DELETE_TCB, SCTP_NULL());
}

/* Worker routine to handle SCTP_CMD_ASSOC_FAILED.  */
static void sctp_cmd_assoc_failed(struct sctp_cmd_seq *commands,
                                  struct sctp_association *asoc,
                                  enum sctp_event_type event_type,
                                  union sctp_subtype subtype,
                                  struct sctp_chunk *chunk,
                                  unsigned int error)
{
        struct sctp_ulpevent *event;
        struct sctp_chunk *abort;

        /* Cancel any partial delivery in progress. */
        asoc->stream.si->abort_pd(&asoc->ulpq, GFP_ATOMIC);

        if (event_type == SCTP_EVENT_T_CHUNK && subtype.chunk == SCTP_CID_ABORT)
                event = sctp_ulpevent_make_assoc_change(asoc, 0, SCTP_COMM_LOST,
                                                (__u16)error, 0, 0, chunk,
                                                GFP_ATOMIC);
        else
                event = sctp_ulpevent_make_assoc_change(asoc, 0, SCTP_COMM_LOST,
                                                (__u16)error, 0, 0, NULL,
                                                GFP_ATOMIC);
        if (event)
                sctp_add_cmd_sf(commands, SCTP_CMD_EVENT_ULP,
                                SCTP_ULPEVENT(event));

        if (asoc->overall_error_count >= asoc->max_retrans) {
                abort = sctp_make_violation_max_retrans(asoc, chunk);
                if (abort)
                        sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                        SCTP_CHUNK(abort));
        }

        sctp_add_cmd_sf(commands, SCTP_CMD_NEW_STATE,
                        SCTP_STATE(SCTP_STATE_CLOSED));

        /* SEND_FAILED sent later when cleaning up the association. */
        asoc->outqueue.error = error;
        sctp_add_cmd_sf(commands, SCTP_CMD_DELETE_TCB, SCTP_NULL());
}

/* Process an init chunk (may be real INIT/INIT-ACK or an embedded INIT
 * inside the cookie.  In reality, this is only used for INIT-ACK processing
 * since all other cases use "temporary" associations and can do all
 * their work in statefuns directly.
 */
static int sctp_cmd_process_init(struct sctp_cmd_seq *commands,
                                 struct sctp_association *asoc,
                                 struct sctp_chunk *chunk,
                                 struct sctp_init_chunk *peer_init,
                                 gfp_t gfp)
{
        int error;

        /* We only process the init as a sideeffect in a single
         * case.   This is when we process the INIT-ACK.   If we
         * fail during INIT processing (due to malloc problems),
         * just return the error and stop processing the stack.
         */
        if (!sctp_process_init(asoc, chunk, sctp_source(chunk), peer_init, gfp))
                error = -ENOMEM;
        else
                error = 0;

        return error;
}

/* Helper function to break out starting up of heartbeat timers.  */
static void sctp_cmd_hb_timers_start(struct sctp_cmd_seq *cmds,
                                     struct sctp_association *asoc)
{
        struct sctp_transport *t;

        /* Start a heartbeat timer for each transport on the association.
         * hold a reference on the transport to make sure none of
         * the needed data structures go away.
         */
        list_for_each_entry(t, &asoc->peer.transport_addr_list, transports)
                sctp_transport_reset_hb_timer(t);
}

static void sctp_cmd_hb_timers_stop(struct sctp_cmd_seq *cmds,
                                    struct sctp_association *asoc)
{
        struct sctp_transport *t;

        /* Stop all heartbeat timers. */

        list_for_each_entry(t, &asoc->peer.transport_addr_list,
                        transports) {
                if (timer_delete(&t->hb_timer))
                        sctp_transport_put(t);
        }
}

/* Helper function to stop any pending T3-RTX timers */
static void sctp_cmd_t3_rtx_timers_stop(struct sctp_cmd_seq *cmds,
                                        struct sctp_association *asoc)
{
        struct sctp_transport *t;

        list_for_each_entry(t, &asoc->peer.transport_addr_list,
                        transports) {
                if (timer_delete(&t->T3_rtx_timer))
                        sctp_transport_put(t);
        }
}


/* Helper function to handle the reception of an HEARTBEAT ACK.  */
static void sctp_cmd_transport_on(struct sctp_cmd_seq *cmds,
                                  struct sctp_association *asoc,
                                  struct sctp_transport *t,
                                  struct sctp_chunk *chunk)
{
        struct sctp_sender_hb_info *hbinfo;
        int was_unconfirmed = 0;

        /* 8.3 Upon the receipt of the HEARTBEAT ACK, the sender of the
         * HEARTBEAT should clear the error counter of the destination
         * transport address to which the HEARTBEAT was sent.
         */
        t->error_count = 0;

        /*
         * Although RFC4960 specifies that the overall error count must
         * be cleared when a HEARTBEAT ACK is received, we make an
         * exception while in SHUTDOWN PENDING. If the peer keeps its
         * window shut forever, we may never be able to transmit our
         * outstanding data and rely on the retransmission limit be reached
         * to shutdown the association.
         */
        if (t->asoc->state < SCTP_STATE_SHUTDOWN_PENDING)
                t->asoc->overall_error_count = 0;

        /* Clear the hb_sent flag to signal that we had a good
         * acknowledgement.
         */
        t->hb_sent = 0;

        /* Mark the destination transport address as active if it is not so
         * marked.
         */
        if ((t->state == SCTP_INACTIVE) || (t->state == SCTP_UNCONFIRMED)) {
                was_unconfirmed = 1;
                sctp_assoc_control_transport(asoc, t, SCTP_TRANSPORT_UP,
                                             SCTP_HEARTBEAT_SUCCESS);
        }

        if (t->state == SCTP_PF)
                sctp_assoc_control_transport(asoc, t, SCTP_TRANSPORT_UP,
                                             SCTP_HEARTBEAT_SUCCESS);

        /* HB-ACK was received for a the proper HB.  Consider this
         * forward progress.
         */
        if (t->dst)
                sctp_transport_dst_confirm(t);

        /* The receiver of the HEARTBEAT ACK should also perform an
         * RTT measurement for that destination transport address
         * using the time value carried in the HEARTBEAT ACK chunk.
         * If the transport's rto_pending variable has been cleared,
         * it was most likely due to a retransmit.  However, we want
         * to re-enable it to properly update the rto.
         */
        if (t->rto_pending == 0)
                t->rto_pending = 1;

        hbinfo = (struct sctp_sender_hb_info *)chunk->skb->data;
        sctp_transport_update_rto(t, (jiffies - hbinfo->sent_at));

        /* Update the heartbeat timer.  */
        sctp_transport_reset_hb_timer(t);

        if (was_unconfirmed && asoc->peer.transport_count == 1)
                sctp_transport_immediate_rtx(t);
}


/* Helper function to process the process SACK command.  */
static int sctp_cmd_process_sack(struct sctp_cmd_seq *cmds,
                                 struct sctp_association *asoc,
                                 struct sctp_chunk *chunk)
{
        int err = 0;

        if (sctp_outq_sack(&asoc->outqueue, chunk)) {
                /* There are no more TSNs awaiting SACK.  */
                err = sctp_do_sm(asoc->base.net, SCTP_EVENT_T_OTHER,
                                 SCTP_ST_OTHER(SCTP_EVENT_NO_PENDING_TSN),
                                 asoc->state, asoc->ep, asoc, NULL,
                                 GFP_ATOMIC);
        }

        return err;
}

/* Helper function to set the timeout value for T2-SHUTDOWN timer and to set
 * the transport for a shutdown chunk.
 */
static void sctp_cmd_setup_t2(struct sctp_cmd_seq *cmds,
                              struct sctp_association *asoc,
                              struct sctp_chunk *chunk)
{
        struct sctp_transport *t;

        if (chunk->transport)
                t = chunk->transport;
        else {
                t = sctp_assoc_choose_alter_transport(asoc,
                                              asoc->shutdown_last_sent_to);
                chunk->transport = t;
        }
        asoc->shutdown_last_sent_to = t;
        asoc->timeouts[SCTP_EVENT_TIMEOUT_T2_SHUTDOWN] = t->rto;
}

/* Helper function to change the state of an association. */
static void sctp_cmd_new_state(struct sctp_cmd_seq *cmds,
                               struct sctp_association *asoc,
                               enum sctp_state state)
{
        struct sock *sk = asoc->base.sk;

        asoc->state = state;

        pr_debug("%s: asoc:%p[%s]\n", __func__, asoc, sctp_state_tbl[state]);

        if (sctp_style(sk, TCP)) {
                /* Change the sk->sk_state of a TCP-style socket that has
                 * successfully completed a connect() call.
                 */
                if (sctp_state(asoc, ESTABLISHED) && sctp_sstate(sk, CLOSED))
                        inet_sk_set_state(sk, SCTP_SS_ESTABLISHED);

                /* Set the RCV_SHUTDOWN flag when a SHUTDOWN is received. */
                if (sctp_state(asoc, SHUTDOWN_RECEIVED) &&
                    sctp_sstate(sk, ESTABLISHED)) {
                        inet_sk_set_state(sk, SCTP_SS_CLOSING);
                        sk->sk_shutdown |= RCV_SHUTDOWN;
                }
        }

        if (sctp_state(asoc, COOKIE_WAIT)) {
                /* Reset init timeouts since they may have been
                 * increased due to timer expirations.
                 */
                asoc->timeouts[SCTP_EVENT_TIMEOUT_T1_INIT] =
                                                asoc->rto_initial;
                asoc->timeouts[SCTP_EVENT_TIMEOUT_T1_COOKIE] =
                                                asoc->rto_initial;
        }

        if (sctp_state(asoc, ESTABLISHED)) {
                kfree(asoc->peer.cookie);
                asoc->peer.cookie = NULL;
        }

        if (sctp_state(asoc, ESTABLISHED) ||
            sctp_state(asoc, CLOSED) ||
            sctp_state(asoc, SHUTDOWN_RECEIVED)) {
                /* Wake up any processes waiting in the asoc's wait queue in
                 * sctp_wait_for_connect() or sctp_wait_for_sndbuf().
                 */
                if (waitqueue_active(&asoc->wait))
                        wake_up_interruptible(&asoc->wait);

                /* Wake up any processes waiting in the sk's sleep queue of
                 * a TCP-style or UDP-style peeled-off socket in
                 * sctp_wait_for_accept() or sctp_wait_for_packet().
                 * For a UDP-style socket, the waiters are woken up by the
                 * notifications.
                 */
                if (!sctp_style(sk, UDP))
                        sk->sk_state_change(sk);
        }

        if (sctp_state(asoc, SHUTDOWN_PENDING) &&
            !sctp_outq_is_empty(&asoc->outqueue))
                sctp_outq_uncork(&asoc->outqueue, GFP_ATOMIC);
}

/* Helper function to delete an association. */
static void sctp_cmd_delete_tcb(struct sctp_cmd_seq *cmds,
                                struct sctp_association *asoc)
{
        struct sock *sk = asoc->base.sk;

        /* If it is a non-temporary association belonging to a TCP-style
         * listening socket that is not closed, do not free it so that accept()
         * can pick it up later.
         */
        if (sctp_style(sk, TCP) && sctp_sstate(sk, LISTENING) &&
            (!asoc->temp) && (sk->sk_shutdown != SHUTDOWN_MASK))
                return;

        sctp_association_free(asoc);
}

/*
 * ADDIP Section 4.1 ASCONF Chunk Procedures
 * A4) Start a T-4 RTO timer, using the RTO value of the selected
 * destination address (we use active path instead of primary path just
 * because primary path may be inactive.
 */
static void sctp_cmd_setup_t4(struct sctp_cmd_seq *cmds,
                              struct sctp_association *asoc,
                              struct sctp_chunk *chunk)
{
        struct sctp_transport *t;

        t = sctp_assoc_choose_alter_transport(asoc, chunk->transport);
        asoc->timeouts[SCTP_EVENT_TIMEOUT_T4_RTO] = t->rto;
        chunk->transport = t;
}

/* Process an incoming Operation Error Chunk. */
static void sctp_cmd_process_operr(struct sctp_cmd_seq *cmds,
                                   struct sctp_association *asoc,
                                   struct sctp_chunk *chunk)
{
        struct sctp_errhdr *err_hdr;
        struct sctp_ulpevent *ev;

        while (chunk->chunk_end > chunk->skb->data) {
                err_hdr = (struct sctp_errhdr *)(chunk->skb->data);

                ev = sctp_ulpevent_make_remote_error(asoc, chunk, 0,
                                                     GFP_ATOMIC);
                if (!ev)
                        return;

                asoc->stream.si->enqueue_event(&asoc->ulpq, ev);

                switch (err_hdr->cause) {
                case SCTP_ERROR_UNKNOWN_CHUNK:
                {
                        struct sctp_chunkhdr *unk_chunk_hdr;

                        unk_chunk_hdr = (struct sctp_chunkhdr *)(err_hdr + 1);
                        switch (unk_chunk_hdr->type) {
                        /* ADDIP 4.1 A9) If the peer responds to an ASCONF with
                         * an ERROR chunk reporting that it did not recognized
                         * the ASCONF chunk type, the sender of the ASCONF MUST
                         * NOT send any further ASCONF chunks and MUST stop its
                         * T-4 timer.
                         */
                        case SCTP_CID_ASCONF:
                                if (asoc->peer.asconf_capable == 0)
                                        break;

                                asoc->peer.asconf_capable = 0;
                                sctp_add_cmd_sf(cmds, SCTP_CMD_TIMER_STOP,
                                        SCTP_TO(SCTP_EVENT_TIMEOUT_T4_RTO));
                                break;
                        default:
                                break;
                        }
                        break;
                }
                default:
                        break;
                }
        }
}

/* Helper function to remove the association non-primary peer
 * transports.
 */
static void sctp_cmd_del_non_primary(struct sctp_association *asoc)
{
        struct sctp_transport *t;
        struct list_head *temp;
        struct list_head *pos;

        list_for_each_safe(pos, temp, &asoc->peer.transport_addr_list) {
                t = list_entry(pos, struct sctp_transport, transports);
                if (!sctp_cmp_addr_exact(&t->ipaddr,
                                         &asoc->peer.primary_addr)) {
                        sctp_assoc_rm_peer(asoc, t);
                }
        }
}

/* Helper function to set sk_err on a 1-1 style socket. */
static void sctp_cmd_set_sk_err(struct sctp_association *asoc, int error)
{
        struct sock *sk = asoc->base.sk;

        if (!sctp_style(sk, UDP))
                sk->sk_err = error;
}

/* Helper function to generate an association change event */
static void sctp_cmd_assoc_change(struct sctp_cmd_seq *commands,
                                  struct sctp_association *asoc,
                                  u8 state)
{
        struct sctp_ulpevent *ev;

        ev = sctp_ulpevent_make_assoc_change(asoc, 0, state, 0,
                                            asoc->c.sinit_num_ostreams,
                                            asoc->c.sinit_max_instreams,
                                            NULL, GFP_ATOMIC);
        if (ev)
                asoc->stream.si->enqueue_event(&asoc->ulpq, ev);
}

static void sctp_cmd_peer_no_auth(struct sctp_cmd_seq *commands,
                                  struct sctp_association *asoc)
{
        struct sctp_ulpevent *ev;

        ev = sctp_ulpevent_make_authkey(asoc, 0, SCTP_AUTH_NO_AUTH, GFP_ATOMIC);
        if (ev)
                asoc->stream.si->enqueue_event(&asoc->ulpq, ev);
}

/* Helper function to generate an adaptation indication event */
static void sctp_cmd_adaptation_ind(struct sctp_cmd_seq *commands,
                                    struct sctp_association *asoc)
{
        struct sctp_ulpevent *ev;

        ev = sctp_ulpevent_make_adaptation_indication(asoc, GFP_ATOMIC);

        if (ev)
                asoc->stream.si->enqueue_event(&asoc->ulpq, ev);
}


static void sctp_cmd_t1_timer_update(struct sctp_association *asoc,
                                     enum sctp_event_timeout timer,
                                     char *name)
{
        struct sctp_transport *t;

        t = asoc->init_last_sent_to;
        asoc->init_err_counter++;

        if (t->init_sent_count > (asoc->init_cycle + 1)) {
                asoc->timeouts[timer] *= 2;
                if (asoc->timeouts[timer] > asoc->max_init_timeo) {
                        asoc->timeouts[timer] = asoc->max_init_timeo;
                }
                asoc->init_cycle++;

                pr_debug("%s: T1[%s] timeout adjustment init_err_counter:%d"
                         " cycle:%d timeout:%ld\n", __func__, name,
                         asoc->init_err_counter, asoc->init_cycle,
                         asoc->timeouts[timer]);
        }

}

/* Send the whole message, chunk by chunk, to the outqueue.
 * This way the whole message is queued up and bundling if
 * encouraged for small fragments.
 */
static void sctp_cmd_send_msg(struct sctp_association *asoc,
                              struct sctp_datamsg *msg, gfp_t gfp)
{
        struct sctp_chunk *chunk;

        list_for_each_entry(chunk, &msg->chunks, frag_list)
                sctp_outq_tail(&asoc->outqueue, chunk, gfp);

        asoc->outqueue.sched->enqueue(&asoc->outqueue, msg);
}


/* These three macros allow us to pull the debugging code out of the
 * main flow of sctp_do_sm() to keep attention focused on the real
 * functionality there.
 */
#define debug_pre_sfn() \
        pr_debug("%s[pre-fn]: ep:%p, %s, %s, asoc:%p[%s], %s\n", __func__, \
                 ep, sctp_evttype_tbl[event_type], (*debug_fn)(subtype),   \
                 asoc, sctp_state_tbl[state], state_fn->name)

#define debug_post_sfn() \
        pr_debug("%s[post-fn]: asoc:%p, status:%s\n", __func__, asoc, \
                 sctp_status_tbl[status])

#define debug_post_sfx() \
        pr_debug("%s[post-sfx]: error:%d, asoc:%p[%s]\n", __func__, error, \
                 asoc, sctp_state_tbl[(asoc && sctp_id2assoc(ep->base.sk, \
                 sctp_assoc2id(asoc))) ? asoc->state : SCTP_STATE_CLOSED])

/*
 * This is the master state machine processing function.
 *
 * If you want to understand all of lksctp, this is a
 * good place to start.
 */
int sctp_do_sm(struct net *net, enum sctp_event_type event_type,
               union sctp_subtype subtype, enum sctp_state state,
               struct sctp_endpoint *ep, struct sctp_association *asoc,
               void *event_arg, gfp_t gfp)
{
        typedef const char *(printfn_t)(union sctp_subtype);
        static printfn_t *table[] = {
                NULL, sctp_cname, sctp_tname, sctp_oname, sctp_pname,
        };
        printfn_t *debug_fn  __attribute__ ((unused)) = table[event_type];
        const struct sctp_sm_table_entry *state_fn;
        struct sctp_cmd_seq commands;
        enum sctp_disposition status;
        int error = 0;

        /* Look up the state function, run it, and then process the
         * side effects.  These three steps are the heart of lksctp.
         */
        state_fn = sctp_sm_lookup_event(net, event_type, state, subtype);

        sctp_init_cmd_seq(&commands);

        debug_pre_sfn();
        status = state_fn->fn(net, ep, asoc, subtype, event_arg, &commands);
        debug_post_sfn();

        error = sctp_side_effects(event_type, subtype, state,
                                  ep, &asoc, event_arg, status,
                                  &commands, gfp);
        debug_post_sfx();

        return error;
}

/*****************************************************************
 * This the master state function side effect processing function.
 *****************************************************************/
static int sctp_side_effects(enum sctp_event_type event_type,
                             union sctp_subtype subtype,
                             enum sctp_state state,
                             struct sctp_endpoint *ep,
                             struct sctp_association **asoc,
                             void *event_arg,
                             enum sctp_disposition status,
                             struct sctp_cmd_seq *commands,
                             gfp_t gfp)
{
        int error;

        /* FIXME - Most of the dispositions left today would be categorized
         * as "exceptional" dispositions.  For those dispositions, it
         * may not be proper to run through any of the commands at all.
         * For example, the command interpreter might be run only with
         * disposition SCTP_DISPOSITION_CONSUME.
         */
        if (0 != (error = sctp_cmd_interpreter(event_type, subtype, state,
                                               ep, *asoc,
                                               event_arg, status,
                                               commands, gfp)))
                goto bail;

        switch (status) {
        case SCTP_DISPOSITION_DISCARD:
                pr_debug("%s: ignored sctp protocol event - state:%d, "
                         "event_type:%d, event_id:%d\n", __func__, state,
                         event_type, subtype.chunk);
                break;

        case SCTP_DISPOSITION_NOMEM:
                /* We ran out of memory, so we need to discard this
                 * packet.
                 */
                /* BUG--we should now recover some memory, probably by
                 * reneging...
                 */
                error = -ENOMEM;
                break;

        case SCTP_DISPOSITION_DELETE_TCB:
        case SCTP_DISPOSITION_ABORT:
                /* This should now be a command. */
                *asoc = NULL;
                break;

        case SCTP_DISPOSITION_CONSUME:
                /*
                 * We should no longer have much work to do here as the
                 * real work has been done as explicit commands above.
                 */
                break;

        case SCTP_DISPOSITION_VIOLATION:
                net_err_ratelimited("protocol violation state %d chunkid %d\n",
                                    state, subtype.chunk);
                break;

        case SCTP_DISPOSITION_NOT_IMPL:
                pr_warn("unimplemented feature in state %d, event_type %d, event_id %d\n",
                        state, event_type, subtype.chunk);
                break;

        case SCTP_DISPOSITION_BUG:
                pr_err("bug in state %d, event_type %d, event_id %d\n",
                       state, event_type, subtype.chunk);
                BUG();
                break;

        default:
                pr_err("impossible disposition %d in state %d, event_type %d, event_id %d\n",
                       status, state, event_type, subtype.chunk);
                error = status;
                if (error >= 0)
                        error = -EINVAL;
                WARN_ON_ONCE(1);
                break;
        }

bail:
        return error;
}

/********************************************************************
 * 2nd Level Abstractions
 ********************************************************************/

/* This is the side-effect interpreter.  */
static int sctp_cmd_interpreter(enum sctp_event_type event_type,
                                union sctp_subtype subtype,
                                enum sctp_state state,
                                struct sctp_endpoint *ep,
                                struct sctp_association *asoc,
                                void *event_arg,
                                enum sctp_disposition status,
                                struct sctp_cmd_seq *commands,
                                gfp_t gfp)
{
        struct sctp_sock *sp = sctp_sk(ep->base.sk);
        struct sctp_chunk *chunk = NULL, *new_obj;
        struct sctp_packet *packet;
        struct sctp_sackhdr sackh;
        struct timer_list *timer;
        struct sctp_transport *t;
        unsigned long timeout;
        struct sctp_cmd *cmd;
        int local_cork = 0;
        int error = 0;
        int force;

        if (SCTP_EVENT_T_TIMEOUT != event_type)
                chunk = event_arg;

        /* Note:  This whole file is a huge candidate for rework.
         * For example, each command could either have its own handler, so
         * the loop would look like:
         *     while (cmds)
         *         cmd->handle(x, y, z)
         * --jgrimm
         */
        while (NULL != (cmd = sctp_next_cmd(commands))) {
                switch (cmd->verb) {
                case SCTP_CMD_NOP:
                        /* Do nothing. */
                        break;

                case SCTP_CMD_NEW_ASOC:
                        /* Register a new association.  */
                        if (local_cork) {
                                sctp_outq_uncork(&asoc->outqueue, gfp);
                                local_cork = 0;
                        }

                        /* Register with the endpoint.  */
                        asoc = cmd->obj.asoc;
                        BUG_ON(asoc->peer.primary_path == NULL);
                        sctp_endpoint_add_asoc(ep, asoc);
                        break;

                case SCTP_CMD_PURGE_OUTQUEUE:
                       sctp_outq_teardown(&asoc->outqueue);
                       break;

                case SCTP_CMD_DELETE_TCB:
                        if (local_cork) {
                                sctp_outq_uncork(&asoc->outqueue, gfp);
                                local_cork = 0;
                        }
                        /* Delete the current association.  */
                        sctp_cmd_delete_tcb(commands, asoc);
                        asoc = NULL;
                        break;

                case SCTP_CMD_NEW_STATE:
                        /* Enter a new state.  */
                        sctp_cmd_new_state(commands, asoc, cmd->obj.state);
                        break;

                case SCTP_CMD_REPORT_TSN:
                        /* Record the arrival of a TSN.  */
                        error = sctp_tsnmap_mark(&asoc->peer.tsn_map,
                                                 cmd->obj.u32, NULL);
                        break;

                case SCTP_CMD_REPORT_FWDTSN:
                        asoc->stream.si->report_ftsn(&asoc->ulpq, cmd->obj.u32);
                        break;

                case SCTP_CMD_PROCESS_FWDTSN:
                        asoc->stream.si->handle_ftsn(&asoc->ulpq,
                                                     cmd->obj.chunk);
                        break;

                case SCTP_CMD_GEN_SACK:
                        /* Generate a Selective ACK.
                         * The argument tells us whether to just count
                         * the packet and MAYBE generate a SACK, or
                         * force a SACK out.
                         */
                        force = cmd->obj.i32;
                        error = sctp_gen_sack(asoc, force, commands);
                        break;

                case SCTP_CMD_PROCESS_SACK:
                        /* Process an inbound SACK.  */
                        error = sctp_cmd_process_sack(commands, asoc,
                                                      cmd->obj.chunk);
                        break;

                case SCTP_CMD_GEN_INIT_ACK:
                        /* Generate an INIT ACK chunk.  */
                        new_obj = sctp_make_init_ack(asoc, chunk, GFP_ATOMIC,
                                                     0);
                        if (!new_obj) {
                                error = -ENOMEM;
                                break;
                        }

                        sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                        SCTP_CHUNK(new_obj));
                        break;

                case SCTP_CMD_PEER_INIT:
                        /* Process a unified INIT from the peer.
                         * Note: Only used during INIT-ACK processing.  If
                         * there is an error just return to the outter
                         * layer which will bail.
                         */
                        error = sctp_cmd_process_init(commands, asoc, chunk,
                                                      cmd->obj.init, gfp);
                        break;

                case SCTP_CMD_GEN_COOKIE_ECHO:
                        /* Generate a COOKIE ECHO chunk.  */
                        new_obj = sctp_make_cookie_echo(asoc, chunk);
                        if (!new_obj) {
                                if (cmd->obj.chunk)
                                        sctp_chunk_free(cmd->obj.chunk);
                                error = -ENOMEM;
                                break;
                        }
                        sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                        SCTP_CHUNK(new_obj));

                        /* If there is an ERROR chunk to be sent along with
                         * the COOKIE_ECHO, send it, too.
                         */
                        if (cmd->obj.chunk)
                                sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                                SCTP_CHUNK(cmd->obj.chunk));

                        if (new_obj->transport) {
                                new_obj->transport->init_sent_count++;
                                asoc->init_last_sent_to = new_obj->transport;
                        }

                        /* FIXME - Eventually come up with a cleaner way to
                         * enabling COOKIE-ECHO + DATA bundling during
                         * multihoming stale cookie scenarios, the following
                         * command plays with asoc->peer.retran_path to
                         * avoid the problem of sending the COOKIE-ECHO and
                         * DATA in different paths, which could result
                         * in the association being ABORTed if the DATA chunk
                         * is processed first by the server.  Checking the
                         * init error counter simply causes this command
                         * to be executed only during failed attempts of
                         * association establishment.
                         */
                        if ((asoc->peer.retran_path !=
                             asoc->peer.primary_path) &&
                            (asoc->init_err_counter > 0)) {
                                sctp_add_cmd_sf(commands,
                                                SCTP_CMD_FORCE_PRIM_RETRAN,
                                                SCTP_NULL());
                        }

                        break;

                case SCTP_CMD_GEN_SHUTDOWN:
                        /* Generate SHUTDOWN when in SHUTDOWN_SENT state.
                         * Reset error counts.
                         */
                        asoc->overall_error_count = 0;

                        /* Generate a SHUTDOWN chunk.  */
                        new_obj = sctp_make_shutdown(asoc, chunk);
                        if (!new_obj) {
                                error = -ENOMEM;
                                break;
                        }
                        sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                        SCTP_CHUNK(new_obj));
                        break;

                case SCTP_CMD_CHUNK_ULP:
                        /* Send a chunk to the sockets layer.  */
                        pr_debug("%s: sm_sideff: chunk_up:%p, ulpq:%p\n",
                                 __func__, cmd->obj.chunk, &asoc->ulpq);

                        asoc->stream.si->ulpevent_data(&asoc->ulpq,
                                                       cmd->obj.chunk,
                                                       GFP_ATOMIC);
                        break;

                case SCTP_CMD_EVENT_ULP:
                        /* Send a notification to the sockets layer.  */
                        pr_debug("%s: sm_sideff: event_up:%p, ulpq:%p\n",
                                 __func__, cmd->obj.ulpevent, &asoc->ulpq);

                        asoc->stream.si->enqueue_event(&asoc->ulpq,
                                                       cmd->obj.ulpevent);
                        break;

                case SCTP_CMD_REPLY:
                        /* If an caller has not already corked, do cork. */
                        if (!asoc->outqueue.cork) {
                                sctp_outq_cork(&asoc->outqueue);
                                local_cork = 1;
                        }
                        /* Send a chunk to our peer.  */
                        sctp_outq_tail(&asoc->outqueue, cmd->obj.chunk, gfp);
                        break;

                case SCTP_CMD_SEND_PKT:
                        /* Send a full packet to our peer.  */
                        packet = cmd->obj.packet;
                        sctp_packet_transmit(packet, gfp);
                        sctp_ootb_pkt_free(packet);
                        break;

                case SCTP_CMD_T1_RETRAN:
                        /* Mark a transport for retransmission.  */
                        sctp_retransmit(&asoc->outqueue, cmd->obj.transport,
                                        SCTP_RTXR_T1_RTX);
                        break;

                case SCTP_CMD_RETRAN:
                        /* Mark a transport for retransmission.  */
                        sctp_retransmit(&asoc->outqueue, cmd->obj.transport,
                                        SCTP_RTXR_T3_RTX);
                        break;

                case SCTP_CMD_ECN_CE:
                        /* Do delayed CE processing.   */
                        sctp_do_ecn_ce_work(asoc, cmd->obj.u32);
                        break;

                case SCTP_CMD_ECN_ECNE:
                        /* Do delayed ECNE processing. */
                        new_obj = sctp_do_ecn_ecne_work(asoc, cmd->obj.u32,
                                                        chunk);
                        if (new_obj)
                                sctp_add_cmd_sf(commands, SCTP_CMD_REPLY,
                                                SCTP_CHUNK(new_obj));
                        break;

                case SCTP_CMD_ECN_CWR:
                        /* Do delayed CWR processing.  */
                        sctp_do_ecn_cwr_work(asoc, cmd->obj.u32);
                        break;

                case SCTP_CMD_SETUP_T2:
                        sctp_cmd_setup_t2(commands, asoc, cmd->obj.chunk);
                        break;

                case SCTP_CMD_TIMER_START_ONCE:
                        timer = &asoc->timers[cmd->obj.to];

                        if (timer_pending(timer))
                                break;
                        fallthrough;

                case SCTP_CMD_TIMER_START:
                        timer = &asoc->timers[cmd->obj.to];
                        timeout = asoc->timeouts[cmd->obj.to];
                        BUG_ON(!timeout);

                        /*
                         * SCTP has a hard time with timer starts.  Because we process
                         * timer starts as side effects, it can be hard to tell if we
                         * have already started a timer or not, which leads to BUG
                         * halts when we call add_timer. So here, instead of just starting
                         * a timer, if the timer is already started, and just mod
                         * the timer with the shorter of the two expiration times
                         */
                        if (!timer_pending(timer))
                                sctp_association_hold(asoc);
                        timer_reduce(timer, jiffies + timeout);
                        break;

                case SCTP_CMD_TIMER_RESTART:
                        timer = &asoc->timers[cmd->obj.to];
                        timeout = asoc->timeouts[cmd->obj.to];
                        if (!mod_timer(timer, jiffies + timeout))
                                sctp_association_hold(asoc);
                        break;

                case SCTP_CMD_TIMER_STOP:
                        timer = &asoc->timers[cmd->obj.to];
                        if (timer_delete(timer))
                                sctp_association_put(asoc);
                        break;

                case SCTP_CMD_INIT_CHOOSE_TRANSPORT:
                        chunk = cmd->obj.chunk;
                        t = sctp_assoc_choose_alter_transport(asoc,
                                                asoc->init_last_sent_to);
                        asoc->init_last_sent_to = t;
                        chunk->transport = t;
                        t->init_sent_count++;
                        /* Set the new transport as primary */
                        sctp_assoc_set_primary(asoc, t);
                        break;

                case SCTP_CMD_INIT_RESTART:
                        /* Do the needed accounting and updates
                         * associated with restarting an initialization
                         * timer. Only multiply the timeout by two if
                         * all transports have been tried at the current
                         * timeout.
                         */
                        sctp_cmd_t1_timer_update(asoc,
                                                SCTP_EVENT_TIMEOUT_T1_INIT,
                                                "INIT");

                        sctp_add_cmd_sf(commands, SCTP_CMD_TIMER_RESTART,
                                        SCTP_TO(SCTP_EVENT_TIMEOUT_T1_INIT));
                        break;

                case SCTP_CMD_COOKIEECHO_RESTART:
                        /* Do the needed accounting and updates
                         * associated with restarting an initialization
                         * timer. Only multiply the timeout by two if
                         * all transports have been tried at the current
                         * timeout.
                         */
                        sctp_cmd_t1_timer_update(asoc,
                                                SCTP_EVENT_TIMEOUT_T1_COOKIE,
                                                "COOKIE");

                        /* If we've sent any data bundled with
                         * COOKIE-ECHO we need to resend.
                         */
                        list_for_each_entry(t, &asoc->peer.transport_addr_list,
                                        transports) {
                                sctp_retransmit_mark(&asoc->outqueue, t,
                                            SCTP_RTXR_T1_RTX);
                        }

                        sctp_add_cmd_sf(commands,
                                        SCTP_CMD_TIMER_RESTART,
                                        SCTP_TO(SCTP_EVENT_TIMEOUT_T1_COOKIE));
                        break;

                case SCTP_CMD_INIT_FAILED:
                        sctp_cmd_init_failed(commands, asoc, cmd->obj.u16);
                        break;

                case SCTP_CMD_ASSOC_FAILED:
                        sctp_cmd_assoc_failed(commands, asoc, event_type,
                                              subtype, chunk, cmd->obj.u16);
                        break;

                case SCTP_CMD_INIT_COUNTER_INC:
                        asoc->init_err_counter++;
                        break;

                case SCTP_CMD_INIT_COUNTER_RESET:
                        asoc->init_err_counter = 0;
                        asoc->init_cycle = 0;
                        list_for_each_entry(t, &asoc->peer.transport_addr_list,
                                            transports) {
                                t->init_sent_count = 0;
                        }
                        break;

                case SCTP_CMD_REPORT_DUP:
                        sctp_tsnmap_mark_dup(&asoc->peer.tsn_map,
                                             cmd->obj.u32);
                        break;

                case SCTP_CMD_REPORT_BAD_TAG:
                        pr_debug("%s: vtag mismatch!\n", __func__);
                        break;

                case SCTP_CMD_STRIKE:
                        /* Mark one strike against a transport.  */
                        sctp_do_8_2_transport_strike(commands, asoc,
                                                    cmd->obj.transport, 0);
                        break;

                case SCTP_CMD_TRANSPORT_IDLE:
                        t = cmd->obj.transport;
                        sctp_transport_lower_cwnd(t, SCTP_LOWER_CWND_INACTIVE);
                        break;

                case SCTP_CMD_TRANSPORT_HB_SENT:
                        t = cmd->obj.transport;
                        sctp_do_8_2_transport_strike(commands, asoc,
                                                     t, 1);
                        t->hb_sent = 1;
                        break;

                case SCTP_CMD_TRANSPORT_ON:
                        t = cmd->obj.transport;
                        sctp_cmd_transport_on(commands, asoc, t, chunk);
                        break;

                case SCTP_CMD_HB_TIMERS_START:
                        sctp_cmd_hb_timers_start(commands, asoc);
                        break;

                case SCTP_CMD_HB_TIMER_UPDATE:
                        t = cmd->obj.transport;
                        sctp_transport_reset_hb_timer(t);
                        break;

                case SCTP_CMD_HB_TIMERS_STOP:
                        sctp_cmd_hb_timers_stop(commands, asoc);
                        break;

                case SCTP_CMD_PROBE_TIMER_UPDATE:
                        t = cmd->obj.transport;
                        sctp_transport_reset_probe_timer(t);
                        break;

                case SCTP_CMD_REPORT_ERROR:
                        error = cmd->obj.error;
                        break;

                case SCTP_CMD_PROCESS_CTSN:
                        /* Dummy up a SACK for processing. */
                        sackh.cum_tsn_ack = cmd->obj.be32;
                        sackh.a_rwnd = htonl(asoc->peer.rwnd +
                                             asoc->outqueue.outstanding_bytes);
                        sackh.num_gap_ack_blocks = 0;
                        sackh.num_dup_tsns = 0;
                        chunk->subh.sack_hdr = &sackh;
                        sctp_add_cmd_sf(commands, SCTP_CMD_PROCESS_SACK,
                                        SCTP_CHUNK(chunk));
                        break;

                case SCTP_CMD_DISCARD_PACKET:
                        /* We need to discard the whole packet.
                         * Uncork the queue since there might be
                         * responses pending
                         */
                        chunk->pdiscard = 1;
                        if (asoc) {
                                sctp_outq_uncork(&asoc->outqueue, gfp);
                                local_cork = 0;
                        }
                        break;

                case SCTP_CMD_RTO_PENDING:
                        t = cmd->obj.transport;
                        t->rto_pending = 1;
                        break;

                case SCTP_CMD_PART_DELIVER:
                        asoc->stream.si->start_pd(&asoc->ulpq, GFP_ATOMIC);
                        break;

                case SCTP_CMD_RENEGE:
                        asoc->stream.si->renege_events(&asoc->ulpq,
                                                       cmd->obj.chunk,
                                                       GFP_ATOMIC);
                        break;

                case SCTP_CMD_SETUP_T4:
                        sctp_cmd_setup_t4(commands, asoc, cmd->obj.chunk);
                        break;

                case SCTP_CMD_PROCESS_OPERR:
                        sctp_cmd_process_operr(commands, asoc, chunk);
                        break;
                case SCTP_CMD_CLEAR_INIT_TAG:
                        asoc->peer.i.init_tag = 0;
                        break;
                case SCTP_CMD_DEL_NON_PRIMARY:
                        sctp_cmd_del_non_primary(asoc);
                        break;
                case SCTP_CMD_T3_RTX_TIMERS_STOP:
                        sctp_cmd_t3_rtx_timers_stop(commands, asoc);
                        break;
                case SCTP_CMD_FORCE_PRIM_RETRAN:
                        t = asoc->peer.retran_path;
                        asoc->peer.retran_path = asoc->peer.primary_path;
                        sctp_outq_uncork(&asoc->outqueue, gfp);
                        local_cork = 0;
                        asoc->peer.retran_path = t;
                        break;
                case SCTP_CMD_SET_SK_ERR:
                        sctp_cmd_set_sk_err(asoc, cmd->obj.error);
                        break;
                case SCTP_CMD_ASSOC_CHANGE:
                        sctp_cmd_assoc_change(commands, asoc,
                                              cmd->obj.u8);
                        break;
                case SCTP_CMD_ADAPTATION_IND:
                        sctp_cmd_adaptation_ind(commands, asoc);
                        break;
                case SCTP_CMD_PEER_NO_AUTH:
                        sctp_cmd_peer_no_auth(commands, asoc);
                        break;

                case SCTP_CMD_ASSOC_SHKEY:
                        error = sctp_auth_asoc_init_active_key(asoc,
                                                GFP_ATOMIC);
                        break;
                case SCTP_CMD_UPDATE_INITTAG:
                        asoc->peer.i.init_tag = cmd->obj.u32;
                        break;
                case SCTP_CMD_SEND_MSG:
                        if (!asoc->outqueue.cork) {
                                sctp_outq_cork(&asoc->outqueue);
                                local_cork = 1;
                        }
                        sctp_cmd_send_msg(asoc, cmd->obj.msg, gfp);
                        break;
                case SCTP_CMD_PURGE_ASCONF_QUEUE:
                        sctp_asconf_queue_teardown(asoc);
                        break;

                case SCTP_CMD_SET_ASOC:
                        if (asoc && local_cork) {
                                sctp_outq_uncork(&asoc->outqueue, gfp);
                                local_cork = 0;
                        }
                        asoc = cmd->obj.asoc;
                        break;

                default:
                        pr_warn("Impossible command: %u\n",
                                cmd->verb);
                        break;
                }

                if (error) {
                        cmd = sctp_next_cmd(commands);
                        while (cmd) {
                                if (cmd->verb == SCTP_CMD_REPLY)
                                        sctp_chunk_free(cmd->obj.chunk);
                                cmd = sctp_next_cmd(commands);
                        }
                        break;
                }
        }

        /* If this is in response to a received chunk, wait until
         * we are done with the packet to open the queue so that we don't
         * send multiple packets in response to a single request.
         */
        if (asoc && SCTP_EVENT_T_CHUNK == event_type && chunk) {
                if (chunk->end_of_packet || chunk->singleton)
                        sctp_outq_uncork(&asoc->outqueue, gfp);
        } else if (local_cork)
                sctp_outq_uncork(&asoc->outqueue, gfp);

        if (sp->data_ready_signalled)
                sp->data_ready_signalled = 0;

        return error;
}



























   10 




    5 


    1 


    5 




    2 


    1 









    2 

















    1 


   60 



   52 


    8 






    7 







    4 



    1 






   60 







   60 


   59 





    1 























   10 











    1 
    2 







    1 









    1 




    1 




    1 




    3 



















    2 
    1 
















    2 






    1 







    1 













    1 














    1 
    1 




    1 















































    1 












    1 








    1 










    1 



























    1 


























    5 
















    5 










































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  SR-IPv6 implementation
 *
 *  Author:
 *  David Lebrun <david.lebrun@uclouvain.be>
 */

#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/net.h>
#include <linux/in6.h>
#include <linux/slab.h>
#include <linux/rhashtable.h>

#include <net/ipv6.h>
#include <net/protocol.h>

#include <net/seg6.h>
#include <net/genetlink.h>
#include <linux/seg6.h>
#include <linux/seg6_genl.h>
#include <net/seg6_hmac.h>

bool seg6_validate_srh(struct ipv6_sr_hdr *srh, int len, bool reduced)
{
        unsigned int tlv_offset;
        int max_last_entry;
        int trailing;

        if (srh->type != IPV6_SRCRT_TYPE_4)
                return false;

        if (((srh->hdrlen + 1) << 3) != len)
                return false;

        if (!reduced && srh->segments_left > srh->first_segment) {
                return false;
        } else {
                max_last_entry = (srh->hdrlen / 2) - 1;

                if (srh->first_segment > max_last_entry)
                        return false;

                if (srh->segments_left > srh->first_segment + 1)
                        return false;
        }

        tlv_offset = sizeof(*srh) + ((srh->first_segment + 1) << 4);

        trailing = len - tlv_offset;
        if (trailing < 0)
                return false;

        while (trailing) {
                struct sr6_tlv *tlv;
                unsigned int tlv_len;

                if (trailing < sizeof(*tlv))
                        return false;

                tlv = (struct sr6_tlv *)((unsigned char *)srh + tlv_offset);
                tlv_len = sizeof(*tlv) + tlv->len;

                trailing -= tlv_len;
                if (trailing < 0)
                        return false;

                tlv_offset += tlv_len;
        }

        return true;
}

struct ipv6_sr_hdr *seg6_get_srh(struct sk_buff *skb, int flags)
{
        struct ipv6_sr_hdr *srh;
        int len, srhoff = 0;

        if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, &flags) < 0)
                return NULL;

        if (!pskb_may_pull(skb, srhoff + sizeof(*srh)))
                return NULL;

        srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);

        len = (srh->hdrlen + 1) << 3;

        if (!pskb_may_pull(skb, srhoff + len))
                return NULL;

        /* note that pskb_may_pull may change pointers in header;
         * for this reason it is necessary to reload them when needed.
         */
        srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);

        if (!seg6_validate_srh(srh, len, true))
                return NULL;

        return srh;
}

/* Determine if an ICMP invoking packet contains a segment routing
 * header.  If it does, extract the offset to the true destination
 * address, which is in the first segment address.
 */
void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt)
{
        __u16 network_header = skb->network_header;
        struct ipv6_sr_hdr *srh;

        /* Update network header to point to the invoking packet
         * inside the ICMP packet, so we can use the seg6_get_srh()
         * helper.
         */
        skb_reset_network_header(skb);

        srh = seg6_get_srh(skb, 0);
        if (!srh)
                goto out;

        if (srh->type != IPV6_SRCRT_TYPE_4)
                goto out;

        opt->flags |= IP6SKB_SEG6;
        opt->srhoff = (unsigned char *)srh - skb->data;

out:
        /* Restore the network header back to the ICMP packet */
        skb->network_header = network_header;
}

static struct genl_family seg6_genl_family;

static const struct nla_policy seg6_genl_policy[SEG6_ATTR_MAX + 1] = {
        [SEG6_ATTR_DST]                                = { .type = NLA_BINARY,
                .len = sizeof(struct in6_addr) },
        [SEG6_ATTR_DSTLEN]                        = { .type = NLA_S32, },
        [SEG6_ATTR_HMACKEYID]                = { .type = NLA_U32, },
        [SEG6_ATTR_SECRET]                        = { .type = NLA_BINARY, },
        [SEG6_ATTR_SECRETLEN]                = { .type = NLA_U8, },
        [SEG6_ATTR_ALGID]                        = { .type = NLA_U8, },
        [SEG6_ATTR_HMACINFO]                = { .type = NLA_NESTED, },
};

#ifdef CONFIG_IPV6_SEG6_HMAC

static int seg6_genl_sethmac(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct seg6_pernet_data *sdata;
        struct seg6_hmac_info *hinfo;
        u32 hmackeyid;
        char *secret;
        int err = 0;
        u8 algid;
        u8 slen;

        sdata = seg6_pernet(net);

        if (!info->attrs[SEG6_ATTR_HMACKEYID] ||
            !info->attrs[SEG6_ATTR_SECRETLEN] ||
            !info->attrs[SEG6_ATTR_ALGID])
                return -EINVAL;

        hmackeyid = nla_get_u32(info->attrs[SEG6_ATTR_HMACKEYID]);
        slen = nla_get_u8(info->attrs[SEG6_ATTR_SECRETLEN]);
        algid = nla_get_u8(info->attrs[SEG6_ATTR_ALGID]);

        if (hmackeyid == 0)
                return -EINVAL;

        if (slen > SEG6_HMAC_SECRET_LEN)
                return -EINVAL;

        mutex_lock(&sdata->lock);
        hinfo = seg6_hmac_info_lookup(net, hmackeyid);

        if (!slen) {
                err = seg6_hmac_info_del(net, hmackeyid);

                goto out_unlock;
        }

        if (!info->attrs[SEG6_ATTR_SECRET]) {
                err = -EINVAL;
                goto out_unlock;
        }

        if (slen > nla_len(info->attrs[SEG6_ATTR_SECRET])) {
                err = -EINVAL;
                goto out_unlock;
        }

        if (hinfo) {
                err = seg6_hmac_info_del(net, hmackeyid);
                if (err)
                        goto out_unlock;
        }

        secret = (char *)nla_data(info->attrs[SEG6_ATTR_SECRET]);

        hinfo = kzalloc(sizeof(*hinfo), GFP_KERNEL);
        if (!hinfo) {
                err = -ENOMEM;
                goto out_unlock;
        }

        memcpy(hinfo->secret, secret, slen);
        hinfo->slen = slen;
        hinfo->alg_id = algid;
        hinfo->hmackeyid = hmackeyid;

        err = seg6_hmac_info_add(net, hmackeyid, hinfo);
        if (err)
                kfree(hinfo);

out_unlock:
        mutex_unlock(&sdata->lock);
        return err;
}

#else

static int seg6_genl_sethmac(struct sk_buff *skb, struct genl_info *info)
{
        return -ENOTSUPP;
}

#endif

static int seg6_genl_set_tunsrc(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct in6_addr *val, *t_old, *t_new;
        struct seg6_pernet_data *sdata;

        sdata = seg6_pernet(net);

        if (!info->attrs[SEG6_ATTR_DST])
                return -EINVAL;

        val = nla_data(info->attrs[SEG6_ATTR_DST]);
        t_new = kmemdup(val, sizeof(*val), GFP_KERNEL);
        if (!t_new)
                return -ENOMEM;

        mutex_lock(&sdata->lock);

        t_old = sdata->tun_src;
        rcu_assign_pointer(sdata->tun_src, t_new);

        mutex_unlock(&sdata->lock);

        synchronize_net();
        kfree(t_old);

        return 0;
}

static int seg6_genl_get_tunsrc(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct in6_addr *tun_src;
        struct sk_buff *msg;
        void *hdr;

        msg = genlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
                          &seg6_genl_family, 0, SEG6_CMD_GET_TUNSRC);
        if (!hdr)
                goto free_msg;

        rcu_read_lock();
        tun_src = rcu_dereference(seg6_pernet(net)->tun_src);

        if (nla_put(msg, SEG6_ATTR_DST, sizeof(struct in6_addr), tun_src))
                goto nla_put_failure;

        rcu_read_unlock();

        genlmsg_end(msg, hdr);
        return genlmsg_reply(msg, info);

nla_put_failure:
        rcu_read_unlock();
free_msg:
        nlmsg_free(msg);
        return -ENOMEM;
}

#ifdef CONFIG_IPV6_SEG6_HMAC

static int __seg6_hmac_fill_info(struct seg6_hmac_info *hinfo,
                                 struct sk_buff *msg)
{
        if (nla_put_u32(msg, SEG6_ATTR_HMACKEYID, hinfo->hmackeyid) ||
            nla_put_u8(msg, SEG6_ATTR_SECRETLEN, hinfo->slen) ||
            nla_put(msg, SEG6_ATTR_SECRET, hinfo->slen, hinfo->secret) ||
            nla_put_u8(msg, SEG6_ATTR_ALGID, hinfo->alg_id))
                return -1;

        return 0;
}

static int __seg6_genl_dumphmac_element(struct seg6_hmac_info *hinfo,
                                        u32 portid, u32 seq, u32 flags,
                                        struct sk_buff *skb, u8 cmd)
{
        void *hdr;

        hdr = genlmsg_put(skb, portid, seq, &seg6_genl_family, flags, cmd);
        if (!hdr)
                return -ENOMEM;

        if (__seg6_hmac_fill_info(hinfo, skb) < 0)
                goto nla_put_failure;

        genlmsg_end(skb, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(skb, hdr);
        return -EMSGSIZE;
}

static int seg6_genl_dumphmac_start(struct netlink_callback *cb)
{
        struct net *net = sock_net(cb->skb->sk);
        struct seg6_pernet_data *sdata;
        struct rhashtable_iter *iter;

        sdata = seg6_pernet(net);
        iter = (struct rhashtable_iter *)cb->args[0];

        if (!iter) {
                iter = kmalloc(sizeof(*iter), GFP_KERNEL);
                if (!iter)
                        return -ENOMEM;

                cb->args[0] = (long)iter;
        }

        rhashtable_walk_enter(&sdata->hmac_infos, iter);

        return 0;
}

static int seg6_genl_dumphmac_done(struct netlink_callback *cb)
{
        struct rhashtable_iter *iter = (struct rhashtable_iter *)cb->args[0];

        rhashtable_walk_exit(iter);

        kfree(iter);

        return 0;
}

static int seg6_genl_dumphmac(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rhashtable_iter *iter = (struct rhashtable_iter *)cb->args[0];
        struct seg6_hmac_info *hinfo;
        int ret;

        rhashtable_walk_start(iter);

        for (;;) {
                hinfo = rhashtable_walk_next(iter);

                if (IS_ERR(hinfo)) {
                        if (PTR_ERR(hinfo) == -EAGAIN)
                                continue;
                        ret = PTR_ERR(hinfo);
                        goto done;
                } else if (!hinfo) {
                        break;
                }

                ret = __seg6_genl_dumphmac_element(hinfo,
                                                   NETLINK_CB(cb->skb).portid,
                                                   cb->nlh->nlmsg_seq,
                                                   NLM_F_MULTI,
                                                   skb, SEG6_CMD_DUMPHMAC);
                if (ret)
                        goto done;
        }

        ret = skb->len;

done:
        rhashtable_walk_stop(iter);
        return ret;
}

#else

static int seg6_genl_dumphmac_start(struct netlink_callback *cb)
{
        return 0;
}

static int seg6_genl_dumphmac_done(struct netlink_callback *cb)
{
        return 0;
}

static int seg6_genl_dumphmac(struct sk_buff *skb, struct netlink_callback *cb)
{
        return -ENOTSUPP;
}

#endif

static int __net_init seg6_net_init(struct net *net)
{
        struct seg6_pernet_data *sdata;

        sdata = kzalloc(sizeof(*sdata), GFP_KERNEL);
        if (!sdata)
                return -ENOMEM;

        mutex_init(&sdata->lock);

        sdata->tun_src = kzalloc(sizeof(*sdata->tun_src), GFP_KERNEL);
        if (!sdata->tun_src) {
                kfree(sdata);
                return -ENOMEM;
        }

        net->ipv6.seg6_data = sdata;

        if (seg6_hmac_net_init(net)) {
                kfree(rcu_dereference_raw(sdata->tun_src));
                kfree(sdata);
                return -ENOMEM;
        }

        return 0;
}

static void __net_exit seg6_net_exit(struct net *net)
{
        struct seg6_pernet_data *sdata = seg6_pernet(net);

        seg6_hmac_net_exit(net);

        kfree(rcu_dereference_raw(sdata->tun_src));
        kfree(sdata);
}

static struct pernet_operations ip6_segments_ops = {
        .init = seg6_net_init,
        .exit = seg6_net_exit,
};

static const struct genl_ops seg6_genl_ops[] = {
        {
                .cmd        = SEG6_CMD_SETHMAC,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = seg6_genl_sethmac,
                .flags        = GENL_ADMIN_PERM,
        },
        {
                .cmd        = SEG6_CMD_DUMPHMAC,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .start        = seg6_genl_dumphmac_start,
                .dumpit        = seg6_genl_dumphmac,
                .done        = seg6_genl_dumphmac_done,
                .flags        = GENL_ADMIN_PERM,
        },
        {
                .cmd        = SEG6_CMD_SET_TUNSRC,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = seg6_genl_set_tunsrc,
                .flags        = GENL_ADMIN_PERM,
        },
        {
                .cmd        = SEG6_CMD_GET_TUNSRC,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .doit        = seg6_genl_get_tunsrc,
                .flags        = GENL_ADMIN_PERM,
        },
};

static struct genl_family seg6_genl_family __ro_after_init = {
        .hdrsize        = 0,
        .name                = SEG6_GENL_NAME,
        .version        = SEG6_GENL_VERSION,
        .maxattr        = SEG6_ATTR_MAX,
        .policy = seg6_genl_policy,
        .netnsok        = true,
        .parallel_ops        = true,
        .ops                = seg6_genl_ops,
        .n_ops                = ARRAY_SIZE(seg6_genl_ops),
        .resv_start_op        = SEG6_CMD_GET_TUNSRC + 1,
        .module                = THIS_MODULE,
};

int __init seg6_init(void)
{
        int err;

        err = register_pernet_subsys(&ip6_segments_ops);
        if (err)
                goto out;

        err = genl_register_family(&seg6_genl_family);
        if (err)
                goto out_unregister_pernet;

        err = seg6_iptunnel_init();
        if (err)
                goto out_unregister_genl;

        err = seg6_local_init();
        if (err)
                goto out_unregister_iptun;

        pr_info("Segment Routing with IPv6\n");

out:
        return err;
out_unregister_iptun:
        seg6_iptunnel_exit();
out_unregister_genl:
        genl_unregister_family(&seg6_genl_family);
out_unregister_pernet:
        unregister_pernet_subsys(&ip6_segments_ops);
        goto out;
}

void seg6_exit(void)
{
        seg6_local_exit();
        seg6_iptunnel_exit();
        genl_unregister_family(&seg6_genl_family);
        unregister_pernet_subsys(&ip6_segments_ops);
}






















































































































































































































































































































    1 




    1 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2020 Facebook
 * Copyright 2020 Google LLC.
 */

#include <linux/pid.h>
#include <linux/sched.h>
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/bpf_local_storage.h>
#include <linux/filter.h>
#include <uapi/linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/rcupdate_trace.h>

DEFINE_BPF_STORAGE_CACHE(task_cache);

static DEFINE_PER_CPU(int, bpf_task_storage_busy);

static void bpf_task_storage_lock(void)
{
        cant_migrate();
        this_cpu_inc(bpf_task_storage_busy);
}

static void bpf_task_storage_unlock(void)
{
        this_cpu_dec(bpf_task_storage_busy);
}

static bool bpf_task_storage_trylock(void)
{
        cant_migrate();
        if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
                this_cpu_dec(bpf_task_storage_busy);
                return false;
        }
        return true;
}

static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
{
        struct task_struct *task = owner;

        return &task->bpf_storage;
}

static struct bpf_local_storage_data *
task_storage_lookup(struct task_struct *task, struct bpf_map *map,
                    bool cacheit_lockit)
{
        struct bpf_local_storage *task_storage;
        struct bpf_local_storage_map *smap;

        task_storage =
                rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
        if (!task_storage)
                return NULL;

        smap = (struct bpf_local_storage_map *)map;
        return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit);
}

void bpf_task_storage_free(struct task_struct *task)
{
        struct bpf_local_storage *local_storage;

        rcu_read_lock_dont_migrate();

        local_storage = rcu_dereference(task->bpf_storage);
        if (!local_storage)
                goto out;

        bpf_task_storage_lock();
        bpf_local_storage_destroy(local_storage);
        bpf_task_storage_unlock();
out:
        rcu_read_unlock_migrate();
}

static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
{
        struct bpf_local_storage_data *sdata;
        struct task_struct *task;
        unsigned int f_flags;
        struct pid *pid;
        int fd, err;

        fd = *(int *)key;
        pid = pidfd_get_pid(fd, &f_flags);
        if (IS_ERR(pid))
                return ERR_CAST(pid);

        /* We should be in an RCU read side critical section, it should be safe
         * to call pid_task.
         */
        WARN_ON_ONCE(!rcu_read_lock_held());
        task = pid_task(pid, PIDTYPE_PID);
        if (!task) {
                err = -ENOENT;
                goto out;
        }

        bpf_task_storage_lock();
        sdata = task_storage_lookup(task, map, true);
        bpf_task_storage_unlock();
        put_pid(pid);
        return sdata ? sdata->data : NULL;
out:
        put_pid(pid);
        return ERR_PTR(err);
}

static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
                                             void *value, u64 map_flags)
{
        struct bpf_local_storage_data *sdata;
        struct task_struct *task;
        unsigned int f_flags;
        struct pid *pid;
        int fd, err;

        if ((map_flags & BPF_F_LOCK) && btf_record_has_field(map->record, BPF_UPTR))
                return -EOPNOTSUPP;

        fd = *(int *)key;
        pid = pidfd_get_pid(fd, &f_flags);
        if (IS_ERR(pid))
                return PTR_ERR(pid);

        /* We should be in an RCU read side critical section, it should be safe
         * to call pid_task.
         */
        WARN_ON_ONCE(!rcu_read_lock_held());
        task = pid_task(pid, PIDTYPE_PID);
        if (!task) {
                err = -ENOENT;
                goto out;
        }

        bpf_task_storage_lock();
        sdata = bpf_local_storage_update(
                task, (struct bpf_local_storage_map *)map, value, map_flags,
                true, GFP_ATOMIC);
        bpf_task_storage_unlock();

        err = PTR_ERR_OR_ZERO(sdata);
out:
        put_pid(pid);
        return err;
}

static int task_storage_delete(struct task_struct *task, struct bpf_map *map,
                               bool nobusy)
{
        struct bpf_local_storage_data *sdata;

        sdata = task_storage_lookup(task, map, false);
        if (!sdata)
                return -ENOENT;

        if (!nobusy)
                return -EBUSY;

        bpf_selem_unlink(SELEM(sdata), false);

        return 0;
}

static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)
{
        struct task_struct *task;
        unsigned int f_flags;
        struct pid *pid;
        int fd, err;

        fd = *(int *)key;
        pid = pidfd_get_pid(fd, &f_flags);
        if (IS_ERR(pid))
                return PTR_ERR(pid);

        /* We should be in an RCU read side critical section, it should be safe
         * to call pid_task.
         */
        WARN_ON_ONCE(!rcu_read_lock_held());
        task = pid_task(pid, PIDTYPE_PID);
        if (!task) {
                err = -ENOENT;
                goto out;
        }

        bpf_task_storage_lock();
        err = task_storage_delete(task, map, true);
        bpf_task_storage_unlock();
out:
        put_pid(pid);
        return err;
}

/* Called by bpf_task_storage_get*() helpers */
static void *__bpf_task_storage_get(struct bpf_map *map,
                                    struct task_struct *task, void *value,
                                    u64 flags, gfp_t gfp_flags, bool nobusy)
{
        struct bpf_local_storage_data *sdata;

        sdata = task_storage_lookup(task, map, nobusy);
        if (sdata)
                return sdata->data;

        /* only allocate new storage, when the task is refcounted */
        if (refcount_read(&task->usage) &&
            (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {
                sdata = bpf_local_storage_update(
                        task, (struct bpf_local_storage_map *)map, value,
                        BPF_NOEXIST, false, gfp_flags);
                return IS_ERR(sdata) ? NULL : sdata->data;
        }

        return NULL;
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *,
           task, void *, value, u64, flags, gfp_t, gfp_flags)
{
        bool nobusy;
        void *data;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task)
                return (unsigned long)NULL;

        nobusy = bpf_task_storage_trylock();
        data = __bpf_task_storage_get(map, task, value, flags,
                                      gfp_flags, nobusy);
        if (nobusy)
                bpf_task_storage_unlock();
        return (unsigned long)data;
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
           task, void *, value, u64, flags, gfp_t, gfp_flags)
{
        void *data;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task)
                return (unsigned long)NULL;

        bpf_task_storage_lock();
        data = __bpf_task_storage_get(map, task, value, flags,
                                      gfp_flags, true);
        bpf_task_storage_unlock();
        return (unsigned long)data;
}

BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *,
           task)
{
        bool nobusy;
        int ret;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (!task)
                return -EINVAL;

        nobusy = bpf_task_storage_trylock();
        /* This helper must only be called from places where the lifetime of the task
         * is guaranteed. Either by being refcounted or by being protected
         * by an RCU read-side critical section.
         */
        ret = task_storage_delete(task, map, nobusy);
        if (nobusy)
                bpf_task_storage_unlock();
        return ret;
}

BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *,
           task)
{
        int ret;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (!task)
                return -EINVAL;

        bpf_task_storage_lock();
        /* This helper must only be called from places where the lifetime of the task
         * is guaranteed. Either by being refcounted or by being protected
         * by an RCU read-side critical section.
         */
        ret = task_storage_delete(task, map, true);
        bpf_task_storage_unlock();
        return ret;
}

static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
        return -ENOTSUPP;
}

static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
{
        return bpf_local_storage_map_alloc(attr, &task_cache, true);
}

static void task_storage_map_free(struct bpf_map *map)
{
        bpf_local_storage_map_free(map, &task_cache, &bpf_task_storage_busy);
}

BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map)
const struct bpf_map_ops task_storage_map_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = bpf_local_storage_map_alloc_check,
        .map_alloc = task_storage_map_alloc,
        .map_free = task_storage_map_free,
        .map_get_next_key = notsupp_get_next_key,
        .map_lookup_elem = bpf_pid_task_storage_lookup_elem,
        .map_update_elem = bpf_pid_task_storage_update_elem,
        .map_delete_elem = bpf_pid_task_storage_delete_elem,
        .map_check_btf = bpf_local_storage_map_check_btf,
        .map_mem_usage = bpf_local_storage_map_mem_usage,
        .map_btf_id = &bpf_local_storage_map_btf_id[0],
        .map_owner_storage_ptr = task_storage_ptr,
};

const struct bpf_func_proto bpf_task_storage_get_recur_proto = {
        .func = bpf_task_storage_get_recur,
        .gpl_only = false,
        .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
        .arg1_type = ARG_CONST_MAP_PTR,
        .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
        .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
        .arg4_type = ARG_ANYTHING,
};

const struct bpf_func_proto bpf_task_storage_get_proto = {
        .func = bpf_task_storage_get,
        .gpl_only = false,
        .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
        .arg1_type = ARG_CONST_MAP_PTR,
        .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
        .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
        .arg4_type = ARG_ANYTHING,
};

const struct bpf_func_proto bpf_task_storage_delete_recur_proto = {
        .func = bpf_task_storage_delete_recur,
        .gpl_only = false,
        .ret_type = RET_INTEGER,
        .arg1_type = ARG_CONST_MAP_PTR,
        .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
};

const struct bpf_func_proto bpf_task_storage_delete_proto = {
        .func = bpf_task_storage_delete,
        .gpl_only = false,
        .ret_type = RET_INTEGER,
        .arg1_type = ARG_CONST_MAP_PTR,
        .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
};










































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _linux_POSIX_TIMERS_H
#define _linux_POSIX_TIMERS_H

#include <linux/alarmtimer.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/pid.h>
#include <linux/posix-timers_types.h>
#include <linux/rcuref.h>
#include <linux/spinlock.h>
#include <linux/timerqueue.h>

struct kernel_siginfo;
struct task_struct;
struct sigqueue;
struct k_itimer;

static inline clockid_t make_process_cpuclock(const unsigned int pid,
                const clockid_t clock)
{
        return ((~pid) << 3) | clock;
}
static inline clockid_t make_thread_cpuclock(const unsigned int tid,
                const clockid_t clock)
{
        return make_process_cpuclock(tid, clock | CPUCLOCK_PERTHREAD_MASK);
}

static inline clockid_t fd_to_clockid(const int fd)
{
        return make_process_cpuclock((unsigned int) fd, CLOCKFD);
}

static inline int clockid_to_fd(const clockid_t clk)
{
        return ~(clk >> 3);
}

static inline bool clockid_aux_valid(clockid_t id)
{
        return IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) && id >= CLOCK_AUX && id <= CLOCK_AUX_LAST;
}

#ifdef CONFIG_POSIX_TIMERS

#include <linux/signal_types.h>

/**
 * cpu_timer - Posix CPU timer representation for k_itimer
 * @node:        timerqueue node to queue in the task/sig
 * @head:        timerqueue head on which this timer is queued
 * @pid:        Pointer to target task PID
 * @elist:        List head for the expiry list
 * @firing:        Timer is currently firing
 * @nanosleep:        Timer is used for nanosleep and is not a regular posix-timer
 * @handling:        Pointer to the task which handles expiry
 */
struct cpu_timer {
        struct timerqueue_node                node;
        struct timerqueue_head                *head;
        struct pid                        *pid;
        struct list_head                elist;
        bool                                firing;
        bool                                nanosleep;
        struct task_struct __rcu        *handling;
};

static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
                                     struct cpu_timer *ctmr)
{
        ctmr->head = head;
        return timerqueue_add(head, &ctmr->node);
}

static inline bool cpu_timer_queued(struct cpu_timer *ctmr)
{
        return !!ctmr->head;
}

static inline bool cpu_timer_dequeue(struct cpu_timer *ctmr)
{
        if (cpu_timer_queued(ctmr)) {
                timerqueue_del(ctmr->head, &ctmr->node);
                ctmr->head = NULL;
                return true;
        }
        return false;
}

static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr)
{
        return ctmr->node.expires;
}

static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp)
{
        ctmr->node.expires = exp;
}

static inline void posix_cputimers_init(struct posix_cputimers *pct)
{
        memset(pct, 0, sizeof(*pct));
        pct->bases[0].nextevt = U64_MAX;
        pct->bases[1].nextevt = U64_MAX;
        pct->bases[2].nextevt = U64_MAX;
}

void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit);

static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct,
                                               u64 runtime)
{
        pct->bases[CPUCLOCK_SCHED].nextevt = runtime;
}

void posixtimer_rearm_itimer(struct task_struct *p);
bool posixtimer_init_sigqueue(struct sigqueue *q);
void posixtimer_send_sigqueue(struct k_itimer *tmr);
bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq);
void posixtimer_free_timer(struct k_itimer *timer);
long posixtimer_create_prctl(unsigned long ctrl);

/* Init task static initializer */
#define INIT_CPU_TIMERBASE(b) {                                                \
        .nextevt        = U64_MAX,                                        \
}

#define INIT_CPU_TIMERBASES(b) {                                        \
        INIT_CPU_TIMERBASE(b[0]),                                        \
        INIT_CPU_TIMERBASE(b[1]),                                        \
        INIT_CPU_TIMERBASE(b[2]),                                        \
}

#define INIT_CPU_TIMERS(s)                                                \
        .posix_cputimers = {                                                \
                .bases = INIT_CPU_TIMERBASES(s.posix_cputimers.bases),        \
        },
#else
struct cpu_timer { };
#define INIT_CPU_TIMERS(s)
static inline void posix_cputimers_init(struct posix_cputimers *pct) { }
static inline void posix_cputimers_group_init(struct posix_cputimers *pct,
                                              u64 cpu_limit) { }
static inline void posixtimer_rearm_itimer(struct task_struct *p) { }
static inline bool posixtimer_deliver_signal(struct kernel_siginfo *info,
                                             struct sigqueue *timer_sigq) { return false; }
static inline void posixtimer_free_timer(struct k_itimer *timer) { }
static inline long posixtimer_create_prctl(unsigned long ctrl) { return -EINVAL; }
#endif

#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
void clear_posix_cputimers_work(struct task_struct *p);
void posix_cputimers_init_work(void);
#else
static inline void clear_posix_cputimers_work(struct task_struct *p) { }
static inline void posix_cputimers_init_work(void) { }
#endif

/**
 * struct k_itimer - POSIX.1b interval timer structure.
 * @list:                List node for binding the timer to tsk::signal::posix_timers
 * @ignored_list:        List node for tracking ignored timers in tsk::signal::ignored_posix_timers
 * @t_hash:                Entry in the posix timer hash table
 * @it_lock:                Lock protecting the timer
 * @kclock:                Pointer to the k_clock struct handling this timer
 * @it_clock:                The posix timer clock id
 * @it_id:                The posix timer id for identifying the timer
 * @it_status:                The status of the timer
 * @it_sig_periodic:        The periodic status at signal delivery
 * @it_overrun:                The overrun counter for pending signals
 * @it_overrun_last:        The overrun at the time of the last delivered signal
 * @it_signal_seq:        Sequence count to control signal delivery
 * @it_sigqueue_seq:        The sequence count at the point where the signal was queued
 * @it_sigev_notify:        The notify word of sigevent struct for signal delivery
 * @it_interval:        The interval for periodic timers
 * @it_signal:                Pointer to the creators signal struct
 * @it_pid:                The pid of the process/task targeted by the signal
 * @it_process:                The task to wakeup on clock_nanosleep (CPU timers)
 * @rcuref:                Reference count for life time management
 * @sigq:                Embedded sigqueue
 * @it:                        Union representing the various posix timer type
 *                        internals.
 * @rcu:                RCU head for freeing the timer.
 */
struct k_itimer {
        /* 1st cacheline contains read-mostly fields */
        struct hlist_node        t_hash;
        struct hlist_node        list;
        timer_t                        it_id;
        clockid_t                it_clock;
        int                        it_sigev_notify;
        enum pid_type                it_pid_type;
        struct signal_struct        *it_signal;
        const struct k_clock        *kclock;

        /* 2nd cacheline and above contain fields which are modified regularly */
        spinlock_t                it_lock;
        int                        it_status;
        bool                        it_sig_periodic;
        s64                        it_overrun;
        s64                        it_overrun_last;
        unsigned int                it_signal_seq;
        unsigned int                it_sigqueue_seq;
        ktime_t                        it_interval;
        struct hlist_node        ignored_list;
        union {
                struct pid                *it_pid;
                struct task_struct        *it_process;
        };
        struct sigqueue                sigq;
        rcuref_t                rcuref;
        union {
                struct {
                        struct hrtimer        timer;
                } real;
                struct cpu_timer        cpu;
                struct {
                        struct alarm        alarmtimer;
                } alarm;
        } it;
        struct rcu_head                rcu;
} ____cacheline_aligned_in_smp;

void run_posix_cpu_timers(void);
void posix_cpu_timers_exit(struct task_struct *task);
void posix_cpu_timers_exit_group(struct task_struct *task);
void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
                           u64 *newval, u64 *oldval);

int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new);

#ifdef CONFIG_POSIX_TIMERS
static inline void posixtimer_putref(struct k_itimer *tmr)
{
        if (rcuref_put(&tmr->rcuref))
                posixtimer_free_timer(tmr);
}

static inline void posixtimer_sigqueue_getref(struct sigqueue *q)
{
        struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);

        WARN_ON_ONCE(!rcuref_get(&tmr->rcuref));
}

static inline void posixtimer_sigqueue_putref(struct sigqueue *q)
{
        struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);

        posixtimer_putref(tmr);
}

static inline bool posixtimer_valid(const struct k_itimer *timer)
{
        unsigned long val = (unsigned long)timer->it_signal;

        return !(val & 0x1UL);
}
#else  /* CONFIG_POSIX_TIMERS */
static inline void posixtimer_sigqueue_getref(struct sigqueue *q) { }
static inline void posixtimer_sigqueue_putref(struct sigqueue *q) { }
#endif /* !CONFIG_POSIX_TIMERS */

#endif















   16 









   11 

    5 









    8 















    8 











   38 



   38 
    6 
    6 







    3 



    3 










    1 



    1 















































































































   31 









    2 



   30 
   30 
   30 



   30 

   30 






   30 
   30 
    9 

   10 
   10 









    9 




   30 




   29 









   30 










    2 



    1 

    1 












   11 








   11 

    1 

    7 
    2 




    7 






   11 
    1 




    1 






















   11 








    6 
    2 
    3 



    8 
    8 
    4 





    8 



    8 














    4 







    4 






    4 
    4 
    4 
    4 






    4 


    4 









    4 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
/* Linux multicast routing support
 * Common logic shared by IPv4 [ipmr] and IPv6 [ip6mr] implementation
 */

#include <linux/rhashtable.h>
#include <linux/mroute_base.h>

/* Sets everything common except 'dev', since that is done under locking */
void vif_device_init(struct vif_device *v,
                     struct net_device *dev,
                     unsigned long rate_limit,
                     unsigned char threshold,
                     unsigned short flags,
                     unsigned short get_iflink_mask)
{
        RCU_INIT_POINTER(v->dev, NULL);
        v->bytes_in = 0;
        v->bytes_out = 0;
        v->pkt_in = 0;
        v->pkt_out = 0;
        v->rate_limit = rate_limit;
        v->flags = flags;
        v->threshold = threshold;
        if (v->flags & get_iflink_mask)
                v->link = dev_get_iflink(dev);
        else
                v->link = dev->ifindex;
}
EXPORT_SYMBOL(vif_device_init);

struct mr_table *
mr_table_alloc(struct net *net, u32 id,
               struct mr_table_ops *ops,
               void (*expire_func)(struct timer_list *t),
               void (*table_set)(struct mr_table *mrt,
                                 struct net *net))
{
        struct mr_table *mrt;
        int err;

        mrt = kzalloc(sizeof(*mrt), GFP_KERNEL);
        if (!mrt)
                return ERR_PTR(-ENOMEM);
        mrt->id = id;
        write_pnet(&mrt->net, net);

        mrt->ops = *ops;
        err = rhltable_init(&mrt->mfc_hash, mrt->ops.rht_params);
        if (err) {
                kfree(mrt);
                return ERR_PTR(err);
        }
        INIT_LIST_HEAD(&mrt->mfc_cache_list);
        INIT_LIST_HEAD(&mrt->mfc_unres_queue);

        timer_setup(&mrt->ipmr_expire_timer, expire_func, 0);

        mrt->mroute_reg_vif_num = -1;
        table_set(mrt, net);
        return mrt;
}
EXPORT_SYMBOL(mr_table_alloc);

void *mr_mfc_find_parent(struct mr_table *mrt, void *hasharg, int parent)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (parent == -1 || parent == c->mfc_parent)
                        return c;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_find_parent);

void *mr_mfc_find_any_parent(struct mr_table *mrt, int vifi)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, mrt->ops.cmparg_any,
                               *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_find_any_parent);

void *mr_mfc_find_any(struct mr_table *mrt, int vifi, void *hasharg)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c, *proxy;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode) {
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

                /* It's ok if the vifi is part of the static tree */
                proxy = mr_mfc_find_any_parent(mrt, c->mfc_parent);
                if (proxy && proxy->mfc_un.res.ttls[vifi] < 255)
                        return c;
        }

        return mr_mfc_find_any_parent(mrt, vifi);
}
EXPORT_SYMBOL(mr_mfc_find_any);

#ifdef CONFIG_PROC_FS
void *mr_vif_seq_idx(struct net *net, struct mr_vif_iter *iter, loff_t pos)
{
        struct mr_table *mrt = iter->mrt;

        for (iter->ct = 0; iter->ct < mrt->maxvif; ++iter->ct) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                if (pos-- == 0)
                        return &mrt->vif_table[iter->ct];
        }
        return NULL;
}
EXPORT_SYMBOL(mr_vif_seq_idx);

void *mr_vif_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct mr_vif_iter *iter = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = iter->mrt;

        ++*pos;
        if (v == SEQ_START_TOKEN)
                return mr_vif_seq_idx(net, iter, 0);

        while (++iter->ct < mrt->maxvif) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                return &mrt->vif_table[iter->ct];
        }
        return NULL;
}
EXPORT_SYMBOL(mr_vif_seq_next);

void *mr_mfc_seq_idx(struct net *net,
                     struct mr_mfc_iter *it, loff_t pos)
{
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *mfc;

        rcu_read_lock();
        it->cache = &mrt->mfc_cache_list;
        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list)
                if (pos-- == 0)
                        return mfc;
        rcu_read_unlock();

        spin_lock_bh(it->lock);
        it->cache = &mrt->mfc_unres_queue;
        list_for_each_entry(mfc, it->cache, list)
                if (pos-- == 0)
                        return mfc;
        spin_unlock_bh(it->lock);

        it->cache = NULL;
        return NULL;
}
EXPORT_SYMBOL(mr_mfc_seq_idx);

void *mr_mfc_seq_next(struct seq_file *seq, void *v,
                      loff_t *pos)
{
        struct mr_mfc_iter *it = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *c = v;

        ++*pos;

        if (v == SEQ_START_TOKEN)
                return mr_mfc_seq_idx(net, seq->private, 0);

        if (c->list.next != it->cache)
                return list_entry(c->list.next, struct mr_mfc, list);

        if (it->cache == &mrt->mfc_unres_queue)
                goto end_of_list;

        /* exhausted cache_array, show unresolved */
        rcu_read_unlock();
        it->cache = &mrt->mfc_unres_queue;

        spin_lock_bh(it->lock);
        if (!list_empty(it->cache))
                return list_first_entry(it->cache, struct mr_mfc, list);

end_of_list:
        spin_unlock_bh(it->lock);
        it->cache = NULL;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_seq_next);
#endif

int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb,
                   struct mr_mfc *c, struct rtmsg *rtm)
{
        struct net_device *vif_dev;
        struct rta_mfc_stats mfcs;
        struct nlattr *mp_attr;
        struct rtnexthop *nhp;
        unsigned long lastuse;
        int ct;

        /* If cache is unresolved, don't try to parse IIF and OIF */
        if (c->mfc_parent >= MAXVIFS) {
                rtm->rtm_flags |= RTNH_F_UNRESOLVED;
                return -ENOENT;
        }

        rcu_read_lock();
        vif_dev = rcu_dereference(mrt->vif_table[c->mfc_parent].dev);
        if (vif_dev && nla_put_u32(skb, RTA_IIF, vif_dev->ifindex) < 0) {
                rcu_read_unlock();
                return -EMSGSIZE;
        }
        rcu_read_unlock();

        if (c->mfc_flags & MFC_OFFLOAD)
                rtm->rtm_flags |= RTNH_F_OFFLOAD;

        mp_attr = nla_nest_start_noflag(skb, RTA_MULTIPATH);
        if (!mp_attr)
                return -EMSGSIZE;

        rcu_read_lock();
        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                struct vif_device *vif = &mrt->vif_table[ct];

                vif_dev = rcu_dereference(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255) {

                        nhp = nla_reserve_nohdr(skb, sizeof(*nhp));
                        if (!nhp) {
                                rcu_read_unlock();
                                nla_nest_cancel(skb, mp_attr);
                                return -EMSGSIZE;
                        }

                        nhp->rtnh_flags = 0;
                        nhp->rtnh_hops = c->mfc_un.res.ttls[ct];
                        nhp->rtnh_ifindex = vif_dev->ifindex;
                        nhp->rtnh_len = sizeof(*nhp);
                }
        }
        rcu_read_unlock();

        nla_nest_end(skb, mp_attr);

        lastuse = READ_ONCE(c->mfc_un.res.lastuse);
        lastuse = time_after_eq(jiffies, lastuse) ? jiffies - lastuse : 0;

        mfcs.mfcs_packets = atomic_long_read(&c->mfc_un.res.pkt);
        mfcs.mfcs_bytes = atomic_long_read(&c->mfc_un.res.bytes);
        mfcs.mfcs_wrong_if = atomic_long_read(&c->mfc_un.res.wrong_if);
        if (nla_put_64bit(skb, RTA_MFC_STATS, sizeof(mfcs), &mfcs, RTA_PAD) ||
            nla_put_u64_64bit(skb, RTA_EXPIRES, jiffies_to_clock_t(lastuse),
                              RTA_PAD))
                return -EMSGSIZE;

        rtm->rtm_type = RTN_MULTICAST;
        return 1;
}
EXPORT_SYMBOL(mr_fill_mroute);

static bool mr_mfc_uses_dev(const struct mr_table *mrt,
                            const struct mr_mfc *c,
                            const struct net_device *dev)
{
        int ct;

        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                const struct net_device *vif_dev;
                const struct vif_device *vif;

                vif = &mrt->vif_table[ct];
                vif_dev = rcu_access_pointer(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255 &&
                    vif_dev == dev)
                        return true;
        }
        return false;
}

int mr_table_dump(struct mr_table *mrt, struct sk_buff *skb,
                  struct netlink_callback *cb,
                  int (*fill)(struct mr_table *mrt, struct sk_buff *skb,
                              u32 portid, u32 seq, struct mr_mfc *c,
                              int cmd, int flags),
                  spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int e = 0, s_e = cb->args[1];
        unsigned int flags = NLM_F_MULTI;
        struct mr_mfc *mfc;
        int err;

        if (filter->filter_set)
                flags |= NLM_F_DUMP_FILTERED;

        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list,
                                lockdep_rtnl_is_held()) {
                if (e < s_e)
                        goto next_entry;
                if (filter->dev &&
                    !mr_mfc_uses_dev(mrt, mfc, filter->dev))
                        goto next_entry;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0)
                        goto out;
next_entry:
                e++;
        }

        spin_lock_bh(lock);
        list_for_each_entry(mfc, &mrt->mfc_unres_queue, list) {
                if (e < s_e)
                        goto next_entry2;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0) {
                        spin_unlock_bh(lock);
                        goto out;
                }
next_entry2:
                e++;
        }
        spin_unlock_bh(lock);
        err = 0;
out:
        cb->args[1] = e;
        return err;
}
EXPORT_SYMBOL(mr_table_dump);

int mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb,
                     struct mr_table *(*iter)(struct net *net,
                                              struct mr_table *mrt),
                     int (*fill)(struct mr_table *mrt,
                                 struct sk_buff *skb,
                                 u32 portid, u32 seq, struct mr_mfc *c,
                                 int cmd, int flags),
                     spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int t = 0, s_t = cb->args[0];
        struct net *net = sock_net(skb->sk);
        struct mr_table *mrt;
        int err;

        /* multicast does not track protocol or have route type other
         * than RTN_MULTICAST
         */
        if (filter->filter_set) {
                if (filter->protocol || filter->flags ||
                    (filter->rt_type && filter->rt_type != RTN_MULTICAST))
                        return skb->len;
        }

        rcu_read_lock();
        for (mrt = iter(net, NULL); mrt; mrt = iter(net, mrt)) {
                if (t < s_t)
                        goto next_table;

                err = mr_table_dump(mrt, skb, cb, fill, lock, filter);
                if (err < 0)
                        break;
                cb->args[1] = 0;
next_table:
                t++;
        }
        rcu_read_unlock();

        cb->args[0] = t;

        return skb->len;
}
EXPORT_SYMBOL(mr_rtm_dumproute);

int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family,
            int (*rules_dump)(struct net *net,
                              struct notifier_block *nb,
                              struct netlink_ext_ack *extack),
            struct mr_table *(*mr_iter)(struct net *net,
                                        struct mr_table *mrt),
            struct netlink_ext_ack *extack)
{
        struct mr_table *mrt;
        int err;

        err = rules_dump(net, nb, extack);
        if (err)
                return err;

        for (mrt = mr_iter(net, NULL); mrt; mrt = mr_iter(net, mrt)) {
                struct vif_device *v = &mrt->vif_table[0];
                struct net_device *vif_dev;
                struct mr_mfc *mfc;
                int vifi;

                /* Notifiy on table VIF entries */
                rcu_read_lock();
                for (vifi = 0; vifi < mrt->maxvif; vifi++, v++) {
                        vif_dev = rcu_dereference(v->dev);
                        if (!vif_dev)
                                continue;

                        err = mr_call_vif_notifier(nb, family,
                                                   FIB_EVENT_VIF_ADD, v,
                                                   vif_dev, vifi,
                                                   mrt->id, extack);
                        if (err)
                                break;
                }
                rcu_read_unlock();

                if (err)
                        return err;

                /* Notify on table MFC entries */
                list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list) {
                        err = mr_call_mfc_notifier(nb, family,
                                                   FIB_EVENT_ENTRY_ADD,
                                                   mfc, mrt->id, extack);
                        if (err)
                                return err;
                }
        }

        return 0;
}
EXPORT_SYMBOL(mr_dump);























































    1 




    4 
    4 
    3 
    3 




    4 
    4 
    4 
    4 




    2 
    2 
    1 




    4 
    4 
    3 
    3 














































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue Code for assembler optimized version of Blowfish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <crypto/algapi.h>
#include <crypto/blowfish.h>
#include <crypto/internal/skcipher.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

#include "ecb_cbc_helpers.h"

/* regular block cipher functions */
asmlinkage void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);

/* 4-way parallel cipher functions */
asmlinkage void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                      const u8 *src);
asmlinkage void __blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                        const u8 *src, bool cbc);

static inline void blowfish_dec_ecb_4way(struct bf_ctx *ctx, u8 *dst,
                                             const u8 *src)
{
        return __blowfish_dec_blk_4way(ctx, dst, src, false);
}

static inline void blowfish_dec_cbc_4way(struct bf_ctx *ctx, u8 *dst,
                                             const u8 *src)
{
        return __blowfish_dec_blk_4way(ctx, dst, src, true);
}

static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
}

static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm,
                                    const u8 *key, unsigned int keylen)
{
        return blowfish_setkey(&tfm->base, key, keylen);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        ECB_WALK_START(req, BF_BLOCK_SIZE, -1);
        ECB_BLOCK(4, blowfish_enc_blk_4way);
        ECB_BLOCK(1, blowfish_enc_blk);
        ECB_WALK_END();
}

static int ecb_decrypt(struct skcipher_request *req)
{
        ECB_WALK_START(req, BF_BLOCK_SIZE, -1);
        ECB_BLOCK(4, blowfish_dec_ecb_4way);
        ECB_BLOCK(1, blowfish_dec_blk);
        ECB_WALK_END();
}

static int cbc_encrypt(struct skcipher_request *req)
{
        CBC_WALK_START(req, BF_BLOCK_SIZE, -1);
        CBC_ENC_BLOCK(blowfish_enc_blk);
        CBC_WALK_END();
}

static int cbc_decrypt(struct skcipher_request *req)
{
        CBC_WALK_START(req, BF_BLOCK_SIZE, -1);
        CBC_DEC_BLOCK(4, blowfish_dec_cbc_4way);
        CBC_DEC_BLOCK(1, blowfish_dec_blk);
        CBC_WALK_END();
}

static struct crypto_alg bf_cipher_alg = {
        .cra_name                = "blowfish",
        .cra_driver_name        = "blowfish-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = BF_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct bf_ctx),
        .cra_module                = THIS_MODULE,
        .cra_u = {
                .cipher = {
                        .cia_min_keysize        = BF_MIN_KEY_SIZE,
                        .cia_max_keysize        = BF_MAX_KEY_SIZE,
                        .cia_setkey                = blowfish_setkey,
                        .cia_encrypt                = blowfish_encrypt,
                        .cia_decrypt                = blowfish_decrypt,
                }
        }
};

static struct skcipher_alg bf_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(blowfish)",
                .base.cra_driver_name        = "ecb-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(blowfish)",
                .base.cra_driver_name        = "cbc-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .ivsize                        = BF_BLOCK_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        },
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, blowfish-x86_64 is slower than generic C
                 * implementation because use of 64bit rotates (which are really
                 * slow on P4). Therefore blacklist P4s.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init blowfish_init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "blowfish-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "blowfish-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&bf_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(bf_skcipher_algs,
                                        ARRAY_SIZE(bf_skcipher_algs));
        if (err)
                crypto_unregister_alg(&bf_cipher_alg);

        return err;
}

static void __exit blowfish_fini(void)
{
        crypto_unregister_alg(&bf_cipher_alg);
        crypto_unregister_skciphers(bf_skcipher_algs,
                                    ARRAY_SIZE(bf_skcipher_algs));
}

module_init(blowfish_init);
module_exit(blowfish_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("blowfish");
MODULE_ALIAS_CRYPTO("blowfish-asm");









































































































































    1 
    1 
































































































    3 












































































































































































































































































































































































































    1 


    1 
































    3 








    3 




    3 











    3 










    3 






























































































































































































































































































































































































































    1 














































































    1 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2020 Google Corporation
 */

#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>
#include <net/bluetooth/mgmt.h>

#include "mgmt_util.h"
#include "msft.h"

#define MSFT_RSSI_THRESHOLD_VALUE_MIN                -127
#define MSFT_RSSI_THRESHOLD_VALUE_MAX                20
#define MSFT_RSSI_LOW_TIMEOUT_MAX                0x3C

#define MSFT_OP_READ_SUPPORTED_FEATURES                0x00
struct msft_cp_read_supported_features {
        __u8   sub_opcode;
} __packed;

struct msft_rp_read_supported_features {
        __u8   status;
        __u8   sub_opcode;
        __le64 features;
        __u8   evt_prefix_len;
        __u8   evt_prefix[];
} __packed;

#define MSFT_OP_LE_MONITOR_ADVERTISEMENT        0x03
#define MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN        0x01
struct msft_le_monitor_advertisement_pattern {
        __u8 length;
        __u8 data_type;
        __u8 start_byte;
        __u8 pattern[];
};

struct msft_le_monitor_advertisement_pattern_data {
        __u8 count;
        __u8 data[];
};

struct msft_cp_le_monitor_advertisement {
        __u8 sub_opcode;
        __s8 rssi_high;
        __s8 rssi_low;
        __u8 rssi_low_interval;
        __u8 rssi_sampling_period;
        __u8 cond_type;
        __u8 data[];
} __packed;

struct msft_rp_le_monitor_advertisement {
        __u8 status;
        __u8 sub_opcode;
        __u8 handle;
} __packed;

#define MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT        0x04
struct msft_cp_le_cancel_monitor_advertisement {
        __u8 sub_opcode;
        __u8 handle;
} __packed;

struct msft_rp_le_cancel_monitor_advertisement {
        __u8 status;
        __u8 sub_opcode;
} __packed;

#define MSFT_OP_LE_SET_ADVERTISEMENT_FILTER_ENABLE        0x05
struct msft_cp_le_set_advertisement_filter_enable {
        __u8 sub_opcode;
        __u8 enable;
} __packed;

struct msft_rp_le_set_advertisement_filter_enable {
        __u8 status;
        __u8 sub_opcode;
} __packed;

#define MSFT_EV_LE_MONITOR_DEVICE        0x02
struct msft_ev_le_monitor_device {
        __u8     addr_type;
        bdaddr_t bdaddr;
        __u8     monitor_handle;
        __u8     monitor_state;
} __packed;

struct msft_monitor_advertisement_handle_data {
        __u8  msft_handle;
        __u16 mgmt_handle;
        __s8 rssi_high;
        __s8 rssi_low;
        __u8 rssi_low_interval;
        __u8 rssi_sampling_period;
        __u8 cond_type;
        struct list_head list;
};

enum monitor_addr_filter_state {
        AF_STATE_IDLE,
        AF_STATE_ADDING,
        AF_STATE_ADDED,
        AF_STATE_REMOVING,
};

#define MSFT_MONITOR_ADVERTISEMENT_TYPE_ADDR        0x04
struct msft_monitor_addr_filter_data {
        __u8     msft_handle;
        __u8     pattern_handle; /* address filters pertain to */
        __u16    mgmt_handle;
        int      state;
        __s8     rssi_high;
        __s8     rssi_low;
        __u8     rssi_low_interval;
        __u8     rssi_sampling_period;
        __u8     addr_type;
        bdaddr_t bdaddr;
        struct list_head list;
};

struct msft_data {
        __u64 features;
        __u8  evt_prefix_len;
        __u8  *evt_prefix;
        struct list_head handle_map;
        struct list_head address_filters;
        __u8 resuming;
        __u8 suspending;
        __u8 filter_enabled;
        /* To synchronize add/remove address filter and monitor device event.*/
        struct mutex filter_lock;
};

bool msft_monitor_supported(struct hci_dev *hdev)
{
        return !!(msft_get_features(hdev) & MSFT_FEATURE_MASK_LE_ADV_MONITOR);
}

static bool read_supported_features(struct hci_dev *hdev,
                                    struct msft_data *msft)
{
        struct msft_cp_read_supported_features cp;
        struct msft_rp_read_supported_features *rp;
        struct sk_buff *skb;

        cp.sub_opcode = MSFT_OP_READ_SUPPORTED_FEATURES;

        skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp,
                             HCI_CMD_TIMEOUT);
        if (IS_ERR(skb)) {
                bt_dev_err(hdev, "Failed to read MSFT supported features (%ld)",
                           PTR_ERR(skb));
                return false;
        }

        if (skb->len < sizeof(*rp)) {
                bt_dev_err(hdev, "MSFT supported features length mismatch");
                goto failed;
        }

        rp = (struct msft_rp_read_supported_features *)skb->data;

        if (rp->sub_opcode != MSFT_OP_READ_SUPPORTED_FEATURES)
                goto failed;

        if (rp->evt_prefix_len > 0) {
                msft->evt_prefix = kmemdup(rp->evt_prefix, rp->evt_prefix_len,
                                           GFP_KERNEL);
                if (!msft->evt_prefix)
                        goto failed;
        }

        msft->evt_prefix_len = rp->evt_prefix_len;
        msft->features = __le64_to_cpu(rp->features);

        if (msft->features & MSFT_FEATURE_MASK_CURVE_VALIDITY)
                hdev->msft_curve_validity = true;

        kfree_skb(skb);
        return true;

failed:
        kfree_skb(skb);
        return false;
}

/* is_mgmt = true matches the handle exposed to userspace via mgmt.
 * is_mgmt = false matches the handle used by the msft controller.
 * This function requires the caller holds hdev->lock
 */
static struct msft_monitor_advertisement_handle_data *msft_find_handle_data
                                (struct hci_dev *hdev, u16 handle, bool is_mgmt)
{
        struct msft_monitor_advertisement_handle_data *entry;
        struct msft_data *msft = hdev->msft_data;

        list_for_each_entry(entry, &msft->handle_map, list) {
                if (is_mgmt && entry->mgmt_handle == handle)
                        return entry;
                if (!is_mgmt && entry->msft_handle == handle)
                        return entry;
        }

        return NULL;
}

/* This function requires the caller holds msft->filter_lock */
static struct msft_monitor_addr_filter_data *msft_find_address_data
                        (struct hci_dev *hdev, u8 addr_type, bdaddr_t *addr,
                         u8 pattern_handle)
{
        struct msft_monitor_addr_filter_data *entry;
        struct msft_data *msft = hdev->msft_data;

        list_for_each_entry(entry, &msft->address_filters, list) {
                if (entry->pattern_handle == pattern_handle &&
                    addr_type == entry->addr_type &&
                    !bacmp(addr, &entry->bdaddr))
                        return entry;
        }

        return NULL;
}

/* This function requires the caller holds hdev->lock */
static int msft_monitor_device_del(struct hci_dev *hdev, __u16 mgmt_handle,
                                   bdaddr_t *bdaddr, __u8 addr_type,
                                   bool notify)
{
        struct monitored_device *dev, *tmp;
        int count = 0;

        list_for_each_entry_safe(dev, tmp, &hdev->monitored_devices, list) {
                /* mgmt_handle == 0 indicates remove all devices, whereas,
                 * bdaddr == NULL indicates remove all devices matching the
                 * mgmt_handle.
                 */
                if ((!mgmt_handle || dev->handle == mgmt_handle) &&
                    (!bdaddr || (!bacmp(bdaddr, &dev->bdaddr) &&
                                 addr_type == dev->addr_type))) {
                        if (notify && dev->notified) {
                                mgmt_adv_monitor_device_lost(hdev, dev->handle,
                                                             &dev->bdaddr,
                                                             dev->addr_type);
                        }

                        list_del(&dev->list);
                        kfree(dev);
                        count++;
                }
        }

        return count;
}

static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode,
                                            struct adv_monitor *monitor,
                                            struct sk_buff *skb)
{
        struct msft_rp_le_monitor_advertisement *rp;
        struct msft_monitor_advertisement_handle_data *handle_data;
        struct msft_data *msft = hdev->msft_data;
        int status = 0;

        hci_dev_lock(hdev);

        rp = (struct msft_rp_le_monitor_advertisement *)skb->data;
        if (skb->len < sizeof(*rp)) {
                status = HCI_ERROR_UNSPECIFIED;
                goto unlock;
        }

        status = rp->status;
        if (status)
                goto unlock;

        handle_data = kmalloc(sizeof(*handle_data), GFP_KERNEL);
        if (!handle_data) {
                status = HCI_ERROR_UNSPECIFIED;
                goto unlock;
        }

        handle_data->mgmt_handle = monitor->handle;
        handle_data->msft_handle = rp->handle;
        handle_data->cond_type   = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN;
        INIT_LIST_HEAD(&handle_data->list);
        list_add(&handle_data->list, &msft->handle_map);

        monitor->state = ADV_MONITOR_STATE_OFFLOADED;

unlock:
        if (status)
                hci_free_adv_monitor(hdev, monitor);

        hci_dev_unlock(hdev);

        return status;
}

/* This function requires the caller holds hci_req_sync_lock */
static void msft_remove_addr_filters_sync(struct hci_dev *hdev, u8 handle)
{
        struct msft_monitor_addr_filter_data *address_filter, *n;
        struct msft_cp_le_cancel_monitor_advertisement cp;
        struct msft_data *msft = hdev->msft_data;
        struct list_head head;
        struct sk_buff *skb;

        INIT_LIST_HEAD(&head);

        /* Cancel all corresponding address monitors */
        mutex_lock(&msft->filter_lock);

        list_for_each_entry_safe(address_filter, n, &msft->address_filters,
                                 list) {
                if (address_filter->pattern_handle != handle)
                        continue;

                list_del(&address_filter->list);

                /* Keep the address filter and let
                 * msft_add_address_filter_sync() remove and free the address
                 * filter.
                 */
                if (address_filter->state == AF_STATE_ADDING) {
                        address_filter->state = AF_STATE_REMOVING;
                        continue;
                }

                /* Keep the address filter and let
                 * msft_cancel_address_filter_sync() remove and free the address
                 * filter
                 */
                if (address_filter->state == AF_STATE_REMOVING)
                        continue;

                list_add_tail(&address_filter->list, &head);
        }

        mutex_unlock(&msft->filter_lock);

        list_for_each_entry_safe(address_filter, n, &head, list) {
                list_del(&address_filter->list);

                cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT;
                cp.handle = address_filter->msft_handle;

                skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp,
                                     HCI_CMD_TIMEOUT);
                if (IS_ERR(skb)) {
                        kfree(address_filter);
                        continue;
                }

                kfree_skb(skb);

                bt_dev_dbg(hdev, "MSFT: Canceled device %pMR address filter",
                           &address_filter->bdaddr);

                kfree(address_filter);
        }
}

static int msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev,
                                                   u16 opcode,
                                                   struct adv_monitor *monitor,
                                                   struct sk_buff *skb)
{
        struct msft_rp_le_cancel_monitor_advertisement *rp;
        struct msft_monitor_advertisement_handle_data *handle_data;
        struct msft_data *msft = hdev->msft_data;
        int status = 0;
        u8 msft_handle;

        rp = (struct msft_rp_le_cancel_monitor_advertisement *)skb->data;
        if (skb->len < sizeof(*rp)) {
                status = HCI_ERROR_UNSPECIFIED;
                goto done;
        }

        status = rp->status;
        if (status)
                goto done;

        hci_dev_lock(hdev);

        handle_data = msft_find_handle_data(hdev, monitor->handle, true);

        if (handle_data) {
                if (monitor->state == ADV_MONITOR_STATE_OFFLOADED)
                        monitor->state = ADV_MONITOR_STATE_REGISTERED;

                /* Do not free the monitor if it is being removed due to
                 * suspend. It will be re-monitored on resume.
                 */
                if (!msft->suspending) {
                        hci_free_adv_monitor(hdev, monitor);

                        /* Clear any monitored devices by this Adv Monitor */
                        msft_monitor_device_del(hdev, handle_data->mgmt_handle,
                                                NULL, 0, false);
                }

                msft_handle = handle_data->msft_handle;

                list_del(&handle_data->list);
                kfree(handle_data);

                hci_dev_unlock(hdev);

                msft_remove_addr_filters_sync(hdev, msft_handle);
        } else {
                hci_dev_unlock(hdev);
        }

done:
        return status;
}

/* This function requires the caller holds hci_req_sync_lock */
static int msft_remove_monitor_sync(struct hci_dev *hdev,
                                    struct adv_monitor *monitor)
{
        struct msft_cp_le_cancel_monitor_advertisement cp;
        struct msft_monitor_advertisement_handle_data *handle_data;
        struct sk_buff *skb;

        handle_data = msft_find_handle_data(hdev, monitor->handle, true);

        /* If no matched handle, just remove without telling controller */
        if (!handle_data)
                return -ENOENT;

        cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT;
        cp.handle = handle_data->msft_handle;

        skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp,
                             HCI_CMD_TIMEOUT);
        if (IS_ERR(skb))
                return PTR_ERR(skb);

        return msft_le_cancel_monitor_advertisement_cb(hdev, hdev->msft_opcode,
                                                       monitor, skb);
}

/* This function requires the caller holds hci_req_sync_lock */
int msft_suspend_sync(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;
        struct adv_monitor *monitor;
        int handle = 0;

        if (!msft || !msft_monitor_supported(hdev))
                return 0;

        msft->suspending = true;

        while (1) {
                monitor = idr_get_next(&hdev->adv_monitors_idr, &handle);
                if (!monitor)
                        break;

                msft_remove_monitor_sync(hdev, monitor);

                handle++;
        }

        /* All monitors have been removed */
        msft->suspending = false;

        return 0;
}

static bool msft_monitor_rssi_valid(struct adv_monitor *monitor)
{
        struct adv_rssi_thresholds *r = &monitor->rssi;

        if (r->high_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN ||
            r->high_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX ||
            r->low_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN ||
            r->low_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX)
                return false;

        /* High_threshold_timeout is not supported,
         * once high_threshold is reached, events are immediately reported.
         */
        if (r->high_threshold_timeout != 0)
                return false;

        if (r->low_threshold_timeout > MSFT_RSSI_LOW_TIMEOUT_MAX)
                return false;

        /* Sampling period from 0x00 to 0xFF are all allowed */
        return true;
}

static bool msft_monitor_pattern_valid(struct adv_monitor *monitor)
{
        return msft_monitor_rssi_valid(monitor);
        /* No additional check needed for pattern-based monitor */
}

static int msft_add_monitor_sync(struct hci_dev *hdev,
                                 struct adv_monitor *monitor)
{
        struct msft_cp_le_monitor_advertisement *cp;
        struct msft_le_monitor_advertisement_pattern_data *pattern_data;
        struct msft_monitor_advertisement_handle_data *handle_data;
        struct msft_le_monitor_advertisement_pattern *pattern;
        struct adv_pattern *entry;
        size_t total_size = sizeof(*cp) + sizeof(*pattern_data);
        ptrdiff_t offset = 0;
        u8 pattern_count = 0;
        struct sk_buff *skb;
        int err;

        if (!msft_monitor_pattern_valid(monitor))
                return -EINVAL;

        list_for_each_entry(entry, &monitor->patterns, list) {
                pattern_count++;
                total_size += sizeof(*pattern) + entry->length;
        }

        cp = kmalloc(total_size, GFP_KERNEL);
        if (!cp)
                return -ENOMEM;

        cp->sub_opcode = MSFT_OP_LE_MONITOR_ADVERTISEMENT;
        cp->rssi_high = monitor->rssi.high_threshold;
        cp->rssi_low = monitor->rssi.low_threshold;
        cp->rssi_low_interval = (u8)monitor->rssi.low_threshold_timeout;
        cp->rssi_sampling_period = monitor->rssi.sampling_period;

        cp->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN;

        pattern_data = (void *)cp->data;
        pattern_data->count = pattern_count;

        list_for_each_entry(entry, &monitor->patterns, list) {
                pattern = (void *)(pattern_data->data + offset);
                /* the length also includes data_type and offset */
                pattern->length = entry->length + 2;
                pattern->data_type = entry->ad_type;
                pattern->start_byte = entry->offset;
                memcpy(pattern->pattern, entry->value, entry->length);
                offset += sizeof(*pattern) + entry->length;
        }

        skb = __hci_cmd_sync(hdev, hdev->msft_opcode, total_size, cp,
                             HCI_CMD_TIMEOUT);

        if (IS_ERR(skb)) {
                err = PTR_ERR(skb);
                goto out_free;
        }

        err = msft_le_monitor_advertisement_cb(hdev, hdev->msft_opcode,
                                               monitor, skb);
        if (err)
                goto out_free;

        handle_data = msft_find_handle_data(hdev, monitor->handle, true);
        if (!handle_data) {
                err = -ENODATA;
                goto out_free;
        }

        handle_data->rssi_high        = cp->rssi_high;
        handle_data->rssi_low        = cp->rssi_low;
        handle_data->rssi_low_interval          = cp->rssi_low_interval;
        handle_data->rssi_sampling_period = cp->rssi_sampling_period;

out_free:
        kfree(cp);
        return err;
}

/* This function requires the caller holds hci_req_sync_lock */
static void reregister_monitor(struct hci_dev *hdev)
{
        struct adv_monitor *monitor;
        struct msft_data *msft = hdev->msft_data;
        int handle = 0;

        if (!msft)
                return;

        msft->resuming = true;

        while (1) {
                monitor = idr_get_next(&hdev->adv_monitors_idr, &handle);
                if (!monitor)
                        break;

                msft_add_monitor_sync(hdev, monitor);

                handle++;
        }

        /* All monitors have been reregistered */
        msft->resuming = false;
}

/* This function requires the caller holds hci_req_sync_lock */
int msft_resume_sync(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;

        if (!msft || !msft_monitor_supported(hdev))
                return 0;

        hci_dev_lock(hdev);

        /* Clear already tracked devices on resume. Once the monitors are
         * reregistered, devices in range will be found again after resume.
         */
        hdev->advmon_pend_notify = false;
        msft_monitor_device_del(hdev, 0, NULL, 0, true);

        hci_dev_unlock(hdev);

        reregister_monitor(hdev);

        return 0;
}

/* This function requires the caller holds hci_req_sync_lock */
void msft_do_open(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;

        if (hdev->msft_opcode == HCI_OP_NOP)
                return;

        if (!msft) {
                bt_dev_err(hdev, "MSFT extension not registered");
                return;
        }

        bt_dev_dbg(hdev, "Initialize MSFT extension");

        /* Reset existing MSFT data before re-reading */
        kfree(msft->evt_prefix);
        msft->evt_prefix = NULL;
        msft->evt_prefix_len = 0;
        msft->features = 0;

        if (!read_supported_features(hdev, msft)) {
                hdev->msft_data = NULL;
                kfree(msft);
                return;
        }

        if (msft_monitor_supported(hdev)) {
                msft->resuming = true;
                msft_set_filter_enable(hdev, true);
                /* Monitors get removed on power off, so we need to explicitly
                 * tell the controller to re-monitor.
                 */
                reregister_monitor(hdev);
        }
}

void msft_do_close(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;
        struct msft_monitor_advertisement_handle_data *handle_data, *tmp;
        struct msft_monitor_addr_filter_data *address_filter, *n;
        struct adv_monitor *monitor;

        if (!msft)
                return;

        bt_dev_dbg(hdev, "Cleanup of MSFT extension");

        /* The controller will silently remove all monitors on power off.
         * Therefore, remove handle_data mapping and reset monitor state.
         */
        list_for_each_entry_safe(handle_data, tmp, &msft->handle_map, list) {
                monitor = idr_find(&hdev->adv_monitors_idr,
                                   handle_data->mgmt_handle);

                if (monitor && monitor->state == ADV_MONITOR_STATE_OFFLOADED)
                        monitor->state = ADV_MONITOR_STATE_REGISTERED;

                list_del(&handle_data->list);
                kfree(handle_data);
        }

        mutex_lock(&msft->filter_lock);
        list_for_each_entry_safe(address_filter, n, &msft->address_filters,
                                 list) {
                list_del(&address_filter->list);
                kfree(address_filter);
        }
        mutex_unlock(&msft->filter_lock);

        hci_dev_lock(hdev);

        /* Clear any devices that are being monitored and notify device lost */
        hdev->advmon_pend_notify = false;
        msft_monitor_device_del(hdev, 0, NULL, 0, true);

        hci_dev_unlock(hdev);
}

static int msft_cancel_address_filter_sync(struct hci_dev *hdev, void *data)
{
        struct msft_monitor_addr_filter_data *address_filter = data;
        struct msft_cp_le_cancel_monitor_advertisement cp;
        struct msft_data *msft = hdev->msft_data;
        struct sk_buff *skb;
        int err = 0;

        if (!msft) {
                bt_dev_err(hdev, "MSFT: msft data is freed");
                return -EINVAL;
        }

        /* The address filter has been removed by hci dev close */
        if (!test_bit(HCI_UP, &hdev->flags))
                return 0;

        mutex_lock(&msft->filter_lock);
        list_del(&address_filter->list);
        mutex_unlock(&msft->filter_lock);

        cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT;
        cp.handle = address_filter->msft_handle;

        skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp,
                             HCI_CMD_TIMEOUT);
        if (IS_ERR(skb)) {
                bt_dev_err(hdev, "MSFT: Failed to cancel address (%pMR) filter",
                           &address_filter->bdaddr);
                err = PTR_ERR(skb);
                goto done;
        }
        kfree_skb(skb);

        bt_dev_dbg(hdev, "MSFT: Canceled device %pMR address filter",
                   &address_filter->bdaddr);

done:
        kfree(address_filter);

        return err;
}

void msft_register(struct hci_dev *hdev)
{
        struct msft_data *msft = NULL;

        bt_dev_dbg(hdev, "Register MSFT extension");

        msft = kzalloc(sizeof(*msft), GFP_KERNEL);
        if (!msft) {
                bt_dev_err(hdev, "Failed to register MSFT extension");
                return;
        }

        INIT_LIST_HEAD(&msft->handle_map);
        INIT_LIST_HEAD(&msft->address_filters);
        hdev->msft_data = msft;
        mutex_init(&msft->filter_lock);
}

void msft_release(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;

        if (!msft)
                return;

        bt_dev_dbg(hdev, "Unregister MSFT extension");

        hdev->msft_data = NULL;

        kfree(msft->evt_prefix);
        mutex_destroy(&msft->filter_lock);
        kfree(msft);
}

/* This function requires the caller holds hdev->lock */
static void msft_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr,
                              __u8 addr_type, __u16 mgmt_handle)
{
        struct monitored_device *dev;

        dev = kmalloc(sizeof(*dev), GFP_KERNEL);
        if (!dev) {
                bt_dev_err(hdev, "MSFT vendor event %u: no memory",
                           MSFT_EV_LE_MONITOR_DEVICE);
                return;
        }

        bacpy(&dev->bdaddr, bdaddr);
        dev->addr_type = addr_type;
        dev->handle = mgmt_handle;
        dev->notified = false;

        INIT_LIST_HEAD(&dev->list);
        list_add(&dev->list, &hdev->monitored_devices);
        hdev->advmon_pend_notify = true;
}

/* This function requires the caller holds hdev->lock */
static void msft_device_lost(struct hci_dev *hdev, bdaddr_t *bdaddr,
                             __u8 addr_type, __u16 mgmt_handle)
{
        if (!msft_monitor_device_del(hdev, mgmt_handle, bdaddr, addr_type,
                                     true)) {
                bt_dev_err(hdev, "MSFT vendor event %u: dev %pMR not in list",
                           MSFT_EV_LE_MONITOR_DEVICE, bdaddr);
        }
}

static void *msft_skb_pull(struct hci_dev *hdev, struct sk_buff *skb,
                           u8 ev, size_t len)
{
        void *data;

        data = skb_pull_data(skb, len);
        if (!data)
                bt_dev_err(hdev, "Malformed MSFT vendor event: 0x%02x", ev);

        return data;
}

static int msft_add_address_filter_sync(struct hci_dev *hdev, void *data)
{
        struct msft_monitor_addr_filter_data *address_filter = data;
        struct msft_rp_le_monitor_advertisement *rp;
        struct msft_cp_le_monitor_advertisement *cp;
        struct msft_data *msft = hdev->msft_data;
        struct sk_buff *skb = NULL;
        bool remove = false;
        size_t size;

        if (!msft) {
                bt_dev_err(hdev, "MSFT: msft data is freed");
                return -EINVAL;
        }

        /* The address filter has been removed by hci dev close */
        if (!test_bit(HCI_UP, &hdev->flags))
                return -ENODEV;

        /* We are safe to use the address filter from now on.
         * msft_monitor_device_evt() wouldn't delete this filter because it's
         * not been added by now.
         * And all other functions that requiring hci_req_sync_lock wouldn't
         * touch this filter before this func completes because it's protected
         * by hci_req_sync_lock.
         */

        if (address_filter->state == AF_STATE_REMOVING) {
                mutex_lock(&msft->filter_lock);
                list_del(&address_filter->list);
                mutex_unlock(&msft->filter_lock);
                kfree(address_filter);
                return 0;
        }

        size = sizeof(*cp) +
               sizeof(address_filter->addr_type) +
               sizeof(address_filter->bdaddr);
        cp = kzalloc(size, GFP_KERNEL);
        if (!cp) {
                bt_dev_err(hdev, "MSFT: Alloc cmd param err");
                remove = true;
                goto done;
        }

        cp->sub_opcode           = MSFT_OP_LE_MONITOR_ADVERTISEMENT;
        cp->rssi_high                 = address_filter->rssi_high;
        cp->rssi_low                 = address_filter->rssi_low;
        cp->rssi_low_interval    = address_filter->rssi_low_interval;
        cp->rssi_sampling_period = address_filter->rssi_sampling_period;
        cp->cond_type            = MSFT_MONITOR_ADVERTISEMENT_TYPE_ADDR;
        cp->data[0]              = address_filter->addr_type;
        memcpy(&cp->data[1], &address_filter->bdaddr,
               sizeof(address_filter->bdaddr));

        skb = __hci_cmd_sync(hdev, hdev->msft_opcode, size, cp,
                             HCI_CMD_TIMEOUT);
        kfree(cp);

        if (IS_ERR(skb)) {
                bt_dev_err(hdev, "Failed to enable address %pMR filter",
                           &address_filter->bdaddr);
                skb = NULL;
                remove = true;
                goto done;
        }

        rp = skb_pull_data(skb, sizeof(*rp));
        if (!rp || rp->sub_opcode != MSFT_OP_LE_MONITOR_ADVERTISEMENT ||
            rp->status)
                remove = true;

done:
        mutex_lock(&msft->filter_lock);

        if (remove) {
                bt_dev_warn(hdev, "MSFT: Remove address (%pMR) filter",
                            &address_filter->bdaddr);
                list_del(&address_filter->list);
                kfree(address_filter);
        } else {
                address_filter->state = AF_STATE_ADDED;
                address_filter->msft_handle = rp->handle;
                bt_dev_dbg(hdev, "MSFT: Address %pMR filter enabled",
                           &address_filter->bdaddr);
        }
        mutex_unlock(&msft->filter_lock);

        kfree_skb(skb);

        return 0;
}

/* This function requires the caller holds msft->filter_lock */
static struct msft_monitor_addr_filter_data *msft_add_address_filter
                (struct hci_dev *hdev, u8 addr_type, bdaddr_t *bdaddr,
                 struct msft_monitor_advertisement_handle_data *handle_data)
{
        struct msft_monitor_addr_filter_data *address_filter = NULL;
        struct msft_data *msft = hdev->msft_data;
        int err;

        address_filter = kzalloc(sizeof(*address_filter), GFP_KERNEL);
        if (!address_filter)
                return NULL;

        address_filter->state             = AF_STATE_ADDING;
        address_filter->msft_handle       = 0xff;
        address_filter->pattern_handle    = handle_data->msft_handle;
        address_filter->mgmt_handle       = handle_data->mgmt_handle;
        address_filter->rssi_high         = handle_data->rssi_high;
        address_filter->rssi_low          = handle_data->rssi_low;
        address_filter->rssi_low_interval = handle_data->rssi_low_interval;
        address_filter->rssi_sampling_period = handle_data->rssi_sampling_period;
        address_filter->addr_type            = addr_type;
        bacpy(&address_filter->bdaddr, bdaddr);

        /* With the above AF_STATE_ADDING, duplicated address filter can be
         * avoided when receiving monitor device event (found/lost) frequently
         * for the same device.
         */
        list_add_tail(&address_filter->list, &msft->address_filters);

        err = hci_cmd_sync_queue(hdev, msft_add_address_filter_sync,
                                 address_filter, NULL);
        if (err < 0) {
                bt_dev_err(hdev, "MSFT: Add address %pMR filter err", bdaddr);
                list_del(&address_filter->list);
                kfree(address_filter);
                return NULL;
        }

        bt_dev_dbg(hdev, "MSFT: Add device %pMR address filter",
                   &address_filter->bdaddr);

        return address_filter;
}

/* This function requires the caller holds hdev->lock */
static void msft_monitor_device_evt(struct hci_dev *hdev, struct sk_buff *skb)
{
        struct msft_monitor_addr_filter_data *n, *address_filter = NULL;
        struct msft_ev_le_monitor_device *ev;
        struct msft_monitor_advertisement_handle_data *handle_data;
        struct msft_data *msft = hdev->msft_data;
        u16 mgmt_handle = 0xffff;
        u8 addr_type;

        ev = msft_skb_pull(hdev, skb, MSFT_EV_LE_MONITOR_DEVICE, sizeof(*ev));
        if (!ev)
                return;

        bt_dev_dbg(hdev,
                   "MSFT vendor event 0x%02x: handle 0x%04x state %d addr %pMR",
                   MSFT_EV_LE_MONITOR_DEVICE, ev->monitor_handle,
                   ev->monitor_state, &ev->bdaddr);

        handle_data = msft_find_handle_data(hdev, ev->monitor_handle, false);

        if (!hci_test_quirk(hdev, HCI_QUIRK_USE_MSFT_EXT_ADDRESS_FILTER)) {
                if (!handle_data)
                        return;
                mgmt_handle = handle_data->mgmt_handle;
                goto report_state;
        }

        if (handle_data) {
                /* Don't report any device found/lost event from pattern
                 * monitors. Pattern monitor always has its address filters for
                 * tracking devices.
                 */

                address_filter = msft_find_address_data(hdev, ev->addr_type,
                                                        &ev->bdaddr,
                                                        handle_data->msft_handle);
                if (address_filter)
                        return;

                if (ev->monitor_state && handle_data->cond_type ==
                                MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN)
                        msft_add_address_filter(hdev, ev->addr_type,
                                                &ev->bdaddr, handle_data);

                return;
        }

        /* This device event is not from pattern monitor.
         * Report it if there is a corresponding address_filter for it.
         */
        list_for_each_entry(n, &msft->address_filters, list) {
                if (n->state == AF_STATE_ADDED &&
                    n->msft_handle == ev->monitor_handle) {
                        mgmt_handle = n->mgmt_handle;
                        address_filter = n;
                        break;
                }
        }

        if (!address_filter) {
                bt_dev_warn(hdev, "MSFT: Unexpected device event %pMR, %u, %u",
                            &ev->bdaddr, ev->monitor_handle, ev->monitor_state);
                return;
        }

report_state:
        switch (ev->addr_type) {
        case ADDR_LE_DEV_PUBLIC:
                addr_type = BDADDR_LE_PUBLIC;
                break;

        case ADDR_LE_DEV_RANDOM:
                addr_type = BDADDR_LE_RANDOM;
                break;

        default:
                bt_dev_err(hdev,
                           "MSFT vendor event 0x%02x: unknown addr type 0x%02x",
                           MSFT_EV_LE_MONITOR_DEVICE, ev->addr_type);
                return;
        }

        if (ev->monitor_state) {
                msft_device_found(hdev, &ev->bdaddr, addr_type, mgmt_handle);
        } else {
                if (address_filter && address_filter->state == AF_STATE_ADDED) {
                        address_filter->state = AF_STATE_REMOVING;
                        hci_cmd_sync_queue(hdev,
                                           msft_cancel_address_filter_sync,
                                           address_filter,
                                           NULL);
                }
                msft_device_lost(hdev, &ev->bdaddr, addr_type, mgmt_handle);
        }
}

void msft_vendor_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb)
{
        struct msft_data *msft = hdev->msft_data;
        u8 *evt_prefix;
        u8 *evt;

        if (!msft)
                return;

        /* When the extension has defined an event prefix, check that it
         * matches, and otherwise just return.
         */
        if (msft->evt_prefix_len > 0) {
                evt_prefix = msft_skb_pull(hdev, skb, 0, msft->evt_prefix_len);
                if (!evt_prefix)
                        return;

                if (memcmp(evt_prefix, msft->evt_prefix, msft->evt_prefix_len))
                        return;
        }

        /* Every event starts at least with an event code and the rest of
         * the data is variable and depends on the event code.
         */
        if (skb->len < 1)
                return;

        evt = msft_skb_pull(hdev, skb, 0, sizeof(*evt));
        if (!evt)
                return;

        hci_dev_lock(hdev);

        switch (*evt) {
        case MSFT_EV_LE_MONITOR_DEVICE:
                mutex_lock(&msft->filter_lock);
                msft_monitor_device_evt(hdev, skb);
                mutex_unlock(&msft->filter_lock);
                break;

        default:
                bt_dev_dbg(hdev, "MSFT vendor event 0x%02x", *evt);
                break;
        }

        hci_dev_unlock(hdev);
}

__u64 msft_get_features(struct hci_dev *hdev)
{
        struct msft_data *msft = hdev->msft_data;

        return msft ? msft->features : 0;
}

static void msft_le_set_advertisement_filter_enable_cb(struct hci_dev *hdev,
                                                       void *user_data,
                                                       u8 status)
{
        struct msft_cp_le_set_advertisement_filter_enable *cp = user_data;
        struct msft_data *msft = hdev->msft_data;

        /* Error 0x0C would be returned if the filter enabled status is
         * already set to whatever we were trying to set.
         * Although the default state should be disabled, some controller set
         * the initial value to enabled. Because there is no way to know the
         * actual initial value before sending this command, here we also treat
         * error 0x0C as success.
         */
        if (status != 0x00 && status != 0x0C)
                return;

        hci_dev_lock(hdev);

        msft->filter_enabled = cp->enable;

        if (status == 0x0C)
                bt_dev_warn(hdev, "MSFT filter_enable is already %s",
                            cp->enable ? "on" : "off");

        hci_dev_unlock(hdev);
}

/* This function requires the caller holds hci_req_sync_lock */
int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor)
{
        struct msft_data *msft = hdev->msft_data;

        if (!msft)
                return -EOPNOTSUPP;

        if (msft->resuming || msft->suspending)
                return -EBUSY;

        return msft_add_monitor_sync(hdev, monitor);
}

/* This function requires the caller holds hci_req_sync_lock */
int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor)
{
        struct msft_data *msft = hdev->msft_data;

        if (!msft)
                return -EOPNOTSUPP;

        if (msft->resuming || msft->suspending)
                return -EBUSY;

        return msft_remove_monitor_sync(hdev, monitor);
}

int msft_set_filter_enable(struct hci_dev *hdev, bool enable)
{
        struct msft_cp_le_set_advertisement_filter_enable cp;
        struct msft_data *msft = hdev->msft_data;
        int err;

        if (!msft)
                return -EOPNOTSUPP;

        cp.sub_opcode = MSFT_OP_LE_SET_ADVERTISEMENT_FILTER_ENABLE;
        cp.enable = enable;
        err = __hci_cmd_sync_status(hdev, hdev->msft_opcode, sizeof(cp), &cp,
                                    HCI_CMD_TIMEOUT);

        msft_le_set_advertisement_filter_enable_cb(hdev, &cp, err);

        return 0;
}

bool msft_curve_validity(struct hci_dev *hdev)
{
        return hdev->msft_curve_validity;
}































































































































































































































































































 16999 



 16993 
 17004 

 17011 












 16998 



 16996 
 17010 


 17181 

 17183 






















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
// SPDX-License-Identifier: GPL-2.0-only
/*
 * AppArmor security module
 *
 * This file contains AppArmor network mediation
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2017 Canonical Ltd.
 */

#include "include/af_unix.h"
#include "include/apparmor.h"
#include "include/audit.h"
#include "include/cred.h"
#include "include/label.h"
#include "include/net.h"
#include "include/policy.h"
#include "include/secid.h"

#include "net_names.h"


struct aa_sfs_entry aa_sfs_entry_network[] = {
        AA_SFS_FILE_STRING("af_mask",        AA_SFS_AF_MASK),
        { }
};

struct aa_sfs_entry aa_sfs_entry_networkv9[] = {
        AA_SFS_FILE_STRING("af_mask",        AA_SFS_AF_MASK),
        AA_SFS_FILE_BOOLEAN("af_unix",        1),
        { }
};

static const char * const net_mask_names[] = {
        "unknown",
        "send",
        "receive",
        "unknown",

        "create",
        "shutdown",
        "connect",
        "unknown",

        "setattr",
        "getattr",
        "setcred",
        "getcred",

        "chmod",
        "chown",
        "chgrp",
        "lock",

        "mmap",
        "mprot",
        "unknown",
        "unknown",

        "accept",
        "bind",
        "listen",
        "unknown",

        "setopt",
        "getopt",
        "unknown",
        "unknown",

        "unknown",
        "unknown",
        "unknown",
        "unknown",
};

static void audit_unix_addr(struct audit_buffer *ab, const char *str,
                            struct sockaddr_un *addr, int addrlen)
{
        int len = unix_addr_len(addrlen);

        if (!addr || len <= 0) {
                audit_log_format(ab, " %s=none", str);
        } else if (addr->sun_path[0]) {
                audit_log_format(ab, " %s=", str);
                audit_log_untrustedstring(ab, addr->sun_path);
        } else {
                audit_log_format(ab, " %s=\"@", str);
                if (audit_string_contains_control(&addr->sun_path[1], len - 1))
                        audit_log_n_hex(ab, &addr->sun_path[1], len - 1);
                else
                        audit_log_format(ab, "%.*s", len - 1,
                                         &addr->sun_path[1]);
                audit_log_format(ab, "\"");
        }
}

static void audit_unix_sk_addr(struct audit_buffer *ab, const char *str,
                               const struct sock *sk)
{
        const struct unix_sock *u = unix_sk(sk);

        if (u && u->addr) {
                int addrlen;
                struct sockaddr_un *addr = aa_sunaddr(u, &addrlen);

                audit_unix_addr(ab, str, addr, addrlen);
        } else {
                audit_unix_addr(ab, str, NULL, 0);

        }
}

/* audit callback for net specific fields */
void audit_net_cb(struct audit_buffer *ab, void *va)
{
        struct common_audit_data *sa = va;
        struct apparmor_audit_data *ad = aad(sa);

        if (address_family_names[ad->common.u.net->family])
                audit_log_format(ab, " family=\"%s\"",
                                 address_family_names[ad->common.u.net->family]);
        else
                audit_log_format(ab, " family=\"unknown(%d)\"",
                                 ad->common.u.net->family);
        if (sock_type_names[ad->net.type])
                audit_log_format(ab, " sock_type=\"%s\"",
                                 sock_type_names[ad->net.type]);
        else
                audit_log_format(ab, " sock_type=\"unknown(%d)\"",
                                 ad->net.type);
        audit_log_format(ab, " protocol=%d", ad->net.protocol);

        if (ad->request & NET_PERMS_MASK) {
                audit_log_format(ab, " requested_mask=");
                aa_audit_perm_mask(ab, ad->request, NULL, 0,
                                   net_mask_names, NET_PERMS_MASK);

                if (ad->denied & NET_PERMS_MASK) {
                        audit_log_format(ab, " denied_mask=");
                        aa_audit_perm_mask(ab, ad->denied, NULL, 0,
                                           net_mask_names, NET_PERMS_MASK);
                }
        }
        if (ad->common.u.net->family == PF_UNIX) {
                if (ad->net.addr || !ad->common.u.net->sk)
                        audit_unix_addr(ab, "addr",
                                        unix_addr(ad->net.addr),
                                        ad->net.addrlen);
                else
                        audit_unix_sk_addr(ab, "addr", ad->common.u.net->sk);
                if (ad->request & NET_PEER_MASK) {
                        audit_unix_addr(ab, "peer_addr",
                                        unix_addr(ad->net.peer.addr),
                                        ad->net.peer.addrlen);
                }
        }
        if (ad->peer) {
                audit_log_format(ab, " peer=");
                aa_label_xaudit(ab, labels_ns(ad->subj_label), ad->peer,
                                FLAGS_NONE, GFP_ATOMIC);
        }
}

/* standard permission lookup pattern - supports early bailout */
int aa_do_perms(struct aa_profile *profile, struct aa_policydb *policy,
                aa_state_t state, u32 request,
                struct aa_perms *p, struct apparmor_audit_data *ad)
{
        struct aa_perms perms;

        AA_BUG(!profile);
        AA_BUG(!policy);


        if (state || !p)
                p = aa_lookup_perms(policy, state);
        perms = *p;
        aa_apply_modes_to_perms(profile, &perms);
        return aa_check_perms(profile, &perms, request, ad,
                              audit_net_cb);
}

/* only continue match if
 *   insufficient current perms at current state
 *   indicates there are more perms in later state
 * Returns: perms struct if early match
 */
static struct aa_perms *early_match(struct aa_policydb *policy,
                                    aa_state_t state, u32 request)
{
        struct aa_perms *p;

        p = aa_lookup_perms(policy, state);
        if (((p->allow & request) != request) && (p->allow & AA_CONT_MATCH))
                return NULL;
        return p;
}

static aa_state_t aa_dfa_match_be16(struct aa_dfa *dfa, aa_state_t state,
                                          u16 data)
{
        __be16 buffer = cpu_to_be16(data);

        return aa_dfa_match_len(dfa, state, (char *) &buffer, 2);
}

/**
 * aa_match_to_prot - match the af, type, protocol triplet
 * @policy: policy being matched
 * @state: state to start in
 * @request: permissions being requested, ignored if @p == NULL
 * @af: socket address family
 * @type: socket type
 * @protocol: socket protocol
 * @p: output - pointer to permission associated with match
 * @info: output - pointer to string describing failure
 *
 * RETURNS: state match stopped in.
 *
 * If @(p) is assigned a value the returned state will be the
 * corresponding state. Will not set @p on failure or if match completes
 * only if an early match occurs
 */
aa_state_t aa_match_to_prot(struct aa_policydb *policy, aa_state_t state,
                            u32 request, u16 af, int type, int protocol,
                            struct aa_perms **p, const char **info)
{
        state = aa_dfa_match_be16(policy->dfa, state, (u16)af);
        if (!state) {
                *info = "failed af match";
                return state;
        }
        state = aa_dfa_match_be16(policy->dfa, state, (u16)type);
        if (state) {
                if (p)
                        *p = early_match(policy, state, request);
                if (!p || !*p) {
                        state = aa_dfa_match_be16(policy->dfa, state, (u16)protocol);
                        if (!state)
                                *info = "failed protocol match";
                }
        } else {
                *info = "failed type match";
        }

        return state;
}

/* Generic af perm */
int aa_profile_af_perm(struct aa_profile *profile,
                       struct apparmor_audit_data *ad, u32 request, u16 family,
                       int type, int protocol)
{
        struct aa_ruleset *rules = profile->label.rules[0];
        struct aa_perms *p = NULL;
        aa_state_t state;

        AA_BUG(family >= AF_MAX);
        AA_BUG(type < 0 || type >= SOCK_MAX);
        AA_BUG(profile_unconfined(profile));

        if (profile_unconfined(profile))
                return 0;
        state = RULE_MEDIATES_NET(rules);
        if (!state)
                return 0;
        state = aa_match_to_prot(rules->policy, state, request, family, type,
                                 protocol, &p, &ad->info);
        return aa_do_perms(profile, rules->policy, state, request, p, ad);
}

int aa_af_perm(const struct cred *subj_cred, struct aa_label *label,
               const char *op, u32 request, u16 family, int type, int protocol)
{
        struct aa_profile *profile;
        DEFINE_AUDIT_NET(ad, op, subj_cred, NULL, family, type, protocol);

        return fn_for_each_confined(label, profile,
                        aa_profile_af_perm(profile, &ad, request, family,
                                           type, protocol));
}

static int aa_label_sk_perm(const struct cred *subj_cred,
                            struct aa_label *label,
                            const char *op, u32 request,
                            struct sock *sk)
{
        struct aa_sk_ctx *ctx = aa_sock(sk);
        int error = 0;

        AA_BUG(!label);
        AA_BUG(!sk);

        if (rcu_access_pointer(ctx->label) != kernel_t && !unconfined(label)) {
                struct aa_profile *profile;
                DEFINE_AUDIT_SK(ad, op, subj_cred, sk);

                ad.subj_cred = subj_cred;
                error = fn_for_each_confined(label, profile,
                            aa_profile_af_sk_perm(profile, &ad, request, sk));
        }

        return error;
}

int aa_sk_perm(const char *op, u32 request, struct sock *sk)
{
        struct aa_label *label;
        int error;

        AA_BUG(!sk);
        AA_BUG(in_interrupt());

        /* TODO: switch to begin_current_label ???? */
        label = begin_current_label_crit_section();
        error = aa_label_sk_perm(current_cred(), label, op, request, sk);
        end_current_label_crit_section(label);

        return error;
}


int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
                      const char *op, u32 request, struct file *file)
{
        struct socket *sock = (struct socket *) file->private_data;

        AA_BUG(!label);
        AA_BUG(!sock);
        AA_BUG(!sock->sk);

        if (sock->sk->sk_family == PF_UNIX)
                return aa_unix_file_perm(subj_cred, label, op, request, file);
        return aa_label_sk_perm(subj_cred, label, op, request, sock->sk);
}

#ifdef CONFIG_NETWORK_SECMARK
static int apparmor_secmark_init(struct aa_secmark *secmark)
{
        struct aa_label *label;

        if (secmark->label[0] == '*') {
                secmark->secid = AA_SECID_WILDCARD;
                return 0;
        }

        label = aa_label_strn_parse(&root_ns->unconfined->label,
                                    secmark->label, strlen(secmark->label),
                                    GFP_ATOMIC, false, false);

        if (IS_ERR(label))
                return PTR_ERR(label);

        secmark->secid = label->secid;

        return 0;
}

static int aa_secmark_perm(struct aa_profile *profile, u32 request, u32 secid,
                           struct apparmor_audit_data *ad)
{
        int i, ret;
        struct aa_perms perms = { };
        struct aa_ruleset *rules = profile->label.rules[0];

        if (rules->secmark_count == 0)
                return 0;

        for (i = 0; i < rules->secmark_count; i++) {
                if (!rules->secmark[i].secid) {
                        ret = apparmor_secmark_init(&rules->secmark[i]);
                        if (ret)
                                return ret;
                }

                if (rules->secmark[i].secid == secid ||
                    rules->secmark[i].secid == AA_SECID_WILDCARD) {
                        if (rules->secmark[i].deny)
                                perms.deny = ALL_PERMS_MASK;
                        else
                                perms.allow = ALL_PERMS_MASK;

                        if (rules->secmark[i].audit)
                                perms.audit = ALL_PERMS_MASK;
                }
        }

        aa_apply_modes_to_perms(profile, &perms);

        return aa_check_perms(profile, &perms, request, ad, audit_net_cb);
}

int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
                           u32 secid, const struct sock *sk)
{
        struct aa_profile *profile;
        DEFINE_AUDIT_SK(ad, op, NULL, sk);

        return fn_for_each_confined(label, profile,
                                    aa_secmark_perm(profile, request, secid,
                                                    &ad));
}
#endif

























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Queued spinlock
 *
 * A 'generic' spinlock implementation that is based on MCS locks. For an
 * architecture that's looking for a 'generic' spinlock, please first consider
 * ticket-lock.h and only come looking here when you've considered all the
 * constraints below and can show your hardware does actually perform better
 * with qspinlock.
 *
 * qspinlock relies on atomic_*_release()/atomic_*_acquire() to be RCsc (or no
 * weaker than RCtso if you're power), where regular code only expects atomic_t
 * to be RCpc.
 *
 * qspinlock relies on a far greater (compared to asm-generic/spinlock.h) set
 * of atomic operations to behave well together, please audit them carefully to
 * ensure they all have forward progress. Many atomic operations may default to
 * cmpxchg() loops which will not have good forward progress properties on
 * LL/SC architectures.
 *
 * One notable example is atomic_fetch_or_acquire(), which x86 cannot (cheaply)
 * do. Carefully read the patches that introduced
 * queued_fetch_set_pending_acquire().
 *
 * qspinlock also heavily relies on mixed size atomic operations, in specific
 * it requires architectures to have xchg16; something which many LL/SC
 * architectures need to implement as a 32bit and+or in order to satisfy the
 * forward progress guarantees mentioned above.
 *
 * Further reading on mixed size atomics that might be relevant:
 *
 *   http://www.cl.cam.ac.uk/~pes20/popl17/mixed-size.pdf
 *
 * (C) Copyright 2013-2015 Hewlett-Packard Development Company, L.P.
 * (C) Copyright 2015 Hewlett-Packard Enterprise Development LP
 *
 * Authors: Waiman Long <waiman.long@hpe.com>
 */
#ifndef __ASM_GENERIC_QSPINLOCK_H
#define __ASM_GENERIC_QSPINLOCK_H

#include <asm-generic/qspinlock_types.h>
#include <linux/atomic.h>

#ifndef queued_spin_is_locked
/**
 * queued_spin_is_locked - is the spinlock locked?
 * @lock: Pointer to queued spinlock structure
 * Return: 1 if it is locked, 0 otherwise
 */
static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
{
        /*
         * Any !0 state indicates it is locked, even if _Q_LOCKED_VAL
         * isn't immediately observable.
         */
        return atomic_read(&lock->val);
}
#endif

/**
 * queued_spin_value_unlocked - is the spinlock structure unlocked?
 * @lock: queued spinlock structure
 * Return: 1 if it is unlocked, 0 otherwise
 *
 * N.B. Whenever there are tasks waiting for the lock, it is considered
 *      locked wrt the lockref code to avoid lock stealing by the lockref
 *      code and change things underneath the lock. This also allows some
 *      optimizations to be applied without conflict with lockref.
 */
static __always_inline int queued_spin_value_unlocked(struct qspinlock lock)
{
        return !lock.val.counter;
}

/**
 * queued_spin_is_contended - check if the lock is contended
 * @lock : Pointer to queued spinlock structure
 * Return: 1 if lock contended, 0 otherwise
 */
static __always_inline int queued_spin_is_contended(struct qspinlock *lock)
{
        return atomic_read(&lock->val) & ~_Q_LOCKED_MASK;
}
/**
 * queued_spin_trylock - try to acquire the queued spinlock
 * @lock : Pointer to queued spinlock structure
 * Return: 1 if lock acquired, 0 if failed
 */
static __always_inline int queued_spin_trylock(struct qspinlock *lock)
{
        int val = atomic_read(&lock->val);

        if (unlikely(val))
                return 0;

        return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
}

extern void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);

#ifndef queued_spin_lock
/**
 * queued_spin_lock - acquire a queued spinlock
 * @lock: Pointer to queued spinlock structure
 */
static __always_inline void queued_spin_lock(struct qspinlock *lock)
{
        int val = 0;

        if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
                return;

        queued_spin_lock_slowpath(lock, val);
}
#endif

#ifndef queued_spin_unlock
/**
 * queued_spin_unlock - release a queued spinlock
 * @lock : Pointer to queued spinlock structure
 */
static __always_inline void queued_spin_unlock(struct qspinlock *lock)
{
        /*
         * unlock() needs release semantics:
         */
        smp_store_release(&lock->locked, 0);
}
#endif

#ifndef virt_spin_lock
static __always_inline bool virt_spin_lock(struct qspinlock *lock)
{
        return false;
}
#endif

#ifndef __no_arch_spinlock_redefine
/*
 * Remapping spinlock architecture specific functions to the corresponding
 * queued spinlock functions.
 */
#define arch_spin_is_locked(l)                queued_spin_is_locked(l)
#define arch_spin_is_contended(l)        queued_spin_is_contended(l)
#define arch_spin_value_unlocked(l)        queued_spin_value_unlocked(l)
#define arch_spin_lock(l)                queued_spin_lock(l)
#define arch_spin_trylock(l)                queued_spin_trylock(l)
#define arch_spin_unlock(l)                queued_spin_unlock(l)
#endif

#endif /* __ASM_GENERIC_QSPINLOCK_H */




















































































































    2 


    1 
    1 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Creates audit record for dropped/accepted packets
 *
 * (C) 2010-2011 Thomas Graf <tgraf@redhat.com>
 * (C) 2010-2011 Red Hat, Inc.
*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/audit.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_AUDIT.h>
#include <linux/netfilter_bridge/ebtables.h>
#include <net/ipv6.h>
#include <net/ip.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Thomas Graf <tgraf@redhat.com>");
MODULE_DESCRIPTION("Xtables: creates audit records for dropped/accepted packets");
MODULE_ALIAS("ipt_AUDIT");
MODULE_ALIAS("ip6t_AUDIT");
MODULE_ALIAS("ebt_AUDIT");
MODULE_ALIAS("arpt_AUDIT");

static bool audit_ip4(struct audit_buffer *ab, struct sk_buff *skb)
{
        struct iphdr _iph;
        const struct iphdr *ih;

        ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_iph), &_iph);
        if (!ih)
                return false;

        audit_log_format(ab, " saddr=%pI4 daddr=%pI4 proto=%hhu",
                         &ih->saddr, &ih->daddr, ih->protocol);

        return true;
}

static bool audit_ip6(struct audit_buffer *ab, struct sk_buff *skb)
{
        struct ipv6hdr _ip6h;
        const struct ipv6hdr *ih;
        u8 nexthdr;
        __be16 frag_off;

        ih = skb_header_pointer(skb, skb_network_offset(skb), sizeof(_ip6h), &_ip6h);
        if (!ih)
                return false;

        nexthdr = ih->nexthdr;
        ipv6_skip_exthdr(skb, skb_network_offset(skb) + sizeof(_ip6h), &nexthdr, &frag_off);

        audit_log_format(ab, " saddr=%pI6c daddr=%pI6c proto=%hhu",
                         &ih->saddr, &ih->daddr, nexthdr);

        return true;
}

static unsigned int
audit_tg(struct sk_buff *skb, const struct xt_action_param *par)
{
        struct audit_buffer *ab;
        int fam = -1;

        if (audit_enabled == AUDIT_OFF)
                goto errout;
        ab = audit_log_start(NULL, GFP_ATOMIC, AUDIT_NETFILTER_PKT);
        if (ab == NULL)
                goto errout;

        audit_log_format(ab, "mark=%#x", skb->mark);

        switch (xt_family(par)) {
        case NFPROTO_BRIDGE:
                switch (eth_hdr(skb)->h_proto) {
                case htons(ETH_P_IP):
                        fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
                        break;
                case htons(ETH_P_IPV6):
                        fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
                        break;
                }
                break;
        case NFPROTO_IPV4:
                fam = audit_ip4(ab, skb) ? NFPROTO_IPV4 : -1;
                break;
        case NFPROTO_IPV6:
                fam = audit_ip6(ab, skb) ? NFPROTO_IPV6 : -1;
                break;
        }

        if (fam == -1)
                audit_log_format(ab, " saddr=? daddr=? proto=-1");

        audit_log_end(ab);

errout:
        return XT_CONTINUE;
}

static unsigned int
audit_tg_ebt(struct sk_buff *skb, const struct xt_action_param *par)
{
        audit_tg(skb, par);
        return EBT_CONTINUE;
}

static int audit_tg_check(const struct xt_tgchk_param *par)
{
        const struct xt_audit_info *info = par->targinfo;

        if (info->type > XT_AUDIT_TYPE_MAX) {
                pr_info_ratelimited("Audit type out of range (valid range: 0..%u)\n",
                                    XT_AUDIT_TYPE_MAX);
                return -ERANGE;
        }

        return 0;
}

static struct xt_target audit_tg_reg[] __read_mostly = {
        {
                .name                = "AUDIT",
                .family                = NFPROTO_UNSPEC,
                .target                = audit_tg,
                .targetsize        = sizeof(struct xt_audit_info),
                .checkentry        = audit_tg_check,
                .me                = THIS_MODULE,
        },
        {
                .name                = "AUDIT",
                .family                = NFPROTO_BRIDGE,
                .target                = audit_tg_ebt,
                .targetsize        = sizeof(struct xt_audit_info),
                .checkentry        = audit_tg_check,
                .me                = THIS_MODULE,
        },
};

static int __init audit_tg_init(void)
{
        return xt_register_targets(audit_tg_reg, ARRAY_SIZE(audit_tg_reg));
}

static void __exit audit_tg_exit(void)
{
        xt_unregister_targets(audit_tg_reg, ARRAY_SIZE(audit_tg_reg));
}

module_init(audit_tg_init);
module_exit(audit_tg_exit);
























































   12 


















 1892 



  290 











 1679 






















  605 
  398 



  288 

























   81 
   82 



    1 























 1196 
 1198 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
// SPDX-License-Identifier: GPL-2.0
/*
 * Out-of-line refcount functions.
 */

#include <linux/mutex.h>
#include <linux/refcount.h>
#include <linux/spinlock.h>
#include <linux/bug.h>

#define REFCOUNT_WARN(str)        WARN_ONCE(1, "refcount_t: " str ".\n")

void refcount_warn_saturate(refcount_t *r, enum refcount_saturation_type t)
{
        refcount_set(r, REFCOUNT_SATURATED);

        switch (t) {
        case REFCOUNT_ADD_NOT_ZERO_OVF:
                REFCOUNT_WARN("saturated; leaking memory");
                break;
        case REFCOUNT_ADD_OVF:
                REFCOUNT_WARN("saturated; leaking memory");
                break;
        case REFCOUNT_ADD_UAF:
                REFCOUNT_WARN("addition on 0; use-after-free");
                break;
        case REFCOUNT_SUB_UAF:
                REFCOUNT_WARN("underflow; use-after-free");
                break;
        case REFCOUNT_DEC_LEAK:
                REFCOUNT_WARN("decrement hit 0; leaking memory");
                break;
        default:
                REFCOUNT_WARN("unknown saturation event!?");
        }
}
EXPORT_SYMBOL(refcount_warn_saturate);

/**
 * refcount_dec_if_one - decrement a refcount if it is 1
 * @r: the refcount
 *
 * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the
 * success thereof.
 *
 * Like all decrement operations, it provides release memory order and provides
 * a control dependency.
 *
 * It can be used like a try-delete operator; this explicit case is provided
 * and not cmpxchg in generic, because that would allow implementing unsafe
 * operations.
 *
 * Return: true if the resulting refcount is 0, false otherwise
 */
bool refcount_dec_if_one(refcount_t *r)
{
        int val = 1;

        return atomic_try_cmpxchg_release(&r->refs, &val, 0);
}
EXPORT_SYMBOL(refcount_dec_if_one);

/**
 * refcount_dec_not_one - decrement a refcount if it is not 1
 * @r: the refcount
 *
 * No atomic_t counterpart, it decrements unless the value is 1, in which case
 * it will return false.
 *
 * Was often done like: atomic_add_unless(&var, -1, 1)
 *
 * Return: true if the decrement operation was successful, false otherwise
 */
bool refcount_dec_not_one(refcount_t *r)
{
        unsigned int new, val = atomic_read(&r->refs);

        do {
                if (unlikely(val == REFCOUNT_SATURATED))
                        return true;

                if (val == 1)
                        return false;

                new = val - 1;
                if (new > val) {
                        WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
                        return true;
                }

        } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));

        return true;
}
EXPORT_SYMBOL(refcount_dec_not_one);

/**
 * refcount_dec_and_mutex_lock - return holding mutex if able to decrement
 *                               refcount to 0
 * @r: the refcount
 * @lock: the mutex to be locked
 *
 * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
 * to decrement when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides a control dependency such that free() must come after.
 * See the comment on top.
 *
 * Return: true and hold mutex if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock)
{
        if (refcount_dec_not_one(r))
                return false;

        mutex_lock(lock);
        if (!refcount_dec_and_test(r)) {
                mutex_unlock(lock);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_mutex_lock);

/**
 * refcount_dec_and_lock - return holding spinlock if able to decrement
 *                         refcount to 0
 * @r: the refcount
 * @lock: the spinlock to be locked
 *
 * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
 * decrement when saturated at REFCOUNT_SATURATED.
 *
 * Provides release memory ordering, such that prior loads and stores are done
 * before, and provides a control dependency such that free() must come after.
 * See the comment on top.
 *
 * Return: true and hold spinlock if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
{
        if (refcount_dec_not_one(r))
                return false;

        spin_lock(lock);
        if (!refcount_dec_and_test(r)) {
                spin_unlock(lock);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_lock);

/**
 * refcount_dec_and_lock_irqsave - return holding spinlock with disabled
 *                                 interrupts if able to decrement refcount to 0
 * @r: the refcount
 * @lock: the spinlock to be locked
 * @flags: saved IRQ-flags if the is acquired
 *
 * Same as refcount_dec_and_lock() above except that the spinlock is acquired
 * with disabled interrupts.
 *
 * Return: true and hold spinlock if able to decrement refcount to 0, false
 *         otherwise
 */
bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock,
                                   unsigned long *flags)
{
        if (refcount_dec_not_one(r))
                return false;

        spin_lock_irqsave(lock, *flags);
        if (!refcount_dec_and_test(r)) {
                spin_unlock_irqrestore(lock, *flags);
                return false;
        }

        return true;
}
EXPORT_SYMBOL(refcount_dec_and_lock_irqsave);






































































   10 



    1 
   10 








    7 






    8 














    8 












   17 



    1 











    7 


    7 

    7 

    7 












































































    5 









    1 

    4 








    3 



    2 



    2 





    4 


















    2 



    2 
    2 

    1 

    1 














    5 



    5 


    4 


























































































    1 


































    2 

    2 

    4 

   16 







   12 







   12 




















    1 






    1 




    1 

    1 





    4 





    6 





    6 









    3 
    4 













    7 









    9 




   10 






    1 





    1 




    1 
    3 









    2 
    1 































    3 







    3 














    4 



    1 





    3 


    2 





    2 












    3 
    3 






    4 


    1 

    3 



    1 

    1 






   16 





    1 


    1 

    1 



    1 







    3 



    2 
    1 















    1 




    1 


    1 
    1 


    1 

    4 

    1 

    2 












    8 








    1 

    1 

    1 



    1 

    1 


    1 


    2 










    1 

    1 

    2 










    5 





    1 



    1 




    1 

    1 
    1 



    2 






    1 
    1 


    2 






    1 


    1 








    2 






















    2 







   17 










   11 


    6 




    1 
    5 








































































   10 










    9 





    1 



















































































   10 





   10 





    5 
    3 
    3 





















   17 






    1 












   17 

    3 











    9 


    8 




    7 


    1 






    2 





    2 







    5 





    1 
    1 





    1 
    8 










   24 








   21 







   23 









    1 




    1 




    1 




    7 





    3 
    4 






   12 







   17 








    6 




    6 


    6 







    5 











    5 











    5 















    5 




    5 




    3 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2010-2011 EIA Electronics,
//                         Pieter Beyens <pieter.beyens@eia.be>
// Copyright (c) 2010-2011 EIA Electronics,
//                         Kurt Van Dijck <kurt.van.dijck@eia.be>
// Copyright (c) 2018 Protonic,
//                         Robin van der Gracht <robin@protonic.nl>
// Copyright (c) 2017-2019 Pengutronix,
//                         Marc Kleine-Budde <kernel@pengutronix.de>
// Copyright (c) 2017-2019 Pengutronix,
//                         Oleksij Rempel <kernel@pengutronix.de>

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/can/can-ml.h>
#include <linux/can/core.h>
#include <linux/can/skb.h>
#include <linux/errqueue.h>
#include <linux/if_arp.h>

#include "j1939-priv.h"

#define J1939_MIN_NAMELEN CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.j1939)

/* conversion function between struct sock::sk_priority from linux and
 * j1939 priority field
 */
static inline priority_t j1939_prio(u32 sk_priority)
{
        sk_priority = min(sk_priority, 7U);

        return 7 - sk_priority;
}

static inline u32 j1939_to_sk_priority(priority_t prio)
{
        return 7 - prio;
}

/* function to see if pgn is to be evaluated */
static inline bool j1939_pgn_is_valid(pgn_t pgn)
{
        return pgn <= J1939_PGN_MAX;
}

/* test function to avoid non-zero DA placeholder for pdu1 pgn's */
static inline bool j1939_pgn_is_clean_pdu(pgn_t pgn)
{
        if (j1939_pgn_is_pdu1(pgn))
                return !(pgn & 0xff);
        else
                return true;
}

static inline void j1939_sock_pending_add(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        atomic_inc(&jsk->skb_pending);
}

static int j1939_sock_pending_get(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        return atomic_read(&jsk->skb_pending);
}

void j1939_sock_pending_del(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* atomic_dec_return returns the new value */
        if (!atomic_dec_return(&jsk->skb_pending))
                wake_up(&jsk->waitq);        /* no pending SKB's */
}

static void j1939_jsk_add(struct j1939_priv *priv, struct j1939_sock *jsk)
{
        jsk->state |= J1939_SOCK_BOUND;
        j1939_priv_get(priv);

        write_lock_bh(&priv->j1939_socks_lock);
        list_add_tail(&jsk->list, &priv->j1939_socks);
        write_unlock_bh(&priv->j1939_socks_lock);
}

static void j1939_jsk_del(struct j1939_priv *priv, struct j1939_sock *jsk)
{
        write_lock_bh(&priv->j1939_socks_lock);
        list_del_init(&jsk->list);
        write_unlock_bh(&priv->j1939_socks_lock);

        j1939_priv_put(priv);
        jsk->state &= ~J1939_SOCK_BOUND;
}

static bool j1939_sk_queue_session(struct j1939_session *session)
{
        struct j1939_sock *jsk = j1939_sk(session->sk);
        bool empty;

        spin_lock_bh(&jsk->sk_session_queue_lock);
        empty = list_empty(&jsk->sk_session_queue);
        j1939_session_get(session);
        list_add_tail(&session->sk_session_queue_entry, &jsk->sk_session_queue);
        spin_unlock_bh(&jsk->sk_session_queue_lock);
        j1939_sock_pending_add(&jsk->sk);

        return empty;
}

static struct
j1939_session *j1939_sk_get_incomplete_session(struct j1939_sock *jsk)
{
        struct j1939_session *session = NULL;

        spin_lock_bh(&jsk->sk_session_queue_lock);
        if (!list_empty(&jsk->sk_session_queue)) {
                session = list_last_entry(&jsk->sk_session_queue,
                                          struct j1939_session,
                                          sk_session_queue_entry);
                if (session->total_queued_size == session->total_message_size)
                        session = NULL;
                else
                        j1939_session_get(session);
        }
        spin_unlock_bh(&jsk->sk_session_queue_lock);

        return session;
}

static void j1939_sk_queue_drop_all(struct j1939_priv *priv,
                                    struct j1939_sock *jsk, int err)
{
        struct j1939_session *session, *tmp;

        netdev_dbg(priv->ndev, "%s: err: %i\n", __func__, err);
        spin_lock_bh(&jsk->sk_session_queue_lock);
        list_for_each_entry_safe(session, tmp, &jsk->sk_session_queue,
                                 sk_session_queue_entry) {
                list_del_init(&session->sk_session_queue_entry);
                session->err = err;
                j1939_session_put(session);
        }
        spin_unlock_bh(&jsk->sk_session_queue_lock);
}

static void j1939_sk_queue_activate_next_locked(struct j1939_session *session)
{
        struct j1939_sock *jsk;
        struct j1939_session *first;
        int err;

        /* RX-Session don't have a socket (yet) */
        if (!session->sk)
                return;

        jsk = j1939_sk(session->sk);
        lockdep_assert_held(&jsk->sk_session_queue_lock);

        err = session->err;

        first = list_first_entry_or_null(&jsk->sk_session_queue,
                                         struct j1939_session,
                                         sk_session_queue_entry);

        /* Some else has already activated the next session */
        if (first != session)
                return;

activate_next:
        list_del_init(&first->sk_session_queue_entry);
        j1939_session_put(first);
        first = list_first_entry_or_null(&jsk->sk_session_queue,
                                         struct j1939_session,
                                         sk_session_queue_entry);
        if (!first)
                return;

        if (j1939_session_activate(first)) {
                netdev_warn_once(first->priv->ndev,
                                 "%s: 0x%p: Identical session is already activated.\n",
                                 __func__, first);
                first->err = -EBUSY;
                goto activate_next;
        } else {
                /* Give receiver some time (arbitrary chosen) to recover */
                int time_ms = 0;

                if (err)
                        time_ms = 10 + get_random_u32_below(16);

                j1939_tp_schedule_txtimer(first, time_ms);
        }
}

void j1939_sk_queue_activate_next(struct j1939_session *session)
{
        struct j1939_sock *jsk;

        if (!session->sk)
                return;

        jsk = j1939_sk(session->sk);

        spin_lock_bh(&jsk->sk_session_queue_lock);
        j1939_sk_queue_activate_next_locked(session);
        spin_unlock_bh(&jsk->sk_session_queue_lock);
}

static bool j1939_sk_match_dst(struct j1939_sock *jsk,
                               const struct j1939_sk_buff_cb *skcb)
{
        if ((jsk->state & J1939_SOCK_PROMISC))
                return true;

        /* Destination address filter */
        if (jsk->addr.src_name && skcb->addr.dst_name) {
                if (jsk->addr.src_name != skcb->addr.dst_name)
                        return false;
        } else {
                /* receive (all sockets) if
                 * - all packages that match our bind() address
                 * - all broadcast on a socket if SO_BROADCAST
                 *   is set
                 */
                if (j1939_address_is_unicast(skcb->addr.da)) {
                        if (jsk->addr.sa != skcb->addr.da)
                                return false;
                } else if (!sock_flag(&jsk->sk, SOCK_BROADCAST)) {
                        /* receiving broadcast without SO_BROADCAST
                         * flag is not allowed
                         */
                        return false;
                }
        }

        /* Source address filter */
        if (jsk->state & J1939_SOCK_CONNECTED) {
                /* receive (all sockets) if
                 * - all packages that match our connect() name or address
                 */
                if (jsk->addr.dst_name && skcb->addr.src_name) {
                        if (jsk->addr.dst_name != skcb->addr.src_name)
                                return false;
                } else {
                        if (jsk->addr.da != skcb->addr.sa)
                                return false;
                }
        }

        /* PGN filter */
        if (j1939_pgn_is_valid(jsk->pgn_rx_filter) &&
            jsk->pgn_rx_filter != skcb->addr.pgn)
                return false;

        return true;
}

/* matches skb control buffer (addr) with a j1939 filter */
static bool j1939_sk_match_filter(struct j1939_sock *jsk,
                                  const struct j1939_sk_buff_cb *skcb)
{
        const struct j1939_filter *f;
        int nfilter;

        spin_lock_bh(&jsk->filters_lock);

        f = jsk->filters;
        nfilter = jsk->nfilters;

        if (!nfilter)
                /* receive all when no filters are assigned */
                goto filter_match_found;

        for (; nfilter; ++f, --nfilter) {
                if ((skcb->addr.pgn & f->pgn_mask) != f->pgn)
                        continue;
                if ((skcb->addr.sa & f->addr_mask) != f->addr)
                        continue;
                if ((skcb->addr.src_name & f->name_mask) != f->name)
                        continue;
                goto filter_match_found;
        }

        spin_unlock_bh(&jsk->filters_lock);
        return false;

filter_match_found:
        spin_unlock_bh(&jsk->filters_lock);
        return true;
}

static bool j1939_sk_recv_match_one(struct j1939_sock *jsk,
                                    const struct j1939_sk_buff_cb *skcb)
{
        if (!(jsk->state & J1939_SOCK_BOUND))
                return false;

        if (!j1939_sk_match_dst(jsk, skcb))
                return false;

        if (!j1939_sk_match_filter(jsk, skcb))
                return false;

        return true;
}

static void j1939_sk_recv_one(struct j1939_sock *jsk, struct sk_buff *oskb)
{
        const struct j1939_sk_buff_cb *oskcb = j1939_skb_to_cb(oskb);
        struct j1939_sk_buff_cb *skcb;
        enum skb_drop_reason reason;
        struct sk_buff *skb;

        if (oskb->sk == &jsk->sk)
                return;

        if (!j1939_sk_recv_match_one(jsk, oskcb))
                return;

        skb = skb_clone(oskb, GFP_ATOMIC);
        if (!skb) {
                pr_warn("skb clone failed\n");
                return;
        }
        can_skb_set_owner(skb, oskb->sk);

        skcb = j1939_skb_to_cb(skb);
        skcb->msg_flags &= ~(MSG_DONTROUTE);
        if (skb->sk)
                skcb->msg_flags |= MSG_DONTROUTE;

        if (sock_queue_rcv_skb_reason(&jsk->sk, skb, &reason) < 0)
                sk_skb_reason_drop(&jsk->sk, skb, reason);
}

bool j1939_sk_recv_match(struct j1939_priv *priv, struct j1939_sk_buff_cb *skcb)
{
        struct j1939_sock *jsk;
        bool match = false;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                match = j1939_sk_recv_match_one(jsk, skcb);
                if (match)
                        break;
        }
        read_unlock_bh(&priv->j1939_socks_lock);

        return match;
}

void j1939_sk_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sock *jsk;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                j1939_sk_recv_one(jsk, skb);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
}

static void j1939_sk_sock_destruct(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* This function will be called by the generic networking code, when
         * the socket is ultimately closed (sk->sk_destruct).
         *
         * The race between
         * - processing a received CAN frame
         *   (can_receive -> j1939_can_recv)
         *   and accessing j1939_priv
         * ... and ...
         * - closing a socket
         *   (j1939_can_rx_unregister -> can_rx_unregister)
         *   and calling the final j1939_priv_put()
         *
         * is avoided by calling the final j1939_priv_put() from this
         * RCU deferred cleanup call.
         */
        if (jsk->priv) {
                j1939_priv_put(jsk->priv);
                jsk->priv = NULL;
        }

        /* call generic CAN sock destruct */
        can_sock_destruct(sk);
}

static int j1939_sk_init(struct sock *sk)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        /* Ensure that "sk" is first member in "struct j1939_sock", so that we
         * can skip it during memset().
         */
        BUILD_BUG_ON(offsetof(struct j1939_sock, sk) != 0);
        memset((void *)jsk + sizeof(jsk->sk), 0x0,
               sizeof(*jsk) - sizeof(jsk->sk));

        INIT_LIST_HEAD(&jsk->list);
        init_waitqueue_head(&jsk->waitq);
        jsk->sk.sk_priority = j1939_to_sk_priority(6);
        jsk->sk.sk_reuse = 1; /* per default */
        jsk->addr.sa = J1939_NO_ADDR;
        jsk->addr.da = J1939_NO_ADDR;
        jsk->addr.pgn = J1939_NO_PGN;
        jsk->pgn_rx_filter = J1939_NO_PGN;
        atomic_set(&jsk->skb_pending, 0);
        spin_lock_init(&jsk->sk_session_queue_lock);
        INIT_LIST_HEAD(&jsk->sk_session_queue);
        spin_lock_init(&jsk->filters_lock);

        /* j1939_sk_sock_destruct() depends on SOCK_RCU_FREE flag */
        sock_set_flag(sk, SOCK_RCU_FREE);
        sk->sk_destruct = j1939_sk_sock_destruct;
        sk->sk_protocol = CAN_J1939;

        return 0;
}

static int j1939_sk_sanity_check(struct sockaddr_can *addr, int len)
{
        if (!addr)
                return -EDESTADDRREQ;
        if (len < J1939_MIN_NAMELEN)
                return -EINVAL;
        if (addr->can_family != AF_CAN)
                return -EINVAL;
        if (!addr->can_ifindex)
                return -ENODEV;
        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn) &&
            !j1939_pgn_is_clean_pdu(addr->can_addr.j1939.pgn))
                return -EINVAL;

        return 0;
}

static int j1939_sk_bind(struct socket *sock, struct sockaddr *uaddr, int len)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct j1939_sock *jsk = j1939_sk(sock->sk);
        struct j1939_priv *priv;
        struct sock *sk;
        struct net *net;
        int ret = 0;

        ret = j1939_sk_sanity_check(addr, len);
        if (ret)
                return ret;

        lock_sock(sock->sk);

        priv = jsk->priv;
        sk = sock->sk;
        net = sock_net(sk);

        /* Already bound to an interface? */
        if (jsk->state & J1939_SOCK_BOUND) {
                /* A re-bind() to a different interface is not
                 * supported.
                 */
                if (jsk->ifindex != addr->can_ifindex) {
                        ret = -EINVAL;
                        goto out_release_sock;
                }

                /* drop old references */
                j1939_jsk_del(priv, jsk);
                j1939_local_ecu_put(priv, jsk->addr.src_name, jsk->addr.sa);
        } else {
                struct can_ml_priv *can_ml;
                struct net_device *ndev;

                ndev = dev_get_by_index(net, addr->can_ifindex);
                if (!ndev) {
                        ret = -ENODEV;
                        goto out_release_sock;
                }

                can_ml = can_get_ml_priv(ndev);
                if (!can_ml) {
                        dev_put(ndev);
                        ret = -ENODEV;
                        goto out_release_sock;
                }

                if (!(ndev->flags & IFF_UP)) {
                        dev_put(ndev);
                        ret = -ENETDOWN;
                        goto out_release_sock;
                }

                priv = j1939_netdev_start(ndev);
                dev_put(ndev);
                if (IS_ERR(priv)) {
                        ret = PTR_ERR(priv);
                        goto out_release_sock;
                }

                jsk->ifindex = addr->can_ifindex;

                /* the corresponding j1939_priv_put() is called via
                 * sk->sk_destruct, which points to j1939_sk_sock_destruct()
                 */
                j1939_priv_get(priv);
                jsk->priv = priv;
        }

        /* set default transmit pgn */
        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                jsk->pgn_rx_filter = addr->can_addr.j1939.pgn;
        jsk->addr.src_name = addr->can_addr.j1939.name;
        jsk->addr.sa = addr->can_addr.j1939.addr;

        /* get new references */
        ret = j1939_local_ecu_get(priv, jsk->addr.src_name, jsk->addr.sa);
        if (ret) {
                j1939_netdev_stop(priv);
                jsk->priv = NULL;
                synchronize_rcu();
                j1939_priv_put(priv);
                goto out_release_sock;
        }

        j1939_jsk_add(priv, jsk);

 out_release_sock: /* fall through */
        release_sock(sock->sk);

        return ret;
}

static int j1939_sk_connect(struct socket *sock, struct sockaddr *uaddr,
                            int len, int flags)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct j1939_sock *jsk = j1939_sk(sock->sk);
        int ret = 0;

        ret = j1939_sk_sanity_check(addr, len);
        if (ret)
                return ret;

        lock_sock(sock->sk);

        /* bind() before connect() is mandatory */
        if (!(jsk->state & J1939_SOCK_BOUND)) {
                ret = -EINVAL;
                goto out_release_sock;
        }

        /* A connect() to a different interface is not supported. */
        if (jsk->ifindex != addr->can_ifindex) {
                ret = -EINVAL;
                goto out_release_sock;
        }

        if (!addr->can_addr.j1939.name &&
            addr->can_addr.j1939.addr == J1939_NO_ADDR &&
            !sock_flag(&jsk->sk, SOCK_BROADCAST)) {
                /* broadcast, but SO_BROADCAST not set */
                ret = -EACCES;
                goto out_release_sock;
        }

        jsk->addr.dst_name = addr->can_addr.j1939.name;
        jsk->addr.da = addr->can_addr.j1939.addr;

        if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                jsk->addr.pgn = addr->can_addr.j1939.pgn;

        jsk->state |= J1939_SOCK_CONNECTED;

 out_release_sock: /* fall through */
        release_sock(sock->sk);

        return ret;
}

static void j1939_sk_sock2sockaddr_can(struct sockaddr_can *addr,
                                       const struct j1939_sock *jsk, int peer)
{
        /* There are two holes (2 bytes and 3 bytes) to clear to avoid
         * leaking kernel information to user space.
         */
        memset(addr, 0, J1939_MIN_NAMELEN);

        addr->can_family = AF_CAN;
        addr->can_ifindex = jsk->ifindex;
        addr->can_addr.j1939.pgn = jsk->addr.pgn;
        if (peer) {
                addr->can_addr.j1939.name = jsk->addr.dst_name;
                addr->can_addr.j1939.addr = jsk->addr.da;
        } else {
                addr->can_addr.j1939.name = jsk->addr.src_name;
                addr->can_addr.j1939.addr = jsk->addr.sa;
        }
}

static int j1939_sk_getname(struct socket *sock, struct sockaddr *uaddr,
                            int peer)
{
        struct sockaddr_can *addr = (struct sockaddr_can *)uaddr;
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int ret = 0;

        lock_sock(sk);

        if (peer && !(jsk->state & J1939_SOCK_CONNECTED)) {
                ret = -EADDRNOTAVAIL;
                goto failure;
        }

        j1939_sk_sock2sockaddr_can(addr, jsk, peer);
        ret = J1939_MIN_NAMELEN;

 failure:
        release_sock(sk);

        return ret;
}

static int j1939_sk_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk;

        if (!sk)
                return 0;

        lock_sock(sk);
        jsk = j1939_sk(sk);

        if (jsk->state & J1939_SOCK_BOUND) {
                struct j1939_priv *priv = jsk->priv;

                if (wait_event_interruptible(jsk->waitq,
                                             !j1939_sock_pending_get(&jsk->sk))) {
                        j1939_cancel_active_session(priv, sk);
                        j1939_sk_queue_drop_all(priv, jsk, ESHUTDOWN);
                }

                j1939_jsk_del(priv, jsk);

                j1939_local_ecu_put(priv, jsk->addr.src_name,
                                    jsk->addr.sa);

                j1939_netdev_stop(priv);
        }

        kfree(jsk->filters);
        sock_orphan(sk);
        sock->sk = NULL;

        release_sock(sk);
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
        sock_put(sk);

        return 0;
}

static int j1939_sk_setsockopt_flag(struct j1939_sock *jsk, sockptr_t optval,
                                    unsigned int optlen, int flag)
{
        int tmp;

        if (optlen != sizeof(tmp))
                return -EINVAL;
        if (copy_from_sockptr(&tmp, optval, optlen))
                return -EFAULT;
        lock_sock(&jsk->sk);
        if (tmp)
                jsk->state |= flag;
        else
                jsk->state &= ~flag;
        release_sock(&jsk->sk);
        return tmp;
}

static int j1939_sk_setsockopt(struct socket *sock, int level, int optname,
                               sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int tmp, count = 0, ret = 0;
        struct j1939_filter *filters = NULL, *ofilters;

        if (level != SOL_CAN_J1939)
                return -EINVAL;

        switch (optname) {
        case SO_J1939_FILTER:
                if (!sockptr_is_null(optval) && optlen != 0) {
                        struct j1939_filter *f;
                        int c;

                        if (optlen % sizeof(*filters) != 0)
                                return -EINVAL;

                        if (optlen > J1939_FILTER_MAX *
                            sizeof(struct j1939_filter))
                                return -EINVAL;

                        count = optlen / sizeof(*filters);
                        filters = memdup_sockptr(optval, optlen);
                        if (IS_ERR(filters))
                                return PTR_ERR(filters);

                        for (f = filters, c = count; c; f++, c--) {
                                f->name &= f->name_mask;
                                f->pgn &= f->pgn_mask;
                                f->addr &= f->addr_mask;
                        }
                }

                lock_sock(&jsk->sk);
                spin_lock_bh(&jsk->filters_lock);
                ofilters = jsk->filters;
                jsk->filters = filters;
                jsk->nfilters = count;
                spin_unlock_bh(&jsk->filters_lock);
                release_sock(&jsk->sk);
                kfree(ofilters);
                return 0;
        case SO_J1939_PROMISC:
                return j1939_sk_setsockopt_flag(jsk, optval, optlen,
                                                J1939_SOCK_PROMISC);
        case SO_J1939_ERRQUEUE:
                ret = j1939_sk_setsockopt_flag(jsk, optval, optlen,
                                               J1939_SOCK_ERRQUEUE);
                if (ret < 0)
                        return ret;

                if (!(jsk->state & J1939_SOCK_ERRQUEUE))
                        skb_queue_purge(&sk->sk_error_queue);
                return ret;
        case SO_J1939_SEND_PRIO:
                if (optlen != sizeof(tmp))
                        return -EINVAL;
                if (copy_from_sockptr(&tmp, optval, optlen))
                        return -EFAULT;
                if (tmp < 0 || tmp > 7)
                        return -EDOM;
                if (tmp < 2 && !capable(CAP_NET_ADMIN))
                        return -EPERM;
                lock_sock(&jsk->sk);
                jsk->sk.sk_priority = j1939_to_sk_priority(tmp);
                release_sock(&jsk->sk);
                return 0;
        default:
                return -ENOPROTOOPT;
        }
}

static int j1939_sk_getsockopt(struct socket *sock, int level, int optname,
                               char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        int ret, ulen;
        /* set defaults for using 'int' properties */
        int tmp = 0;
        int len = sizeof(tmp);
        void *val = &tmp;

        if (level != SOL_CAN_J1939)
                return -EINVAL;
        if (get_user(ulen, optlen))
                return -EFAULT;
        if (ulen < 0)
                return -EINVAL;

        lock_sock(&jsk->sk);
        switch (optname) {
        case SO_J1939_PROMISC:
                tmp = (jsk->state & J1939_SOCK_PROMISC) ? 1 : 0;
                break;
        case SO_J1939_ERRQUEUE:
                tmp = (jsk->state & J1939_SOCK_ERRQUEUE) ? 1 : 0;
                break;
        case SO_J1939_SEND_PRIO:
                tmp = j1939_prio(jsk->sk.sk_priority);
                break;
        default:
                ret = -ENOPROTOOPT;
                goto no_copy;
        }

        /* copy to user, based on 'len' & 'val'
         * but most sockopt's are 'int' properties, and have 'len' & 'val'
         * left unchanged, but instead modified 'tmp'
         */
        if (len > ulen)
                ret = -EFAULT;
        else if (put_user(len, optlen))
                ret = -EFAULT;
        else if (copy_to_user(optval, val, len))
                ret = -EFAULT;
        else
                ret = 0;
 no_copy:
        release_sock(&jsk->sk);
        return ret;
}

static int j1939_sk_recvmsg(struct socket *sock, struct msghdr *msg,
                            size_t size, int flags)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        struct j1939_sk_buff_cb *skcb;
        int ret = 0;

        if (flags & ~(MSG_DONTWAIT | MSG_ERRQUEUE | MSG_CMSG_COMPAT))
                return -EINVAL;

        if (flags & MSG_ERRQUEUE)
                return sock_recv_errqueue(sock->sk, msg, size, SOL_CAN_J1939,
                                          SCM_J1939_ERRQUEUE);

        skb = skb_recv_datagram(sk, flags, &ret);
        if (!skb)
                return ret;

        if (size < skb->len)
                msg->msg_flags |= MSG_TRUNC;
        else
                size = skb->len;

        ret = memcpy_to_msg(msg, skb->data, size);
        if (ret < 0) {
                skb_free_datagram(sk, skb);
                return ret;
        }

        skcb = j1939_skb_to_cb(skb);
        if (j1939_address_is_valid(skcb->addr.da))
                put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_DEST_ADDR,
                         sizeof(skcb->addr.da), &skcb->addr.da);

        if (skcb->addr.dst_name)
                put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_DEST_NAME,
                         sizeof(skcb->addr.dst_name), &skcb->addr.dst_name);

        put_cmsg(msg, SOL_CAN_J1939, SCM_J1939_PRIO,
                 sizeof(skcb->priority), &skcb->priority);

        if (msg->msg_name) {
                struct sockaddr_can *paddr = msg->msg_name;

                msg->msg_namelen = J1939_MIN_NAMELEN;
                memset(msg->msg_name, 0, msg->msg_namelen);
                paddr->can_family = AF_CAN;
                paddr->can_ifindex = skb->skb_iif;
                paddr->can_addr.j1939.name = skcb->addr.src_name;
                paddr->can_addr.j1939.addr = skcb->addr.sa;
                paddr->can_addr.j1939.pgn = skcb->addr.pgn;
        }

        sock_recv_cmsgs(msg, sk, skb);
        msg->msg_flags |= skcb->msg_flags;
        skb_free_datagram(sk, skb);

        return size;
}

static struct sk_buff *j1939_sk_alloc_skb(struct net_device *ndev,
                                          struct sock *sk,
                                          struct msghdr *msg, size_t size,
                                          int *errcode)
{
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_sk_buff_cb *skcb;
        struct sk_buff *skb;
        int ret;

        skb = sock_alloc_send_skb(sk,
                                  size +
                                  sizeof(struct can_frame) -
                                  sizeof(((struct can_frame *)NULL)->data) +
                                  sizeof(struct can_skb_priv),
                                  msg->msg_flags & MSG_DONTWAIT, &ret);
        if (!skb)
                goto failure;

        can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = ndev->ifindex;
        can_skb_prv(skb)->skbcnt = 0;
        skb_reserve(skb, offsetof(struct can_frame, data));

        ret = memcpy_from_msg(skb_put(skb, size), msg, size);
        if (ret < 0)
                goto free_skb;

        skb->dev = ndev;

        skcb = j1939_skb_to_cb(skb);
        memset(skcb, 0, sizeof(*skcb));
        skcb->addr = jsk->addr;
        skcb->priority = j1939_prio(READ_ONCE(sk->sk_priority));

        if (msg->msg_name) {
                struct sockaddr_can *addr = msg->msg_name;

                if (addr->can_addr.j1939.name ||
                    addr->can_addr.j1939.addr != J1939_NO_ADDR) {
                        skcb->addr.dst_name = addr->can_addr.j1939.name;
                        skcb->addr.da = addr->can_addr.j1939.addr;
                }
                if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn))
                        skcb->addr.pgn = addr->can_addr.j1939.pgn;
        }

        *errcode = ret;
        return skb;

free_skb:
        kfree_skb(skb);
failure:
        *errcode = ret;
        return NULL;
}

static size_t j1939_sk_opt_stats_get_size(enum j1939_sk_errqueue_type type)
{
        switch (type) {
        case J1939_ERRQUEUE_RX_RTS:
                return
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_TOTAL_SIZE */
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_PGN */
                        nla_total_size(sizeof(u64)) + /* J1939_NLA_SRC_NAME */
                        nla_total_size(sizeof(u64)) + /* J1939_NLA_DEST_NAME */
                        nla_total_size(sizeof(u8)) +  /* J1939_NLA_SRC_ADDR */
                        nla_total_size(sizeof(u8)) +  /* J1939_NLA_DEST_ADDR */
                        0;
        default:
                return
                        nla_total_size(sizeof(u32)) + /* J1939_NLA_BYTES_ACKED */
                        0;
        }
}

static struct sk_buff *
j1939_sk_get_timestamping_opt_stats(struct j1939_session *session,
                                    enum j1939_sk_errqueue_type type)
{
        struct sk_buff *stats;
        u32 size;

        stats = alloc_skb(j1939_sk_opt_stats_get_size(type), GFP_ATOMIC);
        if (!stats)
                return NULL;

        if (session->skcb.addr.type == J1939_SIMPLE)
                size = session->total_message_size;
        else
                size = min(session->pkt.tx_acked * 7,
                           session->total_message_size);

        switch (type) {
        case J1939_ERRQUEUE_RX_RTS:
                nla_put_u32(stats, J1939_NLA_TOTAL_SIZE,
                            session->total_message_size);
                nla_put_u32(stats, J1939_NLA_PGN,
                            session->skcb.addr.pgn);
                nla_put_u64_64bit(stats, J1939_NLA_SRC_NAME,
                                  session->skcb.addr.src_name, J1939_NLA_PAD);
                nla_put_u64_64bit(stats, J1939_NLA_DEST_NAME,
                                  session->skcb.addr.dst_name, J1939_NLA_PAD);
                nla_put_u8(stats, J1939_NLA_SRC_ADDR,
                           session->skcb.addr.sa);
                nla_put_u8(stats, J1939_NLA_DEST_ADDR,
                           session->skcb.addr.da);
                break;
        default:
                nla_put_u32(stats, J1939_NLA_BYTES_ACKED, size);
        }

        return stats;
}

static void __j1939_sk_errqueue(struct j1939_session *session, struct sock *sk,
                                enum j1939_sk_errqueue_type type)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sock *jsk;
        struct sock_exterr_skb *serr;
        struct sk_buff *skb;
        char *state = "UNK";
        u32 tsflags;
        int err;

        jsk = j1939_sk(sk);

        if (!(jsk->state & J1939_SOCK_ERRQUEUE))
                return;

        tsflags = READ_ONCE(sk->sk_tsflags);
        switch (type) {
        case J1939_ERRQUEUE_TX_ACK:
                if (!(tsflags & SOF_TIMESTAMPING_TX_ACK))
                        return;
                break;
        case J1939_ERRQUEUE_TX_SCHED:
                if (!(tsflags & SOF_TIMESTAMPING_TX_SCHED))
                        return;
                break;
        case J1939_ERRQUEUE_TX_ABORT:
                break;
        case J1939_ERRQUEUE_RX_RTS:
                fallthrough;
        case J1939_ERRQUEUE_RX_DPO:
                fallthrough;
        case J1939_ERRQUEUE_RX_ABORT:
                if (!(tsflags & SOF_TIMESTAMPING_RX_SOFTWARE))
                        return;
                break;
        default:
                netdev_err(priv->ndev, "Unknown errqueue type %i\n", type);
        }

        skb = j1939_sk_get_timestamping_opt_stats(session, type);
        if (!skb)
                return;

        skb->tstamp = ktime_get_real();

        BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));

        serr = SKB_EXT_ERR(skb);
        memset(serr, 0, sizeof(*serr));
        switch (type) {
        case J1939_ERRQUEUE_TX_ACK:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
                serr->ee.ee_info = SCM_TSTAMP_ACK;
                state = "TX ACK";
                break;
        case J1939_ERRQUEUE_TX_SCHED:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
                serr->ee.ee_info = SCM_TSTAMP_SCHED;
                state = "TX SCH";
                break;
        case J1939_ERRQUEUE_TX_ABORT:
                serr->ee.ee_errno = session->err;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_TX_ABORT;
                state = "TX ABT";
                break;
        case J1939_ERRQUEUE_RX_RTS:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_RTS;
                state = "RX RTS";
                break;
        case J1939_ERRQUEUE_RX_DPO:
                serr->ee.ee_errno = ENOMSG;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_DPO;
                state = "RX DPO";
                break;
        case J1939_ERRQUEUE_RX_ABORT:
                serr->ee.ee_errno = session->err;
                serr->ee.ee_origin = SO_EE_ORIGIN_LOCAL;
                serr->ee.ee_info = J1939_EE_INFO_RX_ABORT;
                state = "RX ABT";
                break;
        }

        serr->opt_stats = true;
        if (tsflags & SOF_TIMESTAMPING_OPT_ID)
                serr->ee.ee_data = session->tskey;

        netdev_dbg(session->priv->ndev, "%s: 0x%p tskey: %i, state: %s\n",
                   __func__, session, session->tskey, state);
        err = sock_queue_err_skb(sk, skb);

        if (err)
                kfree_skb(skb);
};

void j1939_sk_errqueue(struct j1939_session *session,
                       enum j1939_sk_errqueue_type type)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sock *jsk;

        if (session->sk) {
                /* send TX notifications to the socket of origin  */
                __j1939_sk_errqueue(session, session->sk, type);
                return;
        }

        /* spread RX notifications to all sockets subscribed to this session */
        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                if (j1939_sk_recv_match_one(jsk, &session->skcb))
                        __j1939_sk_errqueue(session, &jsk->sk, type);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
};

void j1939_sk_send_loop_abort(struct sock *sk, int err)
{
        struct j1939_sock *jsk = j1939_sk(sk);

        if (jsk->state & J1939_SOCK_ERRQUEUE)
                return;

        sk->sk_err = err;

        sk_error_report(sk);
}

static int j1939_sk_send_loop(struct j1939_priv *priv,  struct sock *sk,
                              struct msghdr *msg, size_t size)

{
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_session *session = j1939_sk_get_incomplete_session(jsk);
        struct sk_buff *skb;
        size_t segment_size, todo_size;
        int ret = 0;

        if (session &&
            session->total_message_size != session->total_queued_size + size) {
                j1939_session_put(session);
                return -EIO;
        }

        todo_size = size;

        do {
                struct j1939_sk_buff_cb *skcb;

                segment_size = min_t(size_t, J1939_MAX_TP_PACKET_SIZE,
                                     todo_size);

                /* Allocate skb for one segment */
                skb = j1939_sk_alloc_skb(priv->ndev, sk, msg, segment_size,
                                         &ret);
                if (ret)
                        break;

                skcb = j1939_skb_to_cb(skb);

                if (!session) {
                        /* at this point the size should be full size
                         * of the session
                         */
                        skcb->offset = 0;
                        session = j1939_tp_send(priv, skb, size);
                        if (IS_ERR(session)) {
                                ret = PTR_ERR(session);
                                goto kfree_skb;
                        }
                        if (j1939_sk_queue_session(session)) {
                                /* try to activate session if we a
                                 * fist in the queue
                                 */
                                if (!j1939_session_activate(session)) {
                                        j1939_tp_schedule_txtimer(session, 0);
                                } else {
                                        ret = -EBUSY;
                                        session->err = ret;
                                        j1939_sk_queue_drop_all(priv, jsk,
                                                                EBUSY);
                                        break;
                                }
                        }
                } else {
                        skcb->offset = session->total_queued_size;
                        j1939_session_skb_queue(session, skb);
                }

                todo_size -= segment_size;
                session->total_queued_size += segment_size;
        } while (todo_size);

        switch (ret) {
        case 0: /* OK */
                if (todo_size)
                        netdev_warn(priv->ndev,
                                    "no error found and not completely queued?! %zu\n",
                                    todo_size);
                ret = size;
                break;
        case -ERESTARTSYS:
                ret = -EINTR;
                fallthrough;
        case -EAGAIN: /* OK */
                if (todo_size != size)
                        ret = size - todo_size;
                break;
        default: /* ERROR */
                break;
        }

        if (session)
                j1939_session_put(session);

        return ret;

 kfree_skb:
        kfree_skb(skb);
        return ret;
}

static int j1939_sk_sendmsg(struct socket *sock, struct msghdr *msg,
                            size_t size)
{
        struct sock *sk = sock->sk;
        struct j1939_sock *jsk = j1939_sk(sk);
        struct j1939_priv *priv;
        int ifindex;
        int ret;

        lock_sock(sock->sk);
        /* various socket state tests */
        if (!(jsk->state & J1939_SOCK_BOUND)) {
                ret = -EBADFD;
                goto sendmsg_done;
        }

        priv = jsk->priv;
        ifindex = jsk->ifindex;

        if (!jsk->addr.src_name && jsk->addr.sa == J1939_NO_ADDR) {
                /* no source address assigned yet */
                ret = -EBADFD;
                goto sendmsg_done;
        }

        /* deal with provided destination address info */
        if (msg->msg_name) {
                struct sockaddr_can *addr = msg->msg_name;

                if (msg->msg_namelen < J1939_MIN_NAMELEN) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (addr->can_family != AF_CAN) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (addr->can_ifindex && addr->can_ifindex != ifindex) {
                        ret = -EBADFD;
                        goto sendmsg_done;
                }

                if (j1939_pgn_is_valid(addr->can_addr.j1939.pgn) &&
                    !j1939_pgn_is_clean_pdu(addr->can_addr.j1939.pgn)) {
                        ret = -EINVAL;
                        goto sendmsg_done;
                }

                if (!addr->can_addr.j1939.name &&
                    addr->can_addr.j1939.addr == J1939_NO_ADDR &&
                    !sock_flag(sk, SOCK_BROADCAST)) {
                        /* broadcast, but SO_BROADCAST not set */
                        ret = -EACCES;
                        goto sendmsg_done;
                }
        } else {
                if (!jsk->addr.dst_name && jsk->addr.da == J1939_NO_ADDR &&
                    !sock_flag(sk, SOCK_BROADCAST)) {
                        /* broadcast, but SO_BROADCAST not set */
                        ret = -EACCES;
                        goto sendmsg_done;
                }
        }

        ret = j1939_sk_send_loop(priv, sk, msg, size);

sendmsg_done:
        release_sock(sock->sk);

        return ret;
}

void j1939_sk_netdev_event_netdown(struct j1939_priv *priv)
{
        struct j1939_sock *jsk;
        int error_code = ENETDOWN;

        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                jsk->sk.sk_err = error_code;
                if (!sock_flag(&jsk->sk, SOCK_DEAD))
                        sk_error_report(&jsk->sk);

                j1939_sk_queue_drop_all(priv, jsk, error_code);
        }
        read_unlock_bh(&priv->j1939_socks_lock);
}

void j1939_sk_netdev_event_unregister(struct j1939_priv *priv)
{
        struct sock *sk;
        struct j1939_sock *jsk;
        bool wait_rcu = false;

rescan: /* The caller is holding a ref on this "priv" via j1939_priv_get_by_ndev(). */
        read_lock_bh(&priv->j1939_socks_lock);
        list_for_each_entry(jsk, &priv->j1939_socks, list) {
                /* Skip if j1939_jsk_add() is not called on this socket. */
                if (!(jsk->state & J1939_SOCK_BOUND))
                        continue;
                sk = &jsk->sk;
                sock_hold(sk);
                read_unlock_bh(&priv->j1939_socks_lock);
                /* Check if j1939_jsk_del() is not yet called on this socket after holding
                 * socket's lock, for both j1939_sk_bind() and j1939_sk_release() call
                 * j1939_jsk_del() with socket's lock held.
                 */
                lock_sock(sk);
                if (jsk->state & J1939_SOCK_BOUND) {
                        /* Neither j1939_sk_bind() nor j1939_sk_release() called j1939_jsk_del().
                         * Make this socket no longer bound, by pretending as if j1939_sk_bind()
                         * dropped old references but did not get new references.
                         */
                        j1939_jsk_del(priv, jsk);
                        j1939_local_ecu_put(priv, jsk->addr.src_name, jsk->addr.sa);
                        j1939_netdev_stop(priv);
                        /* Call j1939_priv_put() now and prevent j1939_sk_sock_destruct() from
                         * calling the corresponding j1939_priv_put().
                         *
                         * j1939_sk_sock_destruct() is supposed to call j1939_priv_put() after
                         * an RCU grace period. But since the caller is holding a ref on this
                         * "priv", we can defer synchronize_rcu() until immediately before
                         * the caller calls j1939_priv_put().
                         */
                        j1939_priv_put(priv);
                        jsk->priv = NULL;
                        wait_rcu = true;
                }
                release_sock(sk);
                sock_put(sk);
                goto rescan;
        }
        read_unlock_bh(&priv->j1939_socks_lock);
        if (wait_rcu)
                synchronize_rcu();
}

static int j1939_sk_no_ioctlcmd(struct socket *sock, unsigned int cmd,
                                unsigned long arg)
{
        /* no ioctls for socket layer -> hand it down to NIC layer */
        return -ENOIOCTLCMD;
}

static const struct proto_ops j1939_ops = {
        .family = PF_CAN,
        .release = j1939_sk_release,
        .bind = j1939_sk_bind,
        .connect = j1939_sk_connect,
        .socketpair = sock_no_socketpair,
        .accept = sock_no_accept,
        .getname = j1939_sk_getname,
        .poll = datagram_poll,
        .ioctl = j1939_sk_no_ioctlcmd,
        .listen = sock_no_listen,
        .shutdown = sock_no_shutdown,
        .setsockopt = j1939_sk_setsockopt,
        .getsockopt = j1939_sk_getsockopt,
        .sendmsg = j1939_sk_sendmsg,
        .recvmsg = j1939_sk_recvmsg,
        .mmap = sock_no_mmap,
};

static struct proto j1939_proto __read_mostly = {
        .name = "CAN_J1939",
        .owner = THIS_MODULE,
        .obj_size = sizeof(struct j1939_sock),
        .init = j1939_sk_init,
};

const struct can_proto j1939_can_proto = {
        .type = SOCK_DGRAM,
        .protocol = CAN_J1939,
        .ops = &j1939_ops,
        .prot = &j1939_proto,
};








































































    2 

    4 



    5 



    2 

    4 





    1 






    1 

    3 




    4 

    1 

    3 




    5 









    1 































    5 

    5 




    5 





    4 

    4 












    2 







    1 





    3 
    2 






    2 


    2 








    3 


    3 










    4 













    5 

    2 

    1 

    2 

    3 

    5 



    1 



    5 

    4 

    2 

    1 

    3 

    2 

    3 

    4 





















    5 
    5 










    5 




    5 

    5 

    5 
    5 



    5 






    5 






























































   36 
   36 










    1 




    1 


   32 

    2 



   22 



    1 


    1 


    9 









   11 








    2 






























    1 


    1 



    1 

    7 



    2 
    6 
    1 


    5 
    1 

    1 


    1 
    5 

















    7 





    6 
    1 
    6 
    1 
    6 
    1 
    6 
    1 

    6 
    1 
    1 
    6 


    5 
    2 


    6 

    1 



    6 
    1 


























   32 





   32 






   27 
   27 


   27 
    1 
    1 
    1 



   27 



   36 
   36 


   32 
    4 






    5 
















    5 
    2 



    5 


    5 



    5 


    5 



    5 






    5 



    5 













    1 
    1 


    1 
    1 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/cls_flow.c                Generic flow classifier
 *
 * Copyright (c) 2007, 2008 Patrick McHardy <kaber@trash.net>
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/random.h>
#include <linux/pkt_cls.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/inet_sock.h>

#include <net/pkt_cls.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/flow_dissector.h>
#include <net/tc_wrapper.h>

#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <net/netfilter/nf_conntrack.h>
#endif

struct flow_head {
        struct list_head        filters;
        struct rcu_head                rcu;
};

struct flow_filter {
        struct list_head        list;
        struct tcf_exts                exts;
        struct tcf_ematch_tree        ematches;
        struct tcf_proto        *tp;
        struct timer_list        perturb_timer;
        u32                        perturb_period;
        u32                        handle;

        u32                        nkeys;
        u32                        keymask;
        u32                        mode;
        u32                        mask;
        u32                        xor;
        u32                        rshift;
        u32                        addend;
        u32                        divisor;
        u32                        baseclass;
        u32                        hashrnd;
        struct rcu_work                rwork;
};

static inline u32 addr_fold(void *addr)
{
        unsigned long a = (unsigned long)addr;

        return (a & 0xFFFFFFFF) ^ (BITS_PER_LONG > 32 ? a >> 32 : 0);
}

static u32 flow_get_src(const struct sk_buff *skb, const struct flow_keys *flow)
{
        __be32 src = flow_get_u32_src(flow);

        if (src)
                return ntohl(src);

        return addr_fold(skb->sk);
}

static u32 flow_get_dst(const struct sk_buff *skb, const struct flow_keys *flow)
{
        __be32 dst = flow_get_u32_dst(flow);

        if (dst)
                return ntohl(dst);

        return addr_fold(skb_dst(skb)) ^ (__force u16)skb_protocol(skb, true);
}

static u32 flow_get_proto(const struct sk_buff *skb,
                          const struct flow_keys *flow)
{
        return flow->basic.ip_proto;
}

static u32 flow_get_proto_src(const struct sk_buff *skb,
                              const struct flow_keys *flow)
{
        if (flow->ports.ports)
                return ntohs(flow->ports.src);

        return addr_fold(skb->sk);
}

static u32 flow_get_proto_dst(const struct sk_buff *skb,
                              const struct flow_keys *flow)
{
        if (flow->ports.ports)
                return ntohs(flow->ports.dst);

        return addr_fold(skb_dst(skb)) ^ (__force u16)skb_protocol(skb, true);
}

static u32 flow_get_iif(const struct sk_buff *skb)
{
        return skb->skb_iif;
}

static u32 flow_get_priority(const struct sk_buff *skb)
{
        return skb->priority;
}

static u32 flow_get_mark(const struct sk_buff *skb)
{
        return skb->mark;
}

static u32 flow_get_nfct(const struct sk_buff *skb)
{
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
        return addr_fold(skb_nfct(skb));
#else
        return 0;
#endif
}

#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#define CTTUPLE(skb, member)                                                \
({                                                                        \
        enum ip_conntrack_info ctinfo;                                        \
        const struct nf_conn *ct = nf_ct_get(skb, &ctinfo);                \
        if (ct == NULL)                                                        \
                goto fallback;                                                \
        ct->tuplehash[CTINFO2DIR(ctinfo)].tuple.member;                        \
})
#else
#define CTTUPLE(skb, member)                                                \
({                                                                        \
        goto fallback;                                                        \
        0;                                                                \
})
#endif

static u32 flow_get_nfct_src(const struct sk_buff *skb,
                             const struct flow_keys *flow)
{
        switch (skb_protocol(skb, true)) {
        case htons(ETH_P_IP):
                return ntohl(CTTUPLE(skb, src.u3.ip));
        case htons(ETH_P_IPV6):
                return ntohl(CTTUPLE(skb, src.u3.ip6[3]));
        }
fallback:
        return flow_get_src(skb, flow);
}

static u32 flow_get_nfct_dst(const struct sk_buff *skb,
                             const struct flow_keys *flow)
{
        switch (skb_protocol(skb, true)) {
        case htons(ETH_P_IP):
                return ntohl(CTTUPLE(skb, dst.u3.ip));
        case htons(ETH_P_IPV6):
                return ntohl(CTTUPLE(skb, dst.u3.ip6[3]));
        }
fallback:
        return flow_get_dst(skb, flow);
}

static u32 flow_get_nfct_proto_src(const struct sk_buff *skb,
                                   const struct flow_keys *flow)
{
        return ntohs(CTTUPLE(skb, src.u.all));
fallback:
        return flow_get_proto_src(skb, flow);
}

static u32 flow_get_nfct_proto_dst(const struct sk_buff *skb,
                                   const struct flow_keys *flow)
{
        return ntohs(CTTUPLE(skb, dst.u.all));
fallback:
        return flow_get_proto_dst(skb, flow);
}

static u32 flow_get_rtclassid(const struct sk_buff *skb)
{
#ifdef CONFIG_IP_ROUTE_CLASSID
        if (skb_dst(skb))
                return skb_dst(skb)->tclassid;
#endif
        return 0;
}

static u32 flow_get_skuid(const struct sk_buff *skb)
{
        struct sock *sk = skb_to_full_sk(skb);

        if (sk && sk->sk_socket && sk->sk_socket->file) {
                kuid_t skuid = sk->sk_socket->file->f_cred->fsuid;

                return from_kuid(&init_user_ns, skuid);
        }
        return 0;
}

static u32 flow_get_skgid(const struct sk_buff *skb)
{
        struct sock *sk = skb_to_full_sk(skb);

        if (sk && sk->sk_socket && sk->sk_socket->file) {
                kgid_t skgid = sk->sk_socket->file->f_cred->fsgid;

                return from_kgid(&init_user_ns, skgid);
        }
        return 0;
}

static u32 flow_get_vlan_tag(const struct sk_buff *skb)
{
        u16 tag;

        if (vlan_get_tag(skb, &tag) < 0)
                return 0;
        return tag & VLAN_VID_MASK;
}

static u32 flow_get_rxhash(struct sk_buff *skb)
{
        return skb_get_hash(skb);
}

static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow)
{
        switch (key) {
        case FLOW_KEY_SRC:
                return flow_get_src(skb, flow);
        case FLOW_KEY_DST:
                return flow_get_dst(skb, flow);
        case FLOW_KEY_PROTO:
                return flow_get_proto(skb, flow);
        case FLOW_KEY_PROTO_SRC:
                return flow_get_proto_src(skb, flow);
        case FLOW_KEY_PROTO_DST:
                return flow_get_proto_dst(skb, flow);
        case FLOW_KEY_IIF:
                return flow_get_iif(skb);
        case FLOW_KEY_PRIORITY:
                return flow_get_priority(skb);
        case FLOW_KEY_MARK:
                return flow_get_mark(skb);
        case FLOW_KEY_NFCT:
                return flow_get_nfct(skb);
        case FLOW_KEY_NFCT_SRC:
                return flow_get_nfct_src(skb, flow);
        case FLOW_KEY_NFCT_DST:
                return flow_get_nfct_dst(skb, flow);
        case FLOW_KEY_NFCT_PROTO_SRC:
                return flow_get_nfct_proto_src(skb, flow);
        case FLOW_KEY_NFCT_PROTO_DST:
                return flow_get_nfct_proto_dst(skb, flow);
        case FLOW_KEY_RTCLASSID:
                return flow_get_rtclassid(skb);
        case FLOW_KEY_SKUID:
                return flow_get_skuid(skb);
        case FLOW_KEY_SKGID:
                return flow_get_skgid(skb);
        case FLOW_KEY_VLAN_TAG:
                return flow_get_vlan_tag(skb);
        case FLOW_KEY_RXHASH:
                return flow_get_rxhash(skb);
        default:
                WARN_ON(1);
                return 0;
        }
}

#define FLOW_KEYS_NEEDED ((1 << FLOW_KEY_SRC) |                 \
                          (1 << FLOW_KEY_DST) |                        \
                          (1 << FLOW_KEY_PROTO) |                \
                          (1 << FLOW_KEY_PROTO_SRC) |                \
                          (1 << FLOW_KEY_PROTO_DST) |                 \
                          (1 << FLOW_KEY_NFCT_SRC) |                \
                          (1 << FLOW_KEY_NFCT_DST) |                \
                          (1 << FLOW_KEY_NFCT_PROTO_SRC) |        \
                          (1 << FLOW_KEY_NFCT_PROTO_DST))

TC_INDIRECT_SCOPE int flow_classify(struct sk_buff *skb,
                                    const struct tcf_proto *tp,
                                    struct tcf_result *res)
{
        struct flow_head *head = rcu_dereference_bh(tp->root);
        struct flow_filter *f;
        u32 keymask;
        u32 classid;
        unsigned int n, key;
        int r;

        list_for_each_entry_rcu(f, &head->filters, list) {
                u32 keys[FLOW_KEY_MAX + 1];
                struct flow_keys flow_keys;

                if (!tcf_em_tree_match(skb, &f->ematches, NULL))
                        continue;

                keymask = f->keymask;
                if (keymask & FLOW_KEYS_NEEDED)
                        skb_flow_dissect_flow_keys(skb, &flow_keys, 0);

                for (n = 0; n < f->nkeys; n++) {
                        key = ffs(keymask) - 1;
                        keymask &= ~(1 << key);
                        keys[n] = flow_key_get(skb, key, &flow_keys);
                }

                if (f->mode == FLOW_MODE_HASH)
                        classid = jhash2(keys, f->nkeys, f->hashrnd);
                else {
                        classid = keys[0];
                        classid = (classid & f->mask) ^ f->xor;
                        classid = (classid >> f->rshift) + f->addend;
                }

                if (f->divisor)
                        classid %= f->divisor;

                res->class   = 0;
                res->classid = TC_H_MAKE(f->baseclass, f->baseclass + classid);

                r = tcf_exts_exec(skb, &f->exts, res);
                if (r < 0)
                        continue;
                return r;
        }
        return -1;
}

static void flow_perturbation(struct timer_list *t)
{
        struct flow_filter *f = timer_container_of(f, t, perturb_timer);

        get_random_bytes(&f->hashrnd, 4);
        if (f->perturb_period)
                mod_timer(&f->perturb_timer, jiffies + f->perturb_period);
}

static const struct nla_policy flow_policy[TCA_FLOW_MAX + 1] = {
        [TCA_FLOW_KEYS]                = { .type = NLA_U32 },
        [TCA_FLOW_MODE]                = { .type = NLA_U32 },
        [TCA_FLOW_BASECLASS]        = { .type = NLA_U32 },
        [TCA_FLOW_RSHIFT]        = NLA_POLICY_MAX(NLA_U32,
                                                 31 /* BITS_PER_U32 - 1 */),
        [TCA_FLOW_ADDEND]        = { .type = NLA_U32 },
        [TCA_FLOW_MASK]                = { .type = NLA_U32 },
        [TCA_FLOW_XOR]                = { .type = NLA_U32 },
        [TCA_FLOW_DIVISOR]        = { .type = NLA_U32 },
        [TCA_FLOW_ACT]                = { .type = NLA_NESTED },
        [TCA_FLOW_POLICE]        = { .type = NLA_NESTED },
        [TCA_FLOW_EMATCHES]        = { .type = NLA_NESTED },
        [TCA_FLOW_PERTURB]        = { .type = NLA_U32 },
};

static void __flow_destroy_filter(struct flow_filter *f)
{
        timer_shutdown_sync(&f->perturb_timer);
        tcf_exts_destroy(&f->exts);
        tcf_em_tree_destroy(&f->ematches);
        tcf_exts_put_net(&f->exts);
        kfree(f);
}

static void flow_destroy_filter_work(struct work_struct *work)
{
        struct flow_filter *f = container_of(to_rcu_work(work),
                                             struct flow_filter,
                                             rwork);
        rtnl_lock();
        __flow_destroy_filter(f);
        rtnl_unlock();
}

static int flow_change(struct net *net, struct sk_buff *in_skb,
                       struct tcf_proto *tp, unsigned long base,
                       u32 handle, struct nlattr **tca,
                       void **arg, u32 flags,
                       struct netlink_ext_ack *extack)
{
        struct flow_head *head = rtnl_dereference(tp->root);
        struct flow_filter *fold, *fnew;
        struct nlattr *opt = tca[TCA_OPTIONS];
        struct nlattr *tb[TCA_FLOW_MAX + 1];
        unsigned int nkeys = 0;
        unsigned int perturb_period = 0;
        u32 baseclass = 0;
        u32 keymask = 0;
        u32 mode;
        int err;

        if (opt == NULL)
                return -EINVAL;

        err = nla_parse_nested_deprecated(tb, TCA_FLOW_MAX, opt, flow_policy,
                                          NULL);
        if (err < 0)
                return err;

        if (tb[TCA_FLOW_BASECLASS]) {
                baseclass = nla_get_u32(tb[TCA_FLOW_BASECLASS]);
                if (TC_H_MIN(baseclass) == 0)
                        return -EINVAL;
        }

        if (tb[TCA_FLOW_KEYS]) {
                keymask = nla_get_u32(tb[TCA_FLOW_KEYS]);

                nkeys = hweight32(keymask);
                if (nkeys == 0)
                        return -EINVAL;

                if (fls(keymask) - 1 > FLOW_KEY_MAX)
                        return -EOPNOTSUPP;

                if ((keymask & (FLOW_KEY_SKUID|FLOW_KEY_SKGID)) &&
                    sk_user_ns(NETLINK_CB(in_skb).sk) != &init_user_ns)
                        return -EOPNOTSUPP;
        }

        fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
        if (!fnew)
                return -ENOBUFS;

        err = tcf_em_tree_validate(tp, tb[TCA_FLOW_EMATCHES], &fnew->ematches);
        if (err < 0)
                goto err1;

        err = tcf_exts_init(&fnew->exts, net, TCA_FLOW_ACT, TCA_FLOW_POLICE);
        if (err < 0)
                goto err2;

        err = tcf_exts_validate(net, tp, tb, tca[TCA_RATE], &fnew->exts, flags,
                                extack);
        if (err < 0)
                goto err2;

        fold = *arg;
        if (fold) {
                err = -EINVAL;
                if (fold->handle != handle && handle)
                        goto err2;

                /* Copy fold into fnew */
                fnew->tp = fold->tp;
                fnew->handle = fold->handle;
                fnew->nkeys = fold->nkeys;
                fnew->keymask = fold->keymask;
                fnew->mode = fold->mode;
                fnew->mask = fold->mask;
                fnew->xor = fold->xor;
                fnew->rshift = fold->rshift;
                fnew->addend = fold->addend;
                fnew->divisor = fold->divisor;
                fnew->baseclass = fold->baseclass;
                fnew->hashrnd = fold->hashrnd;

                mode = fold->mode;
                if (tb[TCA_FLOW_MODE])
                        mode = nla_get_u32(tb[TCA_FLOW_MODE]);
                if (mode != FLOW_MODE_HASH && nkeys > 1)
                        goto err2;

                if (mode == FLOW_MODE_HASH)
                        perturb_period = fold->perturb_period;
                if (tb[TCA_FLOW_PERTURB]) {
                        if (mode != FLOW_MODE_HASH)
                                goto err2;
                        perturb_period = nla_get_u32(tb[TCA_FLOW_PERTURB]) * HZ;
                }
        } else {
                err = -EINVAL;
                if (!handle)
                        goto err2;
                if (!tb[TCA_FLOW_KEYS])
                        goto err2;

                mode = FLOW_MODE_MAP;
                if (tb[TCA_FLOW_MODE])
                        mode = nla_get_u32(tb[TCA_FLOW_MODE]);
                if (mode != FLOW_MODE_HASH && nkeys > 1)
                        goto err2;

                if (tb[TCA_FLOW_PERTURB]) {
                        if (mode != FLOW_MODE_HASH)
                                goto err2;
                        perturb_period = nla_get_u32(tb[TCA_FLOW_PERTURB]) * HZ;
                }

                if (TC_H_MAJ(baseclass) == 0) {
                        struct Qdisc *q = tcf_block_q(tp->chain->block);

                        baseclass = TC_H_MAKE(q->handle, baseclass);
                }
                if (TC_H_MIN(baseclass) == 0)
                        baseclass = TC_H_MAKE(baseclass, 1);

                fnew->handle = handle;
                fnew->mask  = ~0U;
                fnew->tp = tp;
                get_random_bytes(&fnew->hashrnd, 4);
        }

        timer_setup(&fnew->perturb_timer, flow_perturbation, TIMER_DEFERRABLE);

        tcf_block_netif_keep_dst(tp->chain->block);

        if (tb[TCA_FLOW_KEYS]) {
                fnew->keymask = keymask;
                fnew->nkeys   = nkeys;
        }

        fnew->mode = mode;

        if (tb[TCA_FLOW_MASK])
                fnew->mask = nla_get_u32(tb[TCA_FLOW_MASK]);
        if (tb[TCA_FLOW_XOR])
                fnew->xor = nla_get_u32(tb[TCA_FLOW_XOR]);
        if (tb[TCA_FLOW_RSHIFT])
                fnew->rshift = nla_get_u32(tb[TCA_FLOW_RSHIFT]);
        if (tb[TCA_FLOW_ADDEND])
                fnew->addend = nla_get_u32(tb[TCA_FLOW_ADDEND]);

        if (tb[TCA_FLOW_DIVISOR])
                fnew->divisor = nla_get_u32(tb[TCA_FLOW_DIVISOR]);
        if (baseclass)
                fnew->baseclass = baseclass;

        fnew->perturb_period = perturb_period;
        if (perturb_period)
                mod_timer(&fnew->perturb_timer, jiffies + perturb_period);

        if (!*arg)
                list_add_tail_rcu(&fnew->list, &head->filters);
        else
                list_replace_rcu(&fold->list, &fnew->list);

        *arg = fnew;

        if (fold) {
                tcf_exts_get_net(&fold->exts);
                tcf_queue_work(&fold->rwork, flow_destroy_filter_work);
        }
        return 0;

err2:
        tcf_exts_destroy(&fnew->exts);
        tcf_em_tree_destroy(&fnew->ematches);
err1:
        kfree(fnew);
        return err;
}

static int flow_delete(struct tcf_proto *tp, void *arg, bool *last,
                       bool rtnl_held, struct netlink_ext_ack *extack)
{
        struct flow_head *head = rtnl_dereference(tp->root);
        struct flow_filter *f = arg;

        list_del_rcu(&f->list);
        tcf_exts_get_net(&f->exts);
        tcf_queue_work(&f->rwork, flow_destroy_filter_work);
        *last = list_empty(&head->filters);
        return 0;
}

static int flow_init(struct tcf_proto *tp)
{
        struct flow_head *head;

        head = kzalloc(sizeof(*head), GFP_KERNEL);
        if (head == NULL)
                return -ENOBUFS;
        INIT_LIST_HEAD(&head->filters);
        rcu_assign_pointer(tp->root, head);
        return 0;
}

static void flow_destroy(struct tcf_proto *tp, bool rtnl_held,
                         struct netlink_ext_ack *extack)
{
        struct flow_head *head = rtnl_dereference(tp->root);
        struct flow_filter *f, *next;

        list_for_each_entry_safe(f, next, &head->filters, list) {
                list_del_rcu(&f->list);
                if (tcf_exts_get_net(&f->exts))
                        tcf_queue_work(&f->rwork, flow_destroy_filter_work);
                else
                        __flow_destroy_filter(f);
        }
        kfree_rcu(head, rcu);
}

static void *flow_get(struct tcf_proto *tp, u32 handle)
{
        struct flow_head *head = rtnl_dereference(tp->root);
        struct flow_filter *f;

        list_for_each_entry(f, &head->filters, list)
                if (f->handle == handle)
                        return f;
        return NULL;
}

static int flow_dump(struct net *net, struct tcf_proto *tp, void *fh,
                     struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
{
        struct flow_filter *f = fh;
        struct nlattr *nest;

        if (f == NULL)
                return skb->len;

        t->tcm_handle = f->handle;

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (nest == NULL)
                goto nla_put_failure;

        if (nla_put_u32(skb, TCA_FLOW_KEYS, f->keymask) ||
            nla_put_u32(skb, TCA_FLOW_MODE, f->mode))
                goto nla_put_failure;

        if (f->mask != ~0 || f->xor != 0) {
                if (nla_put_u32(skb, TCA_FLOW_MASK, f->mask) ||
                    nla_put_u32(skb, TCA_FLOW_XOR, f->xor))
                        goto nla_put_failure;
        }
        if (f->rshift &&
            nla_put_u32(skb, TCA_FLOW_RSHIFT, f->rshift))
                goto nla_put_failure;
        if (f->addend &&
            nla_put_u32(skb, TCA_FLOW_ADDEND, f->addend))
                goto nla_put_failure;

        if (f->divisor &&
            nla_put_u32(skb, TCA_FLOW_DIVISOR, f->divisor))
                goto nla_put_failure;
        if (f->baseclass &&
            nla_put_u32(skb, TCA_FLOW_BASECLASS, f->baseclass))
                goto nla_put_failure;

        if (f->perturb_period &&
            nla_put_u32(skb, TCA_FLOW_PERTURB, f->perturb_period / HZ))
                goto nla_put_failure;

        if (tcf_exts_dump(skb, &f->exts) < 0)
                goto nla_put_failure;
#ifdef CONFIG_NET_EMATCH
        if (f->ematches.hdr.nmatches &&
            tcf_em_tree_dump(skb, &f->ematches, TCA_FLOW_EMATCHES) < 0)
                goto nla_put_failure;
#endif
        nla_nest_end(skb, nest);

        if (tcf_exts_dump_stats(skb, &f->exts) < 0)
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static void flow_walk(struct tcf_proto *tp, struct tcf_walker *arg,
                      bool rtnl_held)
{
        struct flow_head *head = rtnl_dereference(tp->root);
        struct flow_filter *f;

        list_for_each_entry(f, &head->filters, list) {
                if (!tc_cls_stats_dump(tp, arg, f))
                        break;
        }
}

static struct tcf_proto_ops cls_flow_ops __read_mostly = {
        .kind                = "flow",
        .classify        = flow_classify,
        .init                = flow_init,
        .destroy        = flow_destroy,
        .change                = flow_change,
        .delete                = flow_delete,
        .get                = flow_get,
        .dump                = flow_dump,
        .walk                = flow_walk,
        .owner                = THIS_MODULE,
};
MODULE_ALIAS_NET_CLS("flow");

static int __init cls_flow_init(void)
{
        return register_tcf_proto_ops(&cls_flow_ops);
}

static void __exit cls_flow_exit(void)
{
        unregister_tcf_proto_ops(&cls_flow_ops);
}

module_init(cls_flow_init);
module_exit(cls_flow_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
MODULE_DESCRIPTION("TC flow classifier");




























































    5 




















   84 



   86 

   79 


    5 

    5 




    4 


    4 


    4 



    1 







    1 







    3 
    3 



    1 







    1 
    1 





    1 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  Copyright (C) 2004 IBM Corporation
 *
 *  Author: Serge Hallyn <serue@us.ibm.com>
 */

#include <linux/export.h>
#include <linux/uts.h>
#include <linux/utsname.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/cred.h>
#include <linux/user_namespace.h>
#include <linux/proc_ns.h>
#include <linux/nstree.h>
#include <linux/sched/task.h>

static struct kmem_cache *uts_ns_cache __ro_after_init;

static struct ucounts *inc_uts_namespaces(struct user_namespace *ns)
{
        return inc_ucount(ns, current_euid(), UCOUNT_UTS_NAMESPACES);
}

static void dec_uts_namespaces(struct ucounts *ucounts)
{
        dec_ucount(ucounts, UCOUNT_UTS_NAMESPACES);
}

/*
 * Clone a new ns copying an original utsname, setting refcount to 1
 * @old_ns: namespace to clone
 * Return ERR_PTR(-ENOMEM) on error (failure to allocate), new ns otherwise
 */
static struct uts_namespace *clone_uts_ns(struct user_namespace *user_ns,
                                          struct uts_namespace *old_ns)
{
        struct uts_namespace *ns;
        struct ucounts *ucounts;
        int err;

        err = -ENOSPC;
        ucounts = inc_uts_namespaces(user_ns);
        if (!ucounts)
                goto fail;

        err = -ENOMEM;
        ns = kmem_cache_zalloc(uts_ns_cache, GFP_KERNEL);
        if (!ns)
                goto fail_dec;

        err = ns_common_init(ns);
        if (err)
                goto fail_free;

        ns->ucounts = ucounts;
        down_read(&uts_sem);
        memcpy(&ns->name, &old_ns->name, sizeof(ns->name));
        ns->user_ns = get_user_ns(user_ns);
        up_read(&uts_sem);
        ns_tree_add(ns);
        return ns;

fail_free:
        kmem_cache_free(uts_ns_cache, ns);
fail_dec:
        dec_uts_namespaces(ucounts);
fail:
        return ERR_PTR(err);
}

/*
 * Copy task tsk's utsname namespace, or clone it if flags
 * specifies CLONE_NEWUTS.  In latter case, changes to the
 * utsname of this process won't be seen by parent, and vice
 * versa.
 */
struct uts_namespace *copy_utsname(u64 flags,
        struct user_namespace *user_ns, struct uts_namespace *old_ns)
{
        struct uts_namespace *new_ns;

        BUG_ON(!old_ns);
        get_uts_ns(old_ns);

        if (!(flags & CLONE_NEWUTS))
                return old_ns;

        new_ns = clone_uts_ns(user_ns, old_ns);

        put_uts_ns(old_ns);
        return new_ns;
}

void free_uts_ns(struct uts_namespace *ns)
{
        ns_tree_remove(ns);
        dec_uts_namespaces(ns->ucounts);
        put_user_ns(ns->user_ns);
        ns_common_free(ns);
        /* Concurrent nstree traversal depends on a grace period. */
        kfree_rcu(ns, ns.ns_rcu);
}

static struct ns_common *utsns_get(struct task_struct *task)
{
        struct uts_namespace *ns = NULL;
        struct nsproxy *nsproxy;

        task_lock(task);
        nsproxy = task->nsproxy;
        if (nsproxy) {
                ns = nsproxy->uts_ns;
                get_uts_ns(ns);
        }
        task_unlock(task);

        return ns ? &ns->ns : NULL;
}

static void utsns_put(struct ns_common *ns)
{
        put_uts_ns(to_uts_ns(ns));
}

static int utsns_install(struct nsset *nsset, struct ns_common *new)
{
        struct nsproxy *nsproxy = nsset->nsproxy;
        struct uts_namespace *ns = to_uts_ns(new);

        if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN) ||
            !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        get_uts_ns(ns);
        put_uts_ns(nsproxy->uts_ns);
        nsproxy->uts_ns = ns;
        return 0;
}

static struct user_namespace *utsns_owner(struct ns_common *ns)
{
        return to_uts_ns(ns)->user_ns;
}

const struct proc_ns_operations utsns_operations = {
        .name                = "uts",
        .get                = utsns_get,
        .put                = utsns_put,
        .install        = utsns_install,
        .owner                = utsns_owner,
};

void __init uts_ns_init(void)
{
        uts_ns_cache = kmem_cache_create_usercopy(
                        "uts_namespace", sizeof(struct uts_namespace), 0,
                        SLAB_PANIC|SLAB_ACCOUNT,
                        offsetof(struct uts_namespace, name),
                        sizeof_field(struct uts_namespace, name),
                        NULL);
        ns_tree_add(&init_uts_ns);
}























































    2 


    1 

    1 



    2 








































    3 



    2 






    1 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
// SPDX-License-Identifier: GPL-2.0-only
/*
 * (C) 2000-2001 Svenning Soerensen <svenning@post5.tele.dk>
 * Copyright (c) 2011 Patrick McHardy <kaber@trash.net>
 */

#include <linux/ip.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/ipv6.h>
#include <linux/netfilter.h>
#include <linux/netfilter_ipv4.h>
#include <linux/netfilter_ipv6.h>
#include <linux/netfilter/x_tables.h>
#include <net/netfilter/nf_nat.h>

static unsigned int
netmap_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
        const struct nf_nat_range2 *range = par->targinfo;
        struct nf_nat_range2 newrange;
        struct nf_conn *ct;
        enum ip_conntrack_info ctinfo;
        union nf_inet_addr new_addr, netmask;
        unsigned int i;

        ct = nf_ct_get(skb, &ctinfo);
        for (i = 0; i < ARRAY_SIZE(range->min_addr.ip6); i++)
                netmask.ip6[i] = ~(range->min_addr.ip6[i] ^
                                   range->max_addr.ip6[i]);

        if (xt_hooknum(par) == NF_INET_PRE_ROUTING ||
            xt_hooknum(par) == NF_INET_LOCAL_OUT)
                new_addr.in6 = ipv6_hdr(skb)->daddr;
        else
                new_addr.in6 = ipv6_hdr(skb)->saddr;

        for (i = 0; i < ARRAY_SIZE(new_addr.ip6); i++) {
                new_addr.ip6[i] &= ~netmask.ip6[i];
                new_addr.ip6[i] |= range->min_addr.ip6[i] &
                                   netmask.ip6[i];
        }

        newrange.flags        = range->flags | NF_NAT_RANGE_MAP_IPS;
        newrange.min_addr        = new_addr;
        newrange.max_addr        = new_addr;
        newrange.min_proto        = range->min_proto;
        newrange.max_proto        = range->max_proto;

        return nf_nat_setup_info(ct, &newrange, HOOK2MANIP(xt_hooknum(par)));
}

static int netmap_tg6_checkentry(const struct xt_tgchk_param *par)
{
        const struct nf_nat_range2 *range = par->targinfo;

        if (!(range->flags & NF_NAT_RANGE_MAP_IPS))
                return -EINVAL;
        return nf_ct_netns_get(par->net, par->family);
}

static void netmap_tg_destroy(const struct xt_tgdtor_param *par)
{
        nf_ct_netns_put(par->net, par->family);
}

static unsigned int
netmap_tg4(struct sk_buff *skb, const struct xt_action_param *par)
{
        struct nf_conn *ct;
        enum ip_conntrack_info ctinfo;
        __be32 new_ip, netmask;
        const struct nf_nat_ipv4_multi_range_compat *mr = par->targinfo;
        struct nf_nat_range2 newrange;

        WARN_ON(xt_hooknum(par) != NF_INET_PRE_ROUTING &&
                xt_hooknum(par) != NF_INET_POST_ROUTING &&
                xt_hooknum(par) != NF_INET_LOCAL_OUT &&
                xt_hooknum(par) != NF_INET_LOCAL_IN);
        ct = nf_ct_get(skb, &ctinfo);

        netmask = ~(mr->range[0].min_ip ^ mr->range[0].max_ip);

        if (xt_hooknum(par) == NF_INET_PRE_ROUTING ||
            xt_hooknum(par) == NF_INET_LOCAL_OUT)
                new_ip = ip_hdr(skb)->daddr & ~netmask;
        else
                new_ip = ip_hdr(skb)->saddr & ~netmask;
        new_ip |= mr->range[0].min_ip & netmask;

        memset(&newrange.min_addr, 0, sizeof(newrange.min_addr));
        memset(&newrange.max_addr, 0, sizeof(newrange.max_addr));
        newrange.flags             = mr->range[0].flags | NF_NAT_RANGE_MAP_IPS;
        newrange.min_addr.ip = new_ip;
        newrange.max_addr.ip = new_ip;
        newrange.min_proto   = mr->range[0].min;
        newrange.max_proto   = mr->range[0].max;

        /* Hand modified range to generic setup. */
        return nf_nat_setup_info(ct, &newrange, HOOK2MANIP(xt_hooknum(par)));
}

static int netmap_tg4_check(const struct xt_tgchk_param *par)
{
        const struct nf_nat_ipv4_multi_range_compat *mr = par->targinfo;

        if (!(mr->range[0].flags & NF_NAT_RANGE_MAP_IPS)) {
                pr_debug("bad MAP_IPS.\n");
                return -EINVAL;
        }
        if (mr->rangesize != 1) {
                pr_debug("bad rangesize %u.\n", mr->rangesize);
                return -EINVAL;
        }
        return nf_ct_netns_get(par->net, par->family);
}

static struct xt_target netmap_tg_reg[] __read_mostly = {
        {
                .name       = "NETMAP",
                .family     = NFPROTO_IPV6,
                .revision   = 0,
                .target     = netmap_tg6,
                .targetsize = sizeof(struct nf_nat_range),
                .table      = "nat",
                .hooks      = (1 << NF_INET_PRE_ROUTING) |
                              (1 << NF_INET_POST_ROUTING) |
                              (1 << NF_INET_LOCAL_OUT) |
                              (1 << NF_INET_LOCAL_IN),
                .checkentry = netmap_tg6_checkentry,
                .destroy    = netmap_tg_destroy,
                .me         = THIS_MODULE,
        },
        {
                .name       = "NETMAP",
                .family     = NFPROTO_IPV4,
                .revision   = 0,
                .target     = netmap_tg4,
                .targetsize = sizeof(struct nf_nat_ipv4_multi_range_compat),
                .table      = "nat",
                .hooks      = (1 << NF_INET_PRE_ROUTING) |
                              (1 << NF_INET_POST_ROUTING) |
                              (1 << NF_INET_LOCAL_OUT) |
                              (1 << NF_INET_LOCAL_IN),
                .checkentry = netmap_tg4_check,
                .destroy    = netmap_tg_destroy,
                .me         = THIS_MODULE,
        },
};

static int __init netmap_tg_init(void)
{
        return xt_register_targets(netmap_tg_reg, ARRAY_SIZE(netmap_tg_reg));
}

static void netmap_tg_exit(void)
{
        xt_unregister_targets(netmap_tg_reg, ARRAY_SIZE(netmap_tg_reg));
}

module_init(netmap_tg_init);
module_exit(netmap_tg_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Xtables: 1:1 NAT mapping of subnets");
MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
MODULE_ALIAS("ip6t_NETMAP");
MODULE_ALIAS("ipt_NETMAP");




































































































































































































































































































































































































































































































































































































































































































































































































































    5 

















    5 






















   74 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright(c) 2004-2005 Intel Corporation. All rights reserved.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/in.h>
#include <linux/sysfs.h>
#include <linux/ctype.h>
#include <linux/inet.h>
#include <linux/rtnetlink.h>
#include <linux/etherdevice.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <linux/nsproxy.h>

#include <net/bonding.h>

#define to_bond(cd)        ((struct bonding *)(netdev_priv(to_net_dev(cd))))

/* "show" function for the bond_masters attribute.
 * The class parameter is ignored.
 */
static ssize_t bonding_show_bonds(const struct class *cls,
                                  const struct class_attribute *attr,
                                  char *buf)
{
        const struct bond_net *bn =
                container_of_const(attr, struct bond_net, class_attr_bonding_masters);
        struct bonding *bond;
        int res = 0;

        rcu_read_lock();

        list_for_each_entry_rcu(bond, &bn->dev_list, bond_list) {
                if (res > (PAGE_SIZE - IFNAMSIZ)) {
                        /* not enough space for another interface name */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sysfs_emit_at(buf, res, "++more++ ");
                        break;
                }
                res += sysfs_emit_at(buf, res, "%s ", bond->dev->name);
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        rcu_read_unlock();
        return res;
}

static struct net_device *bond_get_by_name(const struct bond_net *bn, const char *ifname)
{
        struct bonding *bond;

        list_for_each_entry(bond, &bn->dev_list, bond_list) {
                if (strncmp(bond->dev->name, ifname, IFNAMSIZ) == 0)
                        return bond->dev;
        }
        return NULL;
}

/* "store" function for the bond_masters attribute.  This is what
 * creates and deletes entire bonds.
 *
 * The class parameter is ignored.
 */
static ssize_t bonding_store_bonds(const struct class *cls,
                                   const struct class_attribute *attr,
                                   const char *buffer, size_t count)
{
        const struct bond_net *bn =
                container_of_const(attr, struct bond_net, class_attr_bonding_masters);
        char command[IFNAMSIZ + 1] = {0, };
        char *ifname;
        int rv, res = count;

        sscanf(buffer, "%16s", command); /* IFNAMSIZ*/
        ifname = command + 1;
        if ((strlen(command) <= 1) ||
            !dev_valid_name(ifname))
                goto err_no_cmd;

        if (command[0] == '+') {
                pr_info("%s is being created...\n", ifname);
                rv = bond_create(bn->net, ifname);
                if (rv) {
                        if (rv == -EEXIST)
                                pr_info("%s already exists\n", ifname);
                        else
                                pr_info("%s creation failed\n", ifname);
                        res = rv;
                }
        } else if (command[0] == '-') {
                struct net_device *bond_dev;

                rtnl_lock();
                bond_dev = bond_get_by_name(bn, ifname);
                if (bond_dev) {
                        pr_info("%s is being deleted...\n", ifname);
                        unregister_netdevice(bond_dev);
                } else {
                        pr_err("unable to delete non-existent %s\n", ifname);
                        res = -ENODEV;
                }
                rtnl_unlock();
        } else
                goto err_no_cmd;

        /* Always return either count or an error.  If you return 0, you'll
         * get called forever, which is bad.
         */
        return res;

err_no_cmd:
        pr_err("no command found in bonding_masters - use +ifname or -ifname\n");
        return -EPERM;
}

/* class attribute for bond_masters file.  This ends up in /sys/class/net */
static const struct class_attribute class_attr_bonding_masters = {
        .attr = {
                .name = "bonding_masters",
                .mode = 0644,
        },
        .show = bonding_show_bonds,
        .store = bonding_store_bonds,
};

/* Generic "store" method for bonding sysfs option setting */
static ssize_t bonding_sysfs_store_option(struct device *d,
                                          struct device_attribute *attr,
                                          const char *buffer, size_t count)
{
        struct bonding *bond = to_bond(d);
        const struct bond_option *opt;
        char *buffer_clone;
        int ret;

        opt = bond_opt_get_by_name(attr->attr.name);
        if (WARN_ON(!opt))
                return -ENOENT;
        buffer_clone = kstrndup(buffer, count, GFP_KERNEL);
        if (!buffer_clone)
                return -ENOMEM;
        ret = bond_opt_tryset_rtnl(bond, opt->id, buffer_clone);
        if (!ret)
                ret = count;
        kfree(buffer_clone);

        return ret;
}

/* Show the slaves in the current bond. */
static ssize_t bonding_show_slaves(struct device *d,
                                   struct device_attribute *attr, char *buf)
{
        struct bonding *bond = to_bond(d);
        struct list_head *iter;
        struct slave *slave;
        int res = 0;

        rcu_read_lock();

        bond_for_each_slave_rcu(bond, slave, iter) {
                if (res > (PAGE_SIZE - IFNAMSIZ)) {
                        /* not enough space for another interface name */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sysfs_emit_at(buf, res, "++more++ ");
                        break;
                }
                res += sysfs_emit_at(buf, res, "%s ", slave->dev->name);
        }

        rcu_read_unlock();

        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        return res;
}
static DEVICE_ATTR(slaves, 0644, bonding_show_slaves,
                   bonding_sysfs_store_option);

/* Show the bonding mode. */
static ssize_t bonding_show_mode(struct device *d,
                                 struct device_attribute *attr, char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_MODE, BOND_MODE(bond));

        return sysfs_emit(buf, "%s %d\n", val->string, BOND_MODE(bond));
}
static DEVICE_ATTR(mode, 0644, bonding_show_mode, bonding_sysfs_store_option);

/* Show the bonding transmit hash method. */
static ssize_t bonding_show_xmit_hash(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_XMIT_HASH, bond->params.xmit_policy);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.xmit_policy);
}
static DEVICE_ATTR(xmit_hash_policy, 0644,
                   bonding_show_xmit_hash, bonding_sysfs_store_option);

/* Show arp_validate. */
static ssize_t bonding_show_arp_validate(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_ARP_VALIDATE,
                               bond->params.arp_validate);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.arp_validate);
}
static DEVICE_ATTR(arp_validate, 0644, bonding_show_arp_validate,
                   bonding_sysfs_store_option);

/* Show arp_all_targets. */
static ssize_t bonding_show_arp_all_targets(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_ARP_ALL_TARGETS,
                               bond->params.arp_all_targets);
        return sysfs_emit(buf, "%s %d\n",
                       val->string, bond->params.arp_all_targets);
}
static DEVICE_ATTR(arp_all_targets, 0644,
                   bonding_show_arp_all_targets, bonding_sysfs_store_option);

/* Show fail_over_mac. */
static ssize_t bonding_show_fail_over_mac(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_FAIL_OVER_MAC,
                               bond->params.fail_over_mac);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.fail_over_mac);
}
static DEVICE_ATTR(fail_over_mac, 0644,
                   bonding_show_fail_over_mac, bonding_sysfs_store_option);

/* Show the arp timer interval. */
static ssize_t bonding_show_arp_interval(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.arp_interval);
}
static DEVICE_ATTR(arp_interval, 0644,
                   bonding_show_arp_interval, bonding_sysfs_store_option);

/* Show the arp targets. */
static ssize_t bonding_show_arp_targets(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);
        int i, res = 0;

        for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) {
                if (bond->params.arp_targets[i])
                        res += sysfs_emit_at(buf, res, "%pI4 ",
                                             &bond->params.arp_targets[i]);
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        return res;
}
static DEVICE_ATTR(arp_ip_target, 0644,
                   bonding_show_arp_targets, bonding_sysfs_store_option);

/* Show the arp missed max. */
static ssize_t bonding_show_missed_max(struct device *d,
                                       struct device_attribute *attr,
                                       char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%u\n", bond->params.missed_max);
}
static DEVICE_ATTR(arp_missed_max, 0644,
                   bonding_show_missed_max, bonding_sysfs_store_option);

/* Show the up and down delays. */
static ssize_t bonding_show_downdelay(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.downdelay * bond->params.miimon);
}
static DEVICE_ATTR(downdelay, 0644,
                   bonding_show_downdelay, bonding_sysfs_store_option);

static ssize_t bonding_show_updelay(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.updelay * bond->params.miimon);

}
static DEVICE_ATTR(updelay, 0644,
                   bonding_show_updelay, bonding_sysfs_store_option);

static ssize_t bonding_show_peer_notif_delay(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n",
                          bond->params.peer_notif_delay * bond->params.miimon);
}
static DEVICE_ATTR(peer_notif_delay, 0644,
                   bonding_show_peer_notif_delay, bonding_sysfs_store_option);

/* Show the LACP activity and interval. */
static ssize_t bonding_show_lacp_active(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_LACP_ACTIVE, bond->params.lacp_active);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_active);
}
static DEVICE_ATTR(lacp_active, 0644,
                   bonding_show_lacp_active, bonding_sysfs_store_option);

static ssize_t bonding_show_lacp_rate(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_LACP_RATE, bond->params.lacp_fast);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_fast);
}
static DEVICE_ATTR(lacp_rate, 0644,
                   bonding_show_lacp_rate, bonding_sysfs_store_option);

static ssize_t bonding_show_min_links(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%u\n", bond->params.min_links);
}
static DEVICE_ATTR(min_links, 0644,
                   bonding_show_min_links, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_select(struct device *d,
                                      struct device_attribute *attr,
                                      char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_AD_SELECT, bond->params.ad_select);

        return sysfs_emit(buf, "%s %d\n", val->string, bond->params.ad_select);
}
static DEVICE_ATTR(ad_select, 0644,
                   bonding_show_ad_select, bonding_sysfs_store_option);

/* Show the number of peer notifications to send after a failover event. */
static ssize_t bonding_show_num_peer_notif(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.num_peer_notif);
}
static DEVICE_ATTR(num_grat_arp, 0644,
                   bonding_show_num_peer_notif, bonding_sysfs_store_option);
static DEVICE_ATTR(num_unsol_na, 0644,
                   bonding_show_num_peer_notif, bonding_sysfs_store_option);

/* Show the MII monitor interval. */
static ssize_t bonding_show_miimon(struct device *d,
                                   struct device_attribute *attr,
                                   char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.miimon);
}
static DEVICE_ATTR(miimon, 0644,
                   bonding_show_miimon, bonding_sysfs_store_option);

/* Show the primary slave. */
static ssize_t bonding_show_primary(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        struct bonding *bond = to_bond(d);
        struct slave *primary;
        int count = 0;

        rcu_read_lock();
        primary = rcu_dereference(bond->primary_slave);
        if (primary)
                count = sysfs_emit(buf, "%s\n", primary->dev->name);
        rcu_read_unlock();

        return count;
}
static DEVICE_ATTR(primary, 0644,
                   bonding_show_primary, bonding_sysfs_store_option);

/* Show the primary_reselect flag. */
static ssize_t bonding_show_primary_reselect(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);
        const struct bond_opt_value *val;

        val = bond_opt_get_val(BOND_OPT_PRIMARY_RESELECT,
                               bond->params.primary_reselect);

        return sysfs_emit(buf, "%s %d\n",
                          val->string, bond->params.primary_reselect);
}
static DEVICE_ATTR(primary_reselect, 0644,
                   bonding_show_primary_reselect, bonding_sysfs_store_option);

/* use_carrier is obsolete, but print value for compatibility */
static ssize_t bonding_show_carrier(struct device *d,
                                    struct device_attribute *attr,
                                    char *buf)
{
        return sysfs_emit(buf, "1\n");
}
static DEVICE_ATTR(use_carrier, 0644,
                   bonding_show_carrier, bonding_sysfs_store_option);


/* Show currently active_slave. */
static ssize_t bonding_show_active_slave(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct bonding *bond = to_bond(d);
        struct net_device *slave_dev;
        int count = 0;

        rcu_read_lock();
        slave_dev = bond_option_active_slave_get_rcu(bond);
        if (slave_dev)
                count = sysfs_emit(buf, "%s\n", slave_dev->name);
        rcu_read_unlock();

        return count;
}
static DEVICE_ATTR(active_slave, 0644,
                   bonding_show_active_slave, bonding_sysfs_store_option);

/* Show link status of the bond interface. */
static ssize_t bonding_show_mii_status(struct device *d,
                                       struct device_attribute *attr,
                                       char *buf)
{
        struct bonding *bond = to_bond(d);
        bool active = netif_carrier_ok(bond->dev);

        return sysfs_emit(buf, "%s\n", active ? "up" : "down");
}
static DEVICE_ATTR(mii_status, 0444, bonding_show_mii_status, NULL);

/* Show current 802.3ad aggregator ID. */
static ssize_t bonding_show_ad_aggregator(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct ad_info ad_info;

                count = sysfs_emit(buf, "%d\n",
                                   bond_3ad_get_active_agg_info(bond, &ad_info)
                                   ?  0 : ad_info.aggregator_id);
        }

        return count;
}
static DEVICE_ATTR(ad_aggregator, 0444, bonding_show_ad_aggregator, NULL);


/* Show number of active 802.3ad ports. */
static ssize_t bonding_show_ad_num_ports(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD) {
                struct ad_info ad_info;

                count = sysfs_emit(buf, "%d\n",
                                   bond_3ad_get_active_agg_info(bond, &ad_info)
                                   ?  0 : ad_info.ports);
        }

        return count;
}
static DEVICE_ATTR(ad_num_ports, 0444, bonding_show_ad_num_ports, NULL);


/* Show current 802.3ad actor key. */
static ssize_t bonding_show_ad_actor_key(struct device *d,
                                         struct device_attribute *attr,
                                         char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                count = sysfs_emit(buf, "%d\n",
                                   bond_3ad_get_active_agg_info(bond, &ad_info)
                                   ?  0 : ad_info.actor_key);
        }

        return count;
}
static DEVICE_ATTR(ad_actor_key, 0444, bonding_show_ad_actor_key, NULL);


/* Show current 802.3ad partner key. */
static ssize_t bonding_show_ad_partner_key(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                count = sysfs_emit(buf, "%d\n",
                                   bond_3ad_get_active_agg_info(bond, &ad_info)
                                   ?  0 : ad_info.partner_key);
        }

        return count;
}
static DEVICE_ATTR(ad_partner_key, 0444, bonding_show_ad_partner_key, NULL);


/* Show current 802.3ad partner mac. */
static ssize_t bonding_show_ad_partner_mac(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        int count = 0;
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
                struct ad_info ad_info;

                if (!bond_3ad_get_active_agg_info(bond, &ad_info))
                        count = sysfs_emit(buf, "%pM\n", ad_info.partner_system);
        }

        return count;
}
static DEVICE_ATTR(ad_partner_mac, 0444, bonding_show_ad_partner_mac, NULL);

/* Show the queue_ids of the slaves in the current bond. */
static ssize_t bonding_show_queue_id(struct device *d,
                                     struct device_attribute *attr,
                                     char *buf)
{
        struct bonding *bond = to_bond(d);
        struct list_head *iter;
        struct slave *slave;
        int res = 0;

        rcu_read_lock();

        bond_for_each_slave_rcu(bond, slave, iter) {
                if (res > (PAGE_SIZE - IFNAMSIZ - 6)) {
                        /* not enough space for another interface_name:queue_id pair */
                        if ((PAGE_SIZE - res) > 10)
                                res = PAGE_SIZE - 10;
                        res += sysfs_emit_at(buf, res, "++more++ ");
                        break;
                }
                res += sysfs_emit_at(buf, res, "%s:%d ",
                                     slave->dev->name,
                                     READ_ONCE(slave->queue_id));
        }
        if (res)
                buf[res-1] = '\n'; /* eat the leftover space */

        rcu_read_unlock();

        return res;
}
static DEVICE_ATTR(queue_id, 0644, bonding_show_queue_id,
                   bonding_sysfs_store_option);


/* Show the all_slaves_active flag. */
static ssize_t bonding_show_slaves_active(struct device *d,
                                          struct device_attribute *attr,
                                          char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.all_slaves_active);
}
static DEVICE_ATTR(all_slaves_active, 0644,
                   bonding_show_slaves_active, bonding_sysfs_store_option);

/* Show the number of IGMP membership reports to send on link failure */
static ssize_t bonding_show_resend_igmp(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.resend_igmp);
}
static DEVICE_ATTR(resend_igmp, 0644,
                   bonding_show_resend_igmp, bonding_sysfs_store_option);


static ssize_t bonding_show_lp_interval(struct device *d,
                                        struct device_attribute *attr,
                                        char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.lp_interval);
}
static DEVICE_ATTR(lp_interval, 0644,
                   bonding_show_lp_interval, bonding_sysfs_store_option);

static ssize_t bonding_show_tlb_dynamic_lb(struct device *d,
                                           struct device_attribute *attr,
                                           char *buf)
{
        struct bonding *bond = to_bond(d);

        return sysfs_emit(buf, "%d\n", bond->params.tlb_dynamic_lb);
}
static DEVICE_ATTR(tlb_dynamic_lb, 0644,
                   bonding_show_tlb_dynamic_lb, bonding_sysfs_store_option);

static ssize_t bonding_show_packets_per_slave(struct device *d,
                                              struct device_attribute *attr,
                                              char *buf)
{
        struct bonding *bond = to_bond(d);
        unsigned int packets_per_slave = bond->params.packets_per_slave;

        return sysfs_emit(buf, "%u\n", packets_per_slave);
}
static DEVICE_ATTR(packets_per_slave, 0644,
                   bonding_show_packets_per_slave, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_actor_sys_prio(struct device *d,
                                              struct device_attribute *attr,
                                              char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sysfs_emit(buf, "%hu\n", bond->params.ad_actor_sys_prio);

        return 0;
}
static DEVICE_ATTR(ad_actor_sys_prio, 0644,
                   bonding_show_ad_actor_sys_prio, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_actor_system(struct device *d,
                                            struct device_attribute *attr,
                                            char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sysfs_emit(buf, "%pM\n", bond->params.ad_actor_system);

        return 0;
}

static DEVICE_ATTR(ad_actor_system, 0644,
                   bonding_show_ad_actor_system, bonding_sysfs_store_option);

static ssize_t bonding_show_ad_user_port_key(struct device *d,
                                             struct device_attribute *attr,
                                             char *buf)
{
        struct bonding *bond = to_bond(d);

        if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
                return sysfs_emit(buf, "%hu\n", bond->params.ad_user_port_key);

        return 0;
}
static DEVICE_ATTR(ad_user_port_key, 0644,
                   bonding_show_ad_user_port_key, bonding_sysfs_store_option);

static struct attribute *per_bond_attrs[] = {
        &dev_attr_slaves.attr,
        &dev_attr_mode.attr,
        &dev_attr_fail_over_mac.attr,
        &dev_attr_arp_validate.attr,
        &dev_attr_arp_all_targets.attr,
        &dev_attr_arp_interval.attr,
        &dev_attr_arp_ip_target.attr,
        &dev_attr_downdelay.attr,
        &dev_attr_updelay.attr,
        &dev_attr_peer_notif_delay.attr,
        &dev_attr_lacp_active.attr,
        &dev_attr_lacp_rate.attr,
        &dev_attr_ad_select.attr,
        &dev_attr_xmit_hash_policy.attr,
        &dev_attr_num_grat_arp.attr,
        &dev_attr_num_unsol_na.attr,
        &dev_attr_miimon.attr,
        &dev_attr_primary.attr,
        &dev_attr_primary_reselect.attr,
        &dev_attr_use_carrier.attr,
        &dev_attr_active_slave.attr,
        &dev_attr_mii_status.attr,
        &dev_attr_ad_aggregator.attr,
        &dev_attr_ad_num_ports.attr,
        &dev_attr_ad_actor_key.attr,
        &dev_attr_ad_partner_key.attr,
        &dev_attr_ad_partner_mac.attr,
        &dev_attr_queue_id.attr,
        &dev_attr_all_slaves_active.attr,
        &dev_attr_resend_igmp.attr,
        &dev_attr_min_links.attr,
        &dev_attr_lp_interval.attr,
        &dev_attr_packets_per_slave.attr,
        &dev_attr_tlb_dynamic_lb.attr,
        &dev_attr_ad_actor_sys_prio.attr,
        &dev_attr_ad_actor_system.attr,
        &dev_attr_ad_user_port_key.attr,
        &dev_attr_arp_missed_max.attr,
        NULL,
};

static const struct attribute_group bonding_group = {
        .name = "bonding",
        .attrs = per_bond_attrs,
};

/* Initialize sysfs.  This sets up the bonding_masters file in
 * /sys/class/net.
 */
int __net_init bond_create_sysfs(struct bond_net *bn)
{
        int ret;

        bn->class_attr_bonding_masters = class_attr_bonding_masters;
        sysfs_attr_init(&bn->class_attr_bonding_masters.attr);

        ret = netdev_class_create_file_ns(&bn->class_attr_bonding_masters,
                                          bn->net);
        /* Permit multiple loads of the module by ignoring failures to
         * create the bonding_masters sysfs file.  Bonding devices
         * created by second or subsequent loads of the module will
         * not be listed in, or controllable by, bonding_masters, but
         * will have the usual "bonding" sysfs directory.
         *
         * This is done to preserve backwards compatibility for
         * initscripts/sysconfig, which load bonding multiple times to
         * configure multiple bonding devices.
         */
        if (ret == -EEXIST) {
                /* Is someone being kinky and naming a device bonding_master? */
                if (netdev_name_in_use(bn->net,
                                       class_attr_bonding_masters.attr.name))
                        pr_err("network device named %s already exists in sysfs\n",
                               class_attr_bonding_masters.attr.name);
                ret = 0;
        }

        return ret;

}

/* Remove /sys/class/net/bonding_masters. */
void __net_exit bond_destroy_sysfs(struct bond_net *bn)
{
        netdev_class_remove_file_ns(&bn->class_attr_bonding_masters, bn->net);
}

/* Initialize sysfs for each bond.  This sets up and registers
 * the 'bondctl' directory for each individual bond under /sys/class/net.
 */
void bond_prepare_sysfs_group(struct bonding *bond)
{
        bond->dev->sysfs_groups[0] = &bonding_group;
}







































































    3 









    3 





    3 






    3 

    3 



















    3 








    3 





    3 












    3 




    3 













    3 


























    3 










    3 




    3 

    3 












    1 



    1 















    1 





    1 






    2 




























































    3 






    4 












    3 










    3 









































    1 




    1 

    1 



    1 





    2 
    3 


























    4 
    4 
    4 
    4 
    4 
    4 
    4 
    4 






    4 









    4 









    4 




    4 



    3 



    3 

    3 

    3 




































    7 













    8 








    8 


























    2 
    2 

    4 










    4 







    3 



























































    1 



    1 









    1 

















    1 








    1 











    2 







    2 











    2 









    1 
    1 

































    3 



    3 




















    3 
































    2 




















    3 























    2 
















    2 




    2 











    1 
















    1 





    1 












    1 





























































































































































































    1 










    1 


























































    1 




    1 


    1 








    1 





    1 




    7 















    3 























































    1 




    1 




















    1 
    1 










    1 






























    1 








    1 





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
 */

#include <linux/device.h>
#include <net/genetlink.h>
#include <net/sock.h>
#include "devl_internal.h"

struct devlink_info_req {
        struct sk_buff *msg;
        void (*version_cb)(const char *version_name,
                           enum devlink_info_version_type version_type,
                           void *version_cb_priv);
        void *version_cb_priv;
};

struct devlink_reload_combination {
        enum devlink_reload_action action;
        enum devlink_reload_limit limit;
};

static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
        {
                /* can't reinitialize driver with no down time */
                .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
                .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
        },
};

static bool
devlink_reload_combination_is_invalid(enum devlink_reload_action action,
                                      enum devlink_reload_limit limit)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
                if (devlink_reload_invalid_combinations[i].action == action &&
                    devlink_reload_invalid_combinations[i].limit == limit)
                        return true;
        return false;
}

static bool
devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
{
        return test_bit(action, &devlink->ops->reload_actions);
}

static bool
devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
{
        return test_bit(limit, &devlink->ops->reload_limits);
}

static int devlink_reload_stat_put(struct sk_buff *msg,
                                   enum devlink_reload_limit limit, u32 value)
{
        struct nlattr *reload_stats_entry;

        reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
        if (!reload_stats_entry)
                return -EMSGSIZE;

        if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
            nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
                goto nla_put_failure;
        nla_nest_end(msg, reload_stats_entry);
        return 0;

nla_put_failure:
        nla_nest_cancel(msg, reload_stats_entry);
        return -EMSGSIZE;
}

static int
devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
{
        struct nlattr *reload_stats_attr, *act_info, *act_stats;
        int i, j, stat_idx;
        u32 value;

        if (!is_remote)
                reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
        else
                reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);

        if (!reload_stats_attr)
                return -EMSGSIZE;

        for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
                if ((!is_remote &&
                     !devlink_reload_action_is_supported(devlink, i)) ||
                    i == DEVLINK_RELOAD_ACTION_UNSPEC)
                        continue;
                act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
                if (!act_info)
                        goto nla_put_failure;

                if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
                        goto action_info_nest_cancel;
                act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
                if (!act_stats)
                        goto action_info_nest_cancel;

                for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
                        /* Remote stats are shown even if not locally supported.
                         * Stats of actions with unspecified limit are shown
                         * though drivers don't need to register unspecified
                         * limit.
                         */
                        if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
                             !devlink_reload_limit_is_supported(devlink, j)) ||
                            devlink_reload_combination_is_invalid(i, j))
                                continue;

                        stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
                        if (!is_remote)
                                value = devlink->stats.reload_stats[stat_idx];
                        else
                                value = devlink->stats.remote_reload_stats[stat_idx];
                        if (devlink_reload_stat_put(msg, j, value))
                                goto action_stats_nest_cancel;
                }
                nla_nest_end(msg, act_stats);
                nla_nest_end(msg, act_info);
        }
        nla_nest_end(msg, reload_stats_attr);
        return 0;

action_stats_nest_cancel:
        nla_nest_cancel(msg, act_stats);
action_info_nest_cancel:
        nla_nest_cancel(msg, act_info);
nla_put_failure:
        nla_nest_cancel(msg, reload_stats_attr);
        return -EMSGSIZE;
}

static int devlink_nl_nested_fill(struct sk_buff *msg, struct devlink *devlink)
{
        unsigned long rel_index;
        void *unused;
        int err;

        xa_for_each(&devlink->nested_rels, rel_index, unused) {
                err = devlink_rel_devlink_handle_put(msg, devlink,
                                                     rel_index,
                                                     DEVLINK_ATTR_NESTED_DEVLINK,
                                                     NULL);
                if (err)
                        return err;
        }
        return 0;
}

static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
                           enum devlink_command cmd, u32 portid,
                           u32 seq, int flags)
{
        struct nlattr *dev_stats;
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
        if (!hdr)
                return -EMSGSIZE;

        if (devlink_nl_put_handle(msg, devlink))
                goto nla_put_failure;
        if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
                goto nla_put_failure;

        dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
        if (!dev_stats)
                goto nla_put_failure;

        if (devlink_reload_stats_put(msg, devlink, false))
                goto dev_stats_nest_cancel;
        if (devlink_reload_stats_put(msg, devlink, true))
                goto dev_stats_nest_cancel;

        nla_nest_end(msg, dev_stats);

        if (devlink_nl_nested_fill(msg, devlink))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

dev_stats_nest_cancel:
        nla_nest_cancel(msg, dev_stats);
nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
{
        struct sk_buff *msg;
        int err;

        WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
        WARN_ON(!devl_is_registered(devlink));

        if (!devlink_nl_notify_need(devlink))
                return;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return;

        err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
        if (err) {
                nlmsg_free(msg);
                return;
        }

        devlink_nl_notify_send(devlink, msg);
}

int devlink_nl_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        struct sk_buff *msg;
        int err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
                              info->snd_portid, info->snd_seq, 0);
        if (err) {
                nlmsg_free(msg);
                return err;
        }

        return genlmsg_reply(msg, info);
}

static int
devlink_nl_get_dump_one(struct sk_buff *msg, struct devlink *devlink,
                        struct netlink_callback *cb, int flags)
{
        return devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
                               NETLINK_CB(cb->skb).portid,
                               cb->nlh->nlmsg_seq, flags);
}

int devlink_nl_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
{
        return devlink_nl_dumpit(msg, cb, devlink_nl_get_dump_one);
}

static void devlink_rel_notify_cb(struct devlink *devlink, u32 obj_index)
{
        devlink_notify(devlink, DEVLINK_CMD_NEW);
}

static void devlink_rel_cleanup_cb(struct devlink *devlink, u32 obj_index,
                                   u32 rel_index)
{
        xa_erase(&devlink->nested_rels, rel_index);
}

int devl_nested_devlink_set(struct devlink *devlink,
                            struct devlink *nested_devlink)
{
        u32 rel_index;
        int err;

        err = devlink_rel_nested_in_add(&rel_index, devlink->index, 0,
                                        devlink_rel_notify_cb,
                                        devlink_rel_cleanup_cb,
                                        nested_devlink);
        if (err)
                return err;
        return xa_insert(&devlink->nested_rels, rel_index,
                         xa_mk_value(0), GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(devl_nested_devlink_set);

void devlink_notify_register(struct devlink *devlink)
{
        devlink_notify(devlink, DEVLINK_CMD_NEW);
        devlink_linecards_notify_register(devlink);
        devlink_ports_notify_register(devlink);
        devlink_trap_policers_notify_register(devlink);
        devlink_trap_groups_notify_register(devlink);
        devlink_traps_notify_register(devlink);
        devlink_rates_notify_register(devlink);
        devlink_regions_notify_register(devlink);
        devlink_params_notify_register(devlink);
}

void devlink_notify_unregister(struct devlink *devlink)
{
        devlink_params_notify_unregister(devlink);
        devlink_regions_notify_unregister(devlink);
        devlink_rates_notify_unregister(devlink);
        devlink_traps_notify_unregister(devlink);
        devlink_trap_groups_notify_unregister(devlink);
        devlink_trap_policers_notify_unregister(devlink);
        devlink_ports_notify_unregister(devlink);
        devlink_linecards_notify_unregister(devlink);
        devlink_notify(devlink, DEVLINK_CMD_DEL);
}

static void devlink_reload_failed_set(struct devlink *devlink,
                                      bool reload_failed)
{
        if (devlink->reload_failed == reload_failed)
                return;
        devlink->reload_failed = reload_failed;
        devlink_notify(devlink, DEVLINK_CMD_NEW);
}

bool devlink_is_reload_failed(const struct devlink *devlink)
{
        return devlink->reload_failed;
}
EXPORT_SYMBOL_GPL(devlink_is_reload_failed);

static void
__devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
                              enum devlink_reload_limit limit, u32 actions_performed)
{
        unsigned long actions = actions_performed;
        int stat_idx;
        int action;

        for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
                stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
                reload_stats[stat_idx]++;
        }
        devlink_notify(devlink, DEVLINK_CMD_NEW);
}

static void
devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
                            u32 actions_performed)
{
        __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
                                      actions_performed);
}

/**
 *        devlink_remote_reload_actions_performed - Update devlink on reload actions
 *          performed which are not a direct result of devlink reload call.
 *
 *        This should be called by a driver after performing reload actions in case it was not
 *        a result of devlink reload call. For example fw_activate was performed as a result
 *        of devlink reload triggered fw_activate on another host.
 *        The motivation for this function is to keep data on reload actions performed on this
 *        function whether it was done due to direct devlink reload call or not.
 *
 *        @devlink: devlink
 *        @limit: reload limit
 *        @actions_performed: bitmask of actions performed
 */
void devlink_remote_reload_actions_performed(struct devlink *devlink,
                                             enum devlink_reload_limit limit,
                                             u32 actions_performed)
{
        if (WARN_ON(!actions_performed ||
                    actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
                    actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
                    limit > DEVLINK_RELOAD_LIMIT_MAX))
                return;

        __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
                                      actions_performed);
}
EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);

static struct net *devlink_netns_get(struct sk_buff *skb,
                                     struct genl_info *info)
{
        struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
        struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
        struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
        struct net *net;

        if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
                NL_SET_ERR_MSG(info->extack, "multiple netns identifying attributes specified");
                return ERR_PTR(-EINVAL);
        }

        if (netns_pid_attr) {
                net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
        } else if (netns_fd_attr) {
                net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
        } else if (netns_id_attr) {
                net = get_net_ns_by_id(sock_net(skb->sk),
                                       nla_get_u32(netns_id_attr));
                if (!net)
                        net = ERR_PTR(-EINVAL);
        } else {
                WARN_ON(1);
                net = ERR_PTR(-EINVAL);
        }
        if (IS_ERR(net)) {
                NL_SET_ERR_MSG(info->extack, "Unknown network namespace");
                return ERR_PTR(-EINVAL);
        }
        if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
                put_net(net);
                return ERR_PTR(-EPERM);
        }
        return net;
}

static void devlink_reload_netns_change(struct devlink *devlink,
                                        struct net *curr_net,
                                        struct net *dest_net)
{
        /* Userspace needs to be notified about devlink objects
         * removed from original and entering new network namespace.
         * The rest of the devlink objects are re-created during
         * reload process so the notifications are generated separatelly.
         */
        devlink_notify_unregister(devlink);
        write_pnet(&devlink->_net, dest_net);
        devlink_notify_register(devlink);
        devlink_rel_nested_in_notify(devlink);
}

static void devlink_reload_reinit_sanity_check(struct devlink *devlink)
{
        WARN_ON(!list_empty(&devlink->trap_policer_list));
        WARN_ON(!list_empty(&devlink->trap_group_list));
        WARN_ON(!list_empty(&devlink->trap_list));
        WARN_ON(!list_empty(&devlink->dpipe_table_list));
        WARN_ON(!list_empty(&devlink->sb_list));
        WARN_ON(!list_empty(&devlink->rate_list));
        WARN_ON(!list_empty(&devlink->linecard_list));
        WARN_ON(!xa_empty(&devlink->ports));
}

int devlink_reload(struct devlink *devlink, struct net *dest_net,
                   enum devlink_reload_action action,
                   enum devlink_reload_limit limit,
                   u32 *actions_performed, struct netlink_ext_ack *extack)
{
        u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
        struct net *curr_net;
        int err;

        /* Make sure the reload operations are invoked with the device lock
         * held to allow drivers to trigger functionality that expects it
         * (e.g., PCI reset) and to close possible races between these
         * operations and probe/remove.
         */
        device_lock_assert(devlink->dev);

        memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
               sizeof(remote_reload_stats));

        err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
        if (err)
                return err;

        curr_net = devlink_net(devlink);
        if (dest_net && !net_eq(dest_net, curr_net))
                devlink_reload_netns_change(devlink, curr_net, dest_net);

        if (action == DEVLINK_RELOAD_ACTION_DRIVER_REINIT) {
                devlink_params_driverinit_load_new(devlink);
                devlink_reload_reinit_sanity_check(devlink);
        }

        err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
        devlink_reload_failed_set(devlink, !!err);
        if (err)
                return err;

        WARN_ON(!(*actions_performed & BIT(action)));
        /* Catch driver on updating the remote action within devlink reload */
        WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
                       sizeof(remote_reload_stats)));
        devlink_reload_stats_update(devlink, limit, *actions_performed);
        return 0;
}

static int
devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
                                        enum devlink_command cmd, struct genl_info *info)
{
        struct sk_buff *msg;
        void *hdr;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
        if (!hdr)
                goto free_msg;

        if (devlink_nl_put_handle(msg, devlink))
                goto nla_put_failure;

        if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
                               actions_performed))
                goto nla_put_failure;
        genlmsg_end(msg, hdr);

        return genlmsg_reply(msg, info);

nla_put_failure:
        genlmsg_cancel(msg, hdr);
free_msg:
        nlmsg_free(msg);
        return -EMSGSIZE;
}

int devlink_nl_reload_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        enum devlink_reload_action action;
        enum devlink_reload_limit limit;
        struct net *dest_net = NULL;
        u32 actions_performed;
        int err;

        err = devlink_resources_validate(devlink, NULL, info);
        if (err) {
                NL_SET_ERR_MSG(info->extack, "resources size validation failed");
                return err;
        }

        action = nla_get_u8_default(info->attrs[DEVLINK_ATTR_RELOAD_ACTION],
                                    DEVLINK_RELOAD_ACTION_DRIVER_REINIT);

        if (!devlink_reload_action_is_supported(devlink, action)) {
                NL_SET_ERR_MSG(info->extack, "Requested reload action is not supported by the driver");
                return -EOPNOTSUPP;
        }

        limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
        if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
                struct nla_bitfield32 limits;
                u32 limits_selected;

                limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
                limits_selected = limits.value & limits.selector;
                if (!limits_selected) {
                        NL_SET_ERR_MSG(info->extack, "Invalid limit selected");
                        return -EINVAL;
                }
                for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
                        if (limits_selected & BIT(limit))
                                break;
                /* UAPI enables multiselection, but currently it is not used */
                if (limits_selected != BIT(limit)) {
                        NL_SET_ERR_MSG(info->extack, "Multiselection of limit is not supported");
                        return -EOPNOTSUPP;
                }
                if (!devlink_reload_limit_is_supported(devlink, limit)) {
                        NL_SET_ERR_MSG(info->extack, "Requested limit is not supported by the driver");
                        return -EOPNOTSUPP;
                }
                if (devlink_reload_combination_is_invalid(action, limit)) {
                        NL_SET_ERR_MSG(info->extack, "Requested limit is invalid for this action");
                        return -EINVAL;
                }
        }
        if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
            info->attrs[DEVLINK_ATTR_NETNS_FD] ||
            info->attrs[DEVLINK_ATTR_NETNS_ID]) {
                dest_net = devlink_netns_get(skb, info);
                if (IS_ERR(dest_net))
                        return PTR_ERR(dest_net);
                if (!net_eq(dest_net, devlink_net(devlink)) &&
                    action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT) {
                        NL_SET_ERR_MSG_MOD(info->extack,
                                           "Changing namespace is only supported for reinit action");
                        return -EOPNOTSUPP;
                }
        }

        err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);

        if (dest_net)
                put_net(dest_net);

        if (err)
                return err;
        /* For backward compatibility generate reply only if attributes used by user */
        if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
                return 0;

        return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
                                                       DEVLINK_CMD_RELOAD, info);
}

bool devlink_reload_actions_valid(const struct devlink_ops *ops)
{
        const struct devlink_reload_combination *comb;
        int i;

        if (!devlink_reload_supported(ops)) {
                if (WARN_ON(ops->reload_actions))
                        return false;
                return true;
        }

        if (WARN_ON(!ops->reload_actions ||
                    ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
                    ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
                return false;

        if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
                    ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
                return false;

        for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
                comb = &devlink_reload_invalid_combinations[i];
                if (ops->reload_actions == BIT(comb->action) &&
                    ops->reload_limits == BIT(comb->limit))
                        return false;
        }
        return true;
}

static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
                                   enum devlink_command cmd, u32 portid,
                                   u32 seq, int flags)
{
        const struct devlink_ops *ops = devlink->ops;
        enum devlink_eswitch_encap_mode encap_mode;
        u8 inline_mode;
        void *hdr;
        int err = 0;
        u16 mode;

        hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
        if (!hdr)
                return -EMSGSIZE;

        err = devlink_nl_put_handle(msg, devlink);
        if (err)
                goto nla_put_failure;

        if (ops->eswitch_mode_get) {
                err = ops->eswitch_mode_get(devlink, &mode);
                if (err)
                        goto nla_put_failure;
                err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
                if (err)
                        goto nla_put_failure;
        }

        if (ops->eswitch_inline_mode_get) {
                err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
                if (err)
                        goto nla_put_failure;
                err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
                                 inline_mode);
                if (err)
                        goto nla_put_failure;
        }

        if (ops->eswitch_encap_mode_get) {
                err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
                if (err)
                        goto nla_put_failure;
                err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
                if (err)
                        goto nla_put_failure;
        }

        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return err;
}

int devlink_nl_eswitch_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        struct sk_buff *msg;
        int err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
                                      info->snd_portid, info->snd_seq, 0);

        if (err) {
                nlmsg_free(msg);
                return err;
        }

        return genlmsg_reply(msg, info);
}

int devlink_nl_eswitch_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        const struct devlink_ops *ops = devlink->ops;
        enum devlink_eswitch_encap_mode encap_mode;
        u8 inline_mode;
        int err = 0;
        u16 mode;

        if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
                if (!ops->eswitch_mode_set)
                        return -EOPNOTSUPP;
                mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
                err = devlink_rate_nodes_check(devlink, mode, info->extack);
                if (err)
                        return err;
                err = ops->eswitch_mode_set(devlink, mode, info->extack);
                if (err)
                        return err;
        }

        if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
                if (!ops->eswitch_inline_mode_set)
                        return -EOPNOTSUPP;
                inline_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
                err = ops->eswitch_inline_mode_set(devlink, inline_mode,
                                                   info->extack);
                if (err)
                        return err;
        }

        if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
                if (!ops->eswitch_encap_mode_set)
                        return -EOPNOTSUPP;
                encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
                err = ops->eswitch_encap_mode_set(devlink, encap_mode,
                                                  info->extack);
                if (err)
                        return err;
        }

        return 0;
}

int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
{
        if (!req->msg)
                return 0;
        return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
}
EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);

int devlink_info_board_serial_number_put(struct devlink_info_req *req,
                                         const char *bsn)
{
        if (!req->msg)
                return 0;
        return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
                              bsn);
}
EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);

static int devlink_info_version_put(struct devlink_info_req *req, int attr,
                                    const char *version_name,
                                    const char *version_value,
                                    enum devlink_info_version_type version_type)
{
        struct nlattr *nest;
        int err;

        if (req->version_cb)
                req->version_cb(version_name, version_type,
                                req->version_cb_priv);

        if (!req->msg || !*version_value)
                return 0;

        nest = nla_nest_start_noflag(req->msg, attr);
        if (!nest)
                return -EMSGSIZE;

        err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
                             version_name);
        if (err)
                goto nla_put_failure;

        err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
                             version_value);
        if (err)
                goto nla_put_failure;

        nla_nest_end(req->msg, nest);

        return 0;

nla_put_failure:
        nla_nest_cancel(req->msg, nest);
        return err;
}

int devlink_info_version_fixed_put(struct devlink_info_req *req,
                                   const char *version_name,
                                   const char *version_value)
{
        return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
                                        version_name, version_value,
                                        DEVLINK_INFO_VERSION_TYPE_NONE);
}
EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);

int devlink_info_version_stored_put(struct devlink_info_req *req,
                                    const char *version_name,
                                    const char *version_value)
{
        return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
                                        version_name, version_value,
                                        DEVLINK_INFO_VERSION_TYPE_NONE);
}
EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);

int devlink_info_version_stored_put_ext(struct devlink_info_req *req,
                                        const char *version_name,
                                        const char *version_value,
                                        enum devlink_info_version_type version_type)
{
        return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
                                        version_name, version_value,
                                        version_type);
}
EXPORT_SYMBOL_GPL(devlink_info_version_stored_put_ext);

int devlink_info_version_running_put(struct devlink_info_req *req,
                                     const char *version_name,
                                     const char *version_value)
{
        return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
                                        version_name, version_value,
                                        DEVLINK_INFO_VERSION_TYPE_NONE);
}
EXPORT_SYMBOL_GPL(devlink_info_version_running_put);

int devlink_info_version_running_put_ext(struct devlink_info_req *req,
                                         const char *version_name,
                                         const char *version_value,
                                         enum devlink_info_version_type version_type)
{
        return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
                                        version_name, version_value,
                                        version_type);
}
EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext);

static int devlink_nl_driver_info_get(struct device_driver *drv,
                                      struct devlink_info_req *req)
{
        if (!drv)
                return 0;

        if (drv->name[0])
                return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME,
                                      drv->name);

        return 0;
}

static int
devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
                     enum devlink_command cmd, u32 portid,
                     u32 seq, int flags, struct netlink_ext_ack *extack)
{
        struct device *dev = devlink_to_dev(devlink);
        struct devlink_info_req req = {};
        void *hdr;
        int err;

        hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
        if (!hdr)
                return -EMSGSIZE;

        err = -EMSGSIZE;
        if (devlink_nl_put_handle(msg, devlink))
                goto err_cancel_msg;

        req.msg = msg;
        if (devlink->ops->info_get) {
                err = devlink->ops->info_get(devlink, &req, extack);
                if (err)
                        goto err_cancel_msg;
        }

        err = devlink_nl_driver_info_get(dev->driver, &req);
        if (err)
                goto err_cancel_msg;

        genlmsg_end(msg, hdr);
        return 0;

err_cancel_msg:
        genlmsg_cancel(msg, hdr);
        return err;
}

int devlink_nl_info_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        struct sk_buff *msg;
        int err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
                                   info->snd_portid, info->snd_seq, 0,
                                   info->extack);
        if (err) {
                nlmsg_free(msg);
                return err;
        }

        return genlmsg_reply(msg, info);
}

static int
devlink_nl_info_get_dump_one(struct sk_buff *msg, struct devlink *devlink,
                             struct netlink_callback *cb, int flags)
{
        int err;

        err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
                                   NETLINK_CB(cb->skb).portid,
                                   cb->nlh->nlmsg_seq, flags,
                                   cb->extack);
        if (err == -EOPNOTSUPP)
                err = 0;
        return err;
}

int devlink_nl_info_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb)
{
        return devlink_nl_dumpit(msg, cb, devlink_nl_info_get_dump_one);
}

static int devlink_nl_flash_update_fill(struct sk_buff *msg,
                                        struct devlink *devlink,
                                        enum devlink_command cmd,
                                        struct devlink_flash_notify *params)
{
        void *hdr;

        hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
        if (!hdr)
                return -EMSGSIZE;

        if (devlink_nl_put_handle(msg, devlink))
                goto nla_put_failure;

        if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
                goto out;

        if (params->status_msg &&
            nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
                           params->status_msg))
                goto nla_put_failure;
        if (params->component &&
            nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
                           params->component))
                goto nla_put_failure;
        if (devlink_nl_put_u64(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
                               params->done))
                goto nla_put_failure;
        if (devlink_nl_put_u64(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
                               params->total))
                goto nla_put_failure;
        if (devlink_nl_put_u64(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
                               params->timeout))
                goto nla_put_failure;

out:
        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static void __devlink_flash_update_notify(struct devlink *devlink,
                                          enum devlink_command cmd,
                                          struct devlink_flash_notify *params)
{
        struct sk_buff *msg;
        int err;

        WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
                cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
                cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);

        if (!devl_is_registered(devlink) || !devlink_nl_notify_need(devlink))
                return;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return;

        err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
        if (err)
                goto out_free_msg;

        devlink_nl_notify_send(devlink, msg);
        return;

out_free_msg:
        nlmsg_free(msg);
}

static void devlink_flash_update_begin_notify(struct devlink *devlink)
{
        struct devlink_flash_notify params = {};

        __devlink_flash_update_notify(devlink,
                                      DEVLINK_CMD_FLASH_UPDATE,
                                      &params);
}

static void devlink_flash_update_end_notify(struct devlink *devlink)
{
        struct devlink_flash_notify params = {};

        __devlink_flash_update_notify(devlink,
                                      DEVLINK_CMD_FLASH_UPDATE_END,
                                      &params);
}

void devlink_flash_update_status_notify(struct devlink *devlink,
                                        const char *status_msg,
                                        const char *component,
                                        unsigned long done,
                                        unsigned long total)
{
        struct devlink_flash_notify params = {
                .status_msg = status_msg,
                .component = component,
                .done = done,
                .total = total,
        };

        __devlink_flash_update_notify(devlink,
                                      DEVLINK_CMD_FLASH_UPDATE_STATUS,
                                      &params);
}
EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);

void devlink_flash_update_timeout_notify(struct devlink *devlink,
                                         const char *status_msg,
                                         const char *component,
                                         unsigned long timeout)
{
        struct devlink_flash_notify params = {
                .status_msg = status_msg,
                .component = component,
                .timeout = timeout,
        };

        __devlink_flash_update_notify(devlink,
                                      DEVLINK_CMD_FLASH_UPDATE_STATUS,
                                      &params);
}
EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);

struct devlink_flash_component_lookup_ctx {
        const char *lookup_name;
        bool lookup_name_found;
};

static void
devlink_flash_component_lookup_cb(const char *version_name,
                                  enum devlink_info_version_type version_type,
                                  void *version_cb_priv)
{
        struct devlink_flash_component_lookup_ctx *lookup_ctx = version_cb_priv;

        if (version_type != DEVLINK_INFO_VERSION_TYPE_COMPONENT ||
            lookup_ctx->lookup_name_found)
                return;

        lookup_ctx->lookup_name_found =
                !strcmp(lookup_ctx->lookup_name, version_name);
}

static int devlink_flash_component_get(struct devlink *devlink,
                                       struct nlattr *nla_component,
                                       const char **p_component,
                                       struct netlink_ext_ack *extack)
{
        struct devlink_flash_component_lookup_ctx lookup_ctx = {};
        struct devlink_info_req req = {};
        const char *component;
        int ret;

        if (!nla_component)
                return 0;

        component = nla_data(nla_component);

        if (!devlink->ops->info_get) {
                NL_SET_ERR_MSG_ATTR(extack, nla_component,
                                    "component update is not supported by this device");
                return -EOPNOTSUPP;
        }

        lookup_ctx.lookup_name = component;
        req.version_cb = devlink_flash_component_lookup_cb;
        req.version_cb_priv = &lookup_ctx;

        ret = devlink->ops->info_get(devlink, &req, NULL);
        if (ret)
                return ret;

        if (!lookup_ctx.lookup_name_found) {
                NL_SET_ERR_MSG_ATTR(extack, nla_component,
                                    "selected component is not supported by this device");
                return -EINVAL;
        }
        *p_component = component;
        return 0;
}

int devlink_nl_flash_update_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr *nla_overwrite_mask, *nla_file_name;
        struct devlink_flash_update_params params = {};
        struct devlink *devlink = info->user_ptr[0];
        const char *file_name;
        u32 supported_params;
        int ret;

        if (!devlink->ops->flash_update)
                return -EOPNOTSUPP;

        if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME))
                return -EINVAL;

        ret = devlink_flash_component_get(devlink,
                                          info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT],
                                          &params.component, info->extack);
        if (ret)
                return ret;

        supported_params = devlink->ops->supported_flash_update_params;

        nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
        if (nla_overwrite_mask) {
                struct nla_bitfield32 sections;

                if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
                        NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
                                            "overwrite settings are not supported by this device");
                        return -EOPNOTSUPP;
                }
                sections = nla_get_bitfield32(nla_overwrite_mask);
                params.overwrite_mask = sections.value & sections.selector;
        }

        nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
        file_name = nla_data(nla_file_name);
        ret = request_firmware(&params.fw, file_name, devlink->dev);
        if (ret) {
                NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name,
                                    "failed to locate the requested firmware file");
                return ret;
        }

        devlink_flash_update_begin_notify(devlink);
        ret = devlink->ops->flash_update(devlink, &params, info->extack);
        devlink_flash_update_end_notify(devlink);

        release_firmware(params.fw);

        return ret;
}

static void __devlink_compat_running_version(struct devlink *devlink,
                                             char *buf, size_t len)
{
        struct devlink_info_req req = {};
        const struct nlattr *nlattr;
        struct sk_buff *msg;
        int rem, err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return;

        req.msg = msg;
        err = devlink->ops->info_get(devlink, &req, NULL);
        if (err)
                goto free_msg;

        nla_for_each_attr_type(nlattr, DEVLINK_ATTR_INFO_VERSION_RUNNING,
                               (void *)msg->data, msg->len, rem) {
                const struct nlattr *kv;
                int rem_kv;

                nla_for_each_nested_type(kv, DEVLINK_ATTR_INFO_VERSION_VALUE,
                                         nlattr, rem_kv) {
                        strlcat(buf, nla_data(kv), len);
                        strlcat(buf, " ", len);
                }
        }
free_msg:
        nlmsg_consume(msg);
}

void devlink_compat_running_version(struct devlink *devlink,
                                    char *buf, size_t len)
{
        if (!devlink->ops->info_get)
                return;

        devl_lock(devlink);
        if (devl_is_registered(devlink))
                __devlink_compat_running_version(devlink, buf, len);
        devl_unlock(devlink);
}

int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
{
        struct devlink_flash_update_params params = {};
        int ret;

        devl_lock(devlink);
        if (!devl_is_registered(devlink)) {
                ret = -ENODEV;
                goto out_unlock;
        }

        if (!devlink->ops->flash_update) {
                ret = -EOPNOTSUPP;
                goto out_unlock;
        }

        ret = request_firmware(&params.fw, file_name, devlink->dev);
        if (ret)
                goto out_unlock;

        devlink_flash_update_begin_notify(devlink);
        ret = devlink->ops->flash_update(devlink, &params, NULL);
        devlink_flash_update_end_notify(devlink);

        release_firmware(params.fw);
out_unlock:
        devl_unlock(devlink);

        return ret;
}

static int
devlink_nl_selftests_fill(struct sk_buff *msg, struct devlink *devlink,
                          u32 portid, u32 seq, int flags,
                          struct netlink_ext_ack *extack)
{
        struct nlattr *selftests;
        void *hdr;
        int err;
        int i;

        hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags,
                          DEVLINK_CMD_SELFTESTS_GET);
        if (!hdr)
                return -EMSGSIZE;

        err = -EMSGSIZE;
        if (devlink_nl_put_handle(msg, devlink))
                goto err_cancel_msg;

        selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
        if (!selftests)
                goto err_cancel_msg;

        for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
             i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
                if (devlink->ops->selftest_check(devlink, i, extack)) {
                        err = nla_put_flag(msg, i);
                        if (err)
                                goto err_cancel_msg;
                }
        }

        nla_nest_end(msg, selftests);
        genlmsg_end(msg, hdr);
        return 0;

err_cancel_msg:
        genlmsg_cancel(msg, hdr);
        return err;
}

int devlink_nl_selftests_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink *devlink = info->user_ptr[0];
        struct sk_buff *msg;
        int err;

        if (!devlink->ops->selftest_check)
                return -EOPNOTSUPP;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = devlink_nl_selftests_fill(msg, devlink, info->snd_portid,
                                        info->snd_seq, 0, info->extack);
        if (err) {
                nlmsg_free(msg);
                return err;
        }

        return genlmsg_reply(msg, info);
}

static int devlink_nl_selftests_get_dump_one(struct sk_buff *msg,
                                             struct devlink *devlink,
                                             struct netlink_callback *cb,
                                             int flags)
{
        if (!devlink->ops->selftest_check)
                return 0;

        return devlink_nl_selftests_fill(msg, devlink,
                                         NETLINK_CB(cb->skb).portid,
                                         cb->nlh->nlmsg_seq, flags,
                                         cb->extack);
}

int devlink_nl_selftests_get_dumpit(struct sk_buff *skb,
                                    struct netlink_callback *cb)
{
        return devlink_nl_dumpit(skb, cb, devlink_nl_selftests_get_dump_one);
}

static int devlink_selftest_result_put(struct sk_buff *skb, unsigned int id,
                                       enum devlink_selftest_status test_status)
{
        struct nlattr *result_attr;

        result_attr = nla_nest_start(skb, DEVLINK_ATTR_SELFTEST_RESULT);
        if (!result_attr)
                return -EMSGSIZE;

        if (nla_put_u32(skb, DEVLINK_ATTR_SELFTEST_RESULT_ID, id) ||
            nla_put_u8(skb, DEVLINK_ATTR_SELFTEST_RESULT_STATUS,
                       test_status))
                goto nla_put_failure;

        nla_nest_end(skb, result_attr);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, result_attr);
        return -EMSGSIZE;
}

static const struct nla_policy devlink_selftest_nl_policy[DEVLINK_ATTR_SELFTEST_ID_MAX + 1] = {
        [DEVLINK_ATTR_SELFTEST_ID_FLASH] = { .type = NLA_FLAG },
};

int devlink_nl_selftests_run_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr *tb[DEVLINK_ATTR_SELFTEST_ID_MAX + 1];
        struct devlink *devlink = info->user_ptr[0];
        struct nlattr *attrs, *selftests;
        struct sk_buff *msg;
        void *hdr;
        int err;
        int i;

        if (!devlink->ops->selftest_run || !devlink->ops->selftest_check)
                return -EOPNOTSUPP;

        if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SELFTESTS))
                return -EINVAL;

        attrs = info->attrs[DEVLINK_ATTR_SELFTESTS];

        err = nla_parse_nested(tb, DEVLINK_ATTR_SELFTEST_ID_MAX, attrs,
                               devlink_selftest_nl_policy, info->extack);
        if (err < 0)
                return err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = -EMSGSIZE;
        hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
                          &devlink_nl_family, 0, DEVLINK_CMD_SELFTESTS_RUN);
        if (!hdr)
                goto free_msg;

        if (devlink_nl_put_handle(msg, devlink))
                goto genlmsg_cancel;

        selftests = nla_nest_start(msg, DEVLINK_ATTR_SELFTESTS);
        if (!selftests)
                goto genlmsg_cancel;

        for (i = DEVLINK_ATTR_SELFTEST_ID_UNSPEC + 1;
             i <= DEVLINK_ATTR_SELFTEST_ID_MAX; i++) {
                enum devlink_selftest_status test_status;

                if (nla_get_flag(tb[i])) {
                        if (!devlink->ops->selftest_check(devlink, i,
                                                          info->extack)) {
                                if (devlink_selftest_result_put(msg, i,
                                                                DEVLINK_SELFTEST_STATUS_SKIP))
                                        goto selftests_nest_cancel;
                                continue;
                        }

                        test_status = devlink->ops->selftest_run(devlink, i,
                                                                 info->extack);
                        if (devlink_selftest_result_put(msg, i, test_status))
                                goto selftests_nest_cancel;
                }
        }

        nla_nest_end(msg, selftests);
        genlmsg_end(msg, hdr);
        return genlmsg_reply(msg, info);

selftests_nest_cancel:
        nla_nest_cancel(msg, selftests);
genlmsg_cancel:
        genlmsg_cancel(msg, hdr);
free_msg:
        nlmsg_free(msg);
        return err;
}

















































































 12036 
 12038 











































 12046 

 12135 

 12139 




































































































































































































   38 




























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   35 















  158 


  156 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
// SPDX-License-Identifier: GPL-2.0-or-later
/* Common capabilities, needed by capability.o.
 */

#include <linux/capability.h>
#include <linux/audit.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/lsm_hooks.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/ptrace.h>
#include <linux/xattr.h>
#include <linux/hugetlb.h>
#include <linux/mount.h>
#include <linux/sched.h>
#include <linux/prctl.h>
#include <linux/securebits.h>
#include <linux/user_namespace.h>
#include <linux/binfmts.h>
#include <linux/personality.h>
#include <linux/mnt_idmapping.h>
#include <uapi/linux/lsm.h>

#define CREATE_TRACE_POINTS
#include <trace/events/capability.h>

/*
 * If a non-root user executes a setuid-root binary in
 * !secure(SECURE_NOROOT) mode, then we raise capabilities.
 * However if fE is also set, then the intent is for only
 * the file capabilities to be applied, and the setuid-root
 * bit is left on either to change the uid (plausible) or
 * to get full privilege on a kernel without file capabilities
 * support.  So in that case we do not raise capabilities.
 *
 * Warn if that happens, once per boot.
 */
static void warn_setuid_and_fcaps_mixed(const char *fname)
{
        static int warned;
        if (!warned) {
                printk(KERN_INFO "warning: `%s' has both setuid-root and"
                        " effective capabilities. Therefore not raising all"
                        " capabilities.\n", fname);
                warned = 1;
        }
}

/**
 * cap_capable_helper - Determine whether a task has a particular effective
 * capability.
 * @cred: The credentials to use
 * @target_ns:  The user namespace of the resource being accessed
 * @cred_ns:  The user namespace of the credentials
 * @cap: The capability to check for
 *
 * Determine whether the nominated task has the specified capability amongst
 * its effective set, returning 0 if it does, -ve if it does not.
 *
 * See cap_capable for more details.
 */
static inline int cap_capable_helper(const struct cred *cred,
                                     struct user_namespace *target_ns,
                                     const struct user_namespace *cred_ns,
                                     int cap)
{
        struct user_namespace *ns = target_ns;

        /* See if cred has the capability in the target user namespace
         * by examining the target user namespace and all of the target
         * user namespace's parents.
         */
        for (;;) {
                /* Do we have the necessary capabilities? */
                if (likely(ns == cred_ns))
                        return cap_raised(cred->cap_effective, cap) ? 0 : -EPERM;

                /*
                 * If we're already at a lower level than we're looking for,
                 * we're done searching.
                 */
                if (ns->level <= cred_ns->level)
                        return -EPERM;

                /* 
                 * The owner of the user namespace in the parent of the
                 * user namespace has all caps.
                 */
                if ((ns->parent == cred_ns) && uid_eq(ns->owner, cred->euid))
                        return 0;

                /*
                 * If you have a capability in a parent user ns, then you have
                 * it over all children user namespaces as well.
                 */
                ns = ns->parent;
        }

        /* We never get here */
}

/**
 * cap_capable - Determine whether a task has a particular effective capability
 * @cred: The credentials to use
 * @target_ns:  The user namespace of the resource being accessed
 * @cap: The capability to check for
 * @opts: Bitmask of options defined in include/linux/security.h (unused)
 *
 * Determine whether the nominated task has the specified capability amongst
 * its effective set, returning 0 if it does, -ve if it does not.
 *
 * NOTE WELL: cap_capable() has reverse semantics to the capable() call
 * and friends. That is cap_capable() returns an int 0 when a task has
 * a capability, while the kernel's capable(), has_ns_capability(),
 * has_ns_capability_noaudit(), and has_capability_noaudit() return a
 * bool true (1) for this case.
 */
int cap_capable(const struct cred *cred, struct user_namespace *target_ns,
                int cap, unsigned int opts)
{
        const struct user_namespace *cred_ns = cred->user_ns;
        int ret = cap_capable_helper(cred, target_ns, cred_ns, cap);

        trace_cap_capable(cred, target_ns, cred_ns, cap, ret);
        return ret;
}

/**
 * cap_settime - Determine whether the current process may set the system clock
 * @ts: The time to set
 * @tz: The timezone to set
 *
 * Determine whether the current process may set the system clock and timezone
 * information, returning 0 if permission granted, -ve if denied.
 */
int cap_settime(const struct timespec64 *ts, const struct timezone *tz)
{
        if (!capable(CAP_SYS_TIME))
                return -EPERM;
        return 0;
}

/**
 * cap_ptrace_access_check - Determine whether the current process may access
 *                           another
 * @child: The process to be accessed
 * @mode: The mode of attachment.
 *
 * If we are in the same or an ancestor user_ns and have all the target
 * task's capabilities, then ptrace access is allowed.
 * If we have the ptrace capability to the target user_ns, then ptrace
 * access is allowed.
 * Else denied.
 *
 * Determine whether a process may access another, returning 0 if permission
 * granted, -ve if denied.
 */
int cap_ptrace_access_check(struct task_struct *child, unsigned int mode)
{
        int ret = 0;
        const struct cred *cred, *child_cred;
        const kernel_cap_t *caller_caps;

        rcu_read_lock();
        cred = current_cred();
        child_cred = __task_cred(child);
        if (mode & PTRACE_MODE_FSCREDS)
                caller_caps = &cred->cap_effective;
        else
                caller_caps = &cred->cap_permitted;
        if (cred->user_ns == child_cred->user_ns &&
            cap_issubset(child_cred->cap_permitted, *caller_caps))
                goto out;
        if (ns_capable(child_cred->user_ns, CAP_SYS_PTRACE))
                goto out;
        ret = -EPERM;
out:
        rcu_read_unlock();
        return ret;
}

/**
 * cap_ptrace_traceme - Determine whether another process may trace the current
 * @parent: The task proposed to be the tracer
 *
 * If parent is in the same or an ancestor user_ns and has all current's
 * capabilities, then ptrace access is allowed.
 * If parent has the ptrace capability to current's user_ns, then ptrace
 * access is allowed.
 * Else denied.
 *
 * Determine whether the nominated task is permitted to trace the current
 * process, returning 0 if permission is granted, -ve if denied.
 */
int cap_ptrace_traceme(struct task_struct *parent)
{
        int ret = 0;
        const struct cred *cred, *child_cred;

        rcu_read_lock();
        cred = __task_cred(parent);
        child_cred = current_cred();
        if (cred->user_ns == child_cred->user_ns &&
            cap_issubset(child_cred->cap_permitted, cred->cap_permitted))
                goto out;
        if (has_ns_capability(parent, child_cred->user_ns, CAP_SYS_PTRACE))
                goto out;
        ret = -EPERM;
out:
        rcu_read_unlock();
        return ret;
}

/**
 * cap_capget - Retrieve a task's capability sets
 * @target: The task from which to retrieve the capability sets
 * @effective: The place to record the effective set
 * @inheritable: The place to record the inheritable set
 * @permitted: The place to record the permitted set
 *
 * This function retrieves the capabilities of the nominated task and returns
 * them to the caller.
 */
int cap_capget(const struct task_struct *target, kernel_cap_t *effective,
               kernel_cap_t *inheritable, kernel_cap_t *permitted)
{
        const struct cred *cred;

        /* Derived from kernel/capability.c:sys_capget. */
        rcu_read_lock();
        cred = __task_cred(target);
        *effective   = cred->cap_effective;
        *inheritable = cred->cap_inheritable;
        *permitted   = cred->cap_permitted;
        rcu_read_unlock();
        return 0;
}

/*
 * Determine whether the inheritable capabilities are limited to the old
 * permitted set.  Returns 1 if they are limited, 0 if they are not.
 */
static inline int cap_inh_is_capped(void)
{
        /* they are so limited unless the current task has the CAP_SETPCAP
         * capability
         */
        if (cap_capable(current_cred(), current_cred()->user_ns,
                        CAP_SETPCAP, CAP_OPT_NONE) == 0)
                return 0;
        return 1;
}

/**
 * cap_capset - Validate and apply proposed changes to current's capabilities
 * @new: The proposed new credentials; alterations should be made here
 * @old: The current task's current credentials
 * @effective: A pointer to the proposed new effective capabilities set
 * @inheritable: A pointer to the proposed new inheritable capabilities set
 * @permitted: A pointer to the proposed new permitted capabilities set
 *
 * This function validates and applies a proposed mass change to the current
 * process's capability sets.  The changes are made to the proposed new
 * credentials, and assuming no error, will be committed by the caller of LSM.
 */
int cap_capset(struct cred *new,
               const struct cred *old,
               const kernel_cap_t *effective,
               const kernel_cap_t *inheritable,
               const kernel_cap_t *permitted)
{
        if (cap_inh_is_capped() &&
            !cap_issubset(*inheritable,
                          cap_combine(old->cap_inheritable,
                                      old->cap_permitted)))
                /* incapable of using this inheritable set */
                return -EPERM;

        if (!cap_issubset(*inheritable,
                          cap_combine(old->cap_inheritable,
                                      old->cap_bset)))
                /* no new pI capabilities outside bounding set */
                return -EPERM;

        /* verify restrictions on target's new Permitted set */
        if (!cap_issubset(*permitted, old->cap_permitted))
                return -EPERM;

        /* verify the _new_Effective_ is a subset of the _new_Permitted_ */
        if (!cap_issubset(*effective, *permitted))
                return -EPERM;

        new->cap_effective   = *effective;
        new->cap_inheritable = *inheritable;
        new->cap_permitted   = *permitted;

        /*
         * Mask off ambient bits that are no longer both permitted and
         * inheritable.
         */
        new->cap_ambient = cap_intersect(new->cap_ambient,
                                         cap_intersect(*permitted,
                                                       *inheritable));
        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EINVAL;
        return 0;
}

/**
 * cap_inode_need_killpriv - Determine if inode change affects privileges
 * @dentry: The inode/dentry in being changed with change marked ATTR_KILL_PRIV
 *
 * Determine if an inode having a change applied that's marked ATTR_KILL_PRIV
 * affects the security markings on that inode, and if it is, should
 * inode_killpriv() be invoked or the change rejected.
 *
 * Return: 1 if security.capability has a value, meaning inode_killpriv()
 * is required, 0 otherwise, meaning inode_killpriv() is not required.
 */
int cap_inode_need_killpriv(struct dentry *dentry)
{
        struct inode *inode = d_backing_inode(dentry);
        int error;

        error = __vfs_getxattr(dentry, inode, XATTR_NAME_CAPS, NULL, 0);
        return error > 0;
}

/**
 * cap_inode_killpriv - Erase the security markings on an inode
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        The inode/dentry to alter
 *
 * Erase the privilege-enhancing security markings on an inode.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * Return: 0 if successful, -ve on error.
 */
int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry)
{
        int error;

        error = __vfs_removexattr(idmap, dentry, XATTR_NAME_CAPS);
        if (error == -EOPNOTSUPP)
                error = 0;
        return error;
}

static bool rootid_owns_currentns(vfsuid_t rootvfsuid)
{
        struct user_namespace *ns;
        kuid_t kroot;

        if (!vfsuid_valid(rootvfsuid))
                return false;

        kroot = vfsuid_into_kuid(rootvfsuid);
        for (ns = current_user_ns();; ns = ns->parent) {
                if (from_kuid(ns, kroot) == 0)
                        return true;
                if (ns == &init_user_ns)
                        break;
        }

        return false;
}

static __u32 sansflags(__u32 m)
{
        return m & ~VFS_CAP_FLAGS_EFFECTIVE;
}

static bool is_v2header(int size, const struct vfs_cap_data *cap)
{
        if (size != XATTR_CAPS_SZ_2)
                return false;
        return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_2;
}

static bool is_v3header(int size, const struct vfs_cap_data *cap)
{
        if (size != XATTR_CAPS_SZ_3)
                return false;
        return sansflags(le32_to_cpu(cap->magic_etc)) == VFS_CAP_REVISION_3;
}

/*
 * getsecurity: We are called for security.* before any attempt to read the
 * xattr from the inode itself.
 *
 * This gives us a chance to read the on-disk value and convert it.  If we
 * return -EOPNOTSUPP, then vfs_getxattr() will call the i_op handler.
 *
 * Note we are not called by vfs_getxattr_alloc(), but that is only called
 * by the integrity subsystem, which really wants the unconverted values -
 * so that's good.
 */
int cap_inode_getsecurity(struct mnt_idmap *idmap,
                          struct inode *inode, const char *name, void **buffer,
                          bool alloc)
{
        int size;
        kuid_t kroot;
        vfsuid_t vfsroot;
        u32 nsmagic, magic;
        uid_t root, mappedroot;
        char *tmpbuf = NULL;
        struct vfs_cap_data *cap;
        struct vfs_ns_cap_data *nscap = NULL;
        struct dentry *dentry;
        struct user_namespace *fs_ns;

        if (strcmp(name, "capability") != 0)
                return -EOPNOTSUPP;

        dentry = d_find_any_alias(inode);
        if (!dentry)
                return -EINVAL;
        size = vfs_getxattr_alloc(idmap, dentry, XATTR_NAME_CAPS, &tmpbuf,
                                  sizeof(struct vfs_ns_cap_data), GFP_NOFS);
        dput(dentry);
        /* gcc11 complains if we don't check for !tmpbuf */
        if (size < 0 || !tmpbuf)
                goto out_free;

        fs_ns = inode->i_sb->s_user_ns;
        cap = (struct vfs_cap_data *) tmpbuf;
        if (is_v2header(size, cap)) {
                root = 0;
        } else if (is_v3header(size, cap)) {
                nscap = (struct vfs_ns_cap_data *) tmpbuf;
                root = le32_to_cpu(nscap->rootid);
        } else {
                size = -EINVAL;
                goto out_free;
        }

        kroot = make_kuid(fs_ns, root);

        /* If this is an idmapped mount shift the kuid. */
        vfsroot = make_vfsuid(idmap, fs_ns, kroot);

        /* If the root kuid maps to a valid uid in current ns, then return
         * this as a nscap. */
        mappedroot = from_kuid(current_user_ns(), vfsuid_into_kuid(vfsroot));
        if (mappedroot != (uid_t)-1 && mappedroot != (uid_t)0) {
                size = sizeof(struct vfs_ns_cap_data);
                if (alloc) {
                        if (!nscap) {
                                /* v2 -> v3 conversion */
                                nscap = kzalloc(size, GFP_ATOMIC);
                                if (!nscap) {
                                        size = -ENOMEM;
                                        goto out_free;
                                }
                                nsmagic = VFS_CAP_REVISION_3;
                                magic = le32_to_cpu(cap->magic_etc);
                                if (magic & VFS_CAP_FLAGS_EFFECTIVE)
                                        nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
                                memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
                                nscap->magic_etc = cpu_to_le32(nsmagic);
                        } else {
                                /* use allocated v3 buffer */
                                tmpbuf = NULL;
                        }
                        nscap->rootid = cpu_to_le32(mappedroot);
                        *buffer = nscap;
                }
                goto out_free;
        }

        if (!rootid_owns_currentns(vfsroot)) {
                size = -EOVERFLOW;
                goto out_free;
        }

        /* This comes from a parent namespace.  Return as a v2 capability */
        size = sizeof(struct vfs_cap_data);
        if (alloc) {
                if (nscap) {
                        /* v3 -> v2 conversion */
                        cap = kzalloc(size, GFP_ATOMIC);
                        if (!cap) {
                                size = -ENOMEM;
                                goto out_free;
                        }
                        magic = VFS_CAP_REVISION_2;
                        nsmagic = le32_to_cpu(nscap->magic_etc);
                        if (nsmagic & VFS_CAP_FLAGS_EFFECTIVE)
                                magic |= VFS_CAP_FLAGS_EFFECTIVE;
                        memcpy(&cap->data, &nscap->data, sizeof(__le32) * 2 * VFS_CAP_U32);
                        cap->magic_etc = cpu_to_le32(magic);
                } else {
                        /* use unconverted v2 */
                        tmpbuf = NULL;
                }
                *buffer = cap;
        }
out_free:
        kfree(tmpbuf);
        return size;
}

/**
 * rootid_from_xattr - translate root uid of vfs caps
 *
 * @value:        vfs caps value which may be modified by this function
 * @size:        size of @ivalue
 * @task_ns:        user namespace of the caller
 */
static vfsuid_t rootid_from_xattr(const void *value, size_t size,
                                  struct user_namespace *task_ns)
{
        const struct vfs_ns_cap_data *nscap = value;
        uid_t rootid = 0;

        if (size == XATTR_CAPS_SZ_3)
                rootid = le32_to_cpu(nscap->rootid);

        return VFSUIDT_INIT(make_kuid(task_ns, rootid));
}

static bool validheader(size_t size, const struct vfs_cap_data *cap)
{
        return is_v2header(size, cap) || is_v3header(size, cap);
}

/**
 * cap_convert_nscap - check vfs caps
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        used to retrieve inode to check permissions on
 * @ivalue:        vfs caps value which may be modified by this function
 * @size:        size of @ivalue
 *
 * User requested a write of security.capability.  If needed, update the
 * xattr to change from v2 to v3, or to fixup the v3 rootid.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * Return: On success, return the new size; on error, return < 0.
 */
int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
                      const void **ivalue, size_t size)
{
        struct vfs_ns_cap_data *nscap;
        uid_t nsrootid;
        const struct vfs_cap_data *cap = *ivalue;
        __u32 magic, nsmagic;
        struct inode *inode = d_backing_inode(dentry);
        struct user_namespace *task_ns = current_user_ns(),
                *fs_ns = inode->i_sb->s_user_ns;
        kuid_t rootid;
        vfsuid_t vfsrootid;
        size_t newsize;

        if (!*ivalue)
                return -EINVAL;
        if (!validheader(size, cap))
                return -EINVAL;
        if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
                return -EPERM;
        if (size == XATTR_CAPS_SZ_2 && (idmap == &nop_mnt_idmap))
                if (ns_capable(inode->i_sb->s_user_ns, CAP_SETFCAP))
                        /* user is privileged, just write the v2 */
                        return size;

        vfsrootid = rootid_from_xattr(*ivalue, size, task_ns);
        if (!vfsuid_valid(vfsrootid))
                return -EINVAL;

        rootid = from_vfsuid(idmap, fs_ns, vfsrootid);
        if (!uid_valid(rootid))
                return -EINVAL;

        nsrootid = from_kuid(fs_ns, rootid);
        if (nsrootid == -1)
                return -EINVAL;

        newsize = sizeof(struct vfs_ns_cap_data);
        nscap = kmalloc(newsize, GFP_ATOMIC);
        if (!nscap)
                return -ENOMEM;
        nscap->rootid = cpu_to_le32(nsrootid);
        nsmagic = VFS_CAP_REVISION_3;
        magic = le32_to_cpu(cap->magic_etc);
        if (magic & VFS_CAP_FLAGS_EFFECTIVE)
                nsmagic |= VFS_CAP_FLAGS_EFFECTIVE;
        nscap->magic_etc = cpu_to_le32(nsmagic);
        memcpy(&nscap->data, &cap->data, sizeof(__le32) * 2 * VFS_CAP_U32);

        *ivalue = nscap;
        return newsize;
}

/*
 * Calculate the new process capability sets from the capability sets attached
 * to a file.
 */
static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps,
                                          struct linux_binprm *bprm,
                                          bool *effective,
                                          bool *has_fcap)
{
        struct cred *new = bprm->cred;
        int ret = 0;

        if (caps->magic_etc & VFS_CAP_FLAGS_EFFECTIVE)
                *effective = true;

        if (caps->magic_etc & VFS_CAP_REVISION_MASK)
                *has_fcap = true;

        /*
         * pP' = (X & fP) | (pI & fI)
         * The addition of pA' is handled later.
         */
        new->cap_permitted.val =
                (new->cap_bset.val & caps->permitted.val) |
                (new->cap_inheritable.val & caps->inheritable.val);

        if (caps->permitted.val & ~new->cap_permitted.val)
                /* insufficient to execute correctly */
                ret = -EPERM;

        /*
         * For legacy apps, with no internal support for recognizing they
         * do not have enough capabilities, we return an error if they are
         * missing some "forced" (aka file-permitted) capabilities.
         */
        return *effective ? ret : 0;
}

/**
 * get_vfs_caps_from_disk - retrieve vfs caps from disk
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        dentry from which @inode is retrieved
 * @cpu_caps:        vfs capabilities
 *
 * Extract the on-exec-apply capability sets for an executable file.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 */
int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
                           const struct dentry *dentry,
                           struct cpu_vfs_cap_data *cpu_caps)
{
        struct inode *inode = d_backing_inode(dentry);
        __u32 magic_etc;
        int size;
        struct vfs_ns_cap_data data, *nscaps = &data;
        struct vfs_cap_data *caps = (struct vfs_cap_data *) &data;
        kuid_t rootkuid;
        vfsuid_t rootvfsuid;
        struct user_namespace *fs_ns;

        memset(cpu_caps, 0, sizeof(struct cpu_vfs_cap_data));

        if (!inode)
                return -ENODATA;

        fs_ns = inode->i_sb->s_user_ns;
        size = __vfs_getxattr((struct dentry *)dentry, inode,
                              XATTR_NAME_CAPS, &data, XATTR_CAPS_SZ);
        if (size == -ENODATA || size == -EOPNOTSUPP)
                /* no data, that's ok */
                return -ENODATA;

        if (size < 0)
                return size;

        if (size < sizeof(magic_etc))
                return -EINVAL;

        cpu_caps->magic_etc = magic_etc = le32_to_cpu(caps->magic_etc);

        rootkuid = make_kuid(fs_ns, 0);
        switch (magic_etc & VFS_CAP_REVISION_MASK) {
        case VFS_CAP_REVISION_1:
                if (size != XATTR_CAPS_SZ_1)
                        return -EINVAL;
                break;
        case VFS_CAP_REVISION_2:
                if (size != XATTR_CAPS_SZ_2)
                        return -EINVAL;
                break;
        case VFS_CAP_REVISION_3:
                if (size != XATTR_CAPS_SZ_3)
                        return -EINVAL;
                rootkuid = make_kuid(fs_ns, le32_to_cpu(nscaps->rootid));
                break;

        default:
                return -EINVAL;
        }

        rootvfsuid = make_vfsuid(idmap, fs_ns, rootkuid);
        if (!vfsuid_valid(rootvfsuid))
                return -ENODATA;

        /* Limit the caps to the mounter of the filesystem
         * or the more limited uid specified in the xattr.
         */
        if (!rootid_owns_currentns(rootvfsuid))
                return -ENODATA;

        cpu_caps->permitted.val = le32_to_cpu(caps->data[0].permitted);
        cpu_caps->inheritable.val = le32_to_cpu(caps->data[0].inheritable);

        /*
         * Rev1 had just a single 32-bit word, later expanded
         * to a second one for the high bits
         */
        if ((magic_etc & VFS_CAP_REVISION_MASK) != VFS_CAP_REVISION_1) {
                cpu_caps->permitted.val += (u64)le32_to_cpu(caps->data[1].permitted) << 32;
                cpu_caps->inheritable.val += (u64)le32_to_cpu(caps->data[1].inheritable) << 32;
        }

        cpu_caps->permitted.val &= CAP_VALID_MASK;
        cpu_caps->inheritable.val &= CAP_VALID_MASK;

        cpu_caps->rootid = vfsuid_into_kuid(rootvfsuid);

        return 0;
}

/*
 * Attempt to get the on-exec apply capability sets for an executable file from
 * its xattrs and, if present, apply them to the proposed credentials being
 * constructed by execve().
 */
static int get_file_caps(struct linux_binprm *bprm, const struct file *file,
                         bool *effective, bool *has_fcap)
{
        int rc = 0;
        struct cpu_vfs_cap_data vcaps;

        cap_clear(bprm->cred->cap_permitted);

        if (!file_caps_enabled)
                return 0;

        if (!mnt_may_suid(file->f_path.mnt))
                return 0;

        /*
         * This check is redundant with mnt_may_suid() but is kept to make
         * explicit that capability bits are limited to s_user_ns and its
         * descendants.
         */
        if (!current_in_userns(file->f_path.mnt->mnt_sb->s_user_ns))
                return 0;

        rc = get_vfs_caps_from_disk(file_mnt_idmap(file),
                                    file->f_path.dentry, &vcaps);
        if (rc < 0) {
                if (rc == -EINVAL)
                        printk(KERN_NOTICE "Invalid argument reading file caps for %s\n",
                                        bprm->filename);
                else if (rc == -ENODATA)
                        rc = 0;
                goto out;
        }

        rc = bprm_caps_from_vfs_caps(&vcaps, bprm, effective, has_fcap);

out:
        if (rc)
                cap_clear(bprm->cred->cap_permitted);

        return rc;
}

static inline bool root_privileged(void) { return !issecure(SECURE_NOROOT); }

static inline bool __is_real(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->uid, uid); }

static inline bool __is_eff(kuid_t uid, struct cred *cred)
{ return uid_eq(cred->euid, uid); }

static inline bool __is_suid(kuid_t uid, struct cred *cred)
{ return !__is_real(uid, cred) && __is_eff(uid, cred); }

/*
 * handle_privileged_root - Handle case of privileged root
 * @bprm: The execution parameters, including the proposed creds
 * @has_fcap: Are any file capabilities set?
 * @effective: Do we have effective root privilege?
 * @root_uid: This namespace' root UID WRT initial USER namespace
 *
 * Handle the case where root is privileged and hasn't been neutered by
 * SECURE_NOROOT.  If file capabilities are set, they won't be combined with
 * set UID root and nothing is changed.  If we are root, cap_permitted is
 * updated.  If we have become set UID root, the effective bit is set.
 */
static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap,
                                   bool *effective, kuid_t root_uid)
{
        const struct cred *old = current_cred();
        struct cred *new = bprm->cred;

        if (!root_privileged())
                return;
        /*
         * If the legacy file capability is set, then don't set privs
         * for a setuid root binary run by a non-root user.  Do set it
         * for a root user just to cause least surprise to an admin.
         */
        if (has_fcap && __is_suid(root_uid, new)) {
                warn_setuid_and_fcaps_mixed(bprm->filename);
                return;
        }
        /*
         * To support inheritance of root-permissions and suid-root
         * executables under compatibility mode, we override the
         * capability sets for the file.
         */
        if (__is_eff(root_uid, new) || __is_real(root_uid, new)) {
                /* pP' = (cap_bset & ~0) | (pI & ~0) */
                new->cap_permitted = cap_combine(old->cap_bset,
                                                 old->cap_inheritable);
        }
        /*
         * If only the real uid is 0, we do not set the effective bit.
         */
        if (__is_eff(root_uid, new))
                *effective = true;
}

#define __cap_gained(field, target, source) \
        !cap_issubset(target->cap_##field, source->cap_##field)
#define __cap_grew(target, source, cred) \
        !cap_issubset(cred->cap_##target, cred->cap_##source)
#define __cap_full(field, cred) \
        cap_issubset(CAP_FULL_SET, cred->cap_##field)

/*
 * 1) Audit candidate if current->cap_effective is set
 *
 * We do not bother to audit if 3 things are true:
 *   1) cap_effective has all caps
 *   2) we became root *OR* are were already root
 *   3) root is supposed to have all caps (SECURE_NOROOT)
 * Since this is just a normal root execing a process.
 *
 * Number 1 above might fail if you don't have a full bset, but I think
 * that is interesting information to audit.
 *
 * A number of other conditions require logging:
 * 2) something prevented setuid root getting all caps
 * 3) non-setuid root gets fcaps
 * 4) non-setuid root gets ambient
 */
static inline bool nonroot_raised_pE(struct cred *new, const struct cred *old,
                                     kuid_t root, bool has_fcap)
{
        bool ret = false;

        if ((__cap_grew(effective, ambient, new) &&
             !(__cap_full(effective, new) &&
               (__is_eff(root, new) || __is_real(root, new)) &&
               root_privileged())) ||
            (root_privileged() &&
             __is_suid(root, new) &&
             !__cap_full(effective, new)) ||
            (uid_eq(new->euid, old->euid) &&
             ((has_fcap &&
               __cap_gained(permitted, new, old)) ||
              __cap_gained(ambient, new, old))))

                ret = true;

        return ret;
}

/**
 * cap_bprm_creds_from_file - Set up the proposed credentials for execve().
 * @bprm: The execution parameters, including the proposed creds
 * @file: The file to pull the credentials from
 *
 * Set up the proposed credentials for a new execution context being
 * constructed by execve().  The proposed creds in @bprm->cred is altered,
 * which won't take effect immediately.
 *
 * Return: 0 if successful, -ve on error.
 */
int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file)
{
        /* Process setpcap binaries and capabilities for uid 0 */
        const struct cred *old = current_cred();
        struct cred *new = bprm->cred;
        bool effective = false, has_fcap = false, id_changed;
        int ret;
        kuid_t root_uid;

        if (WARN_ON(!cap_ambient_invariant_ok(old)))
                return -EPERM;

        ret = get_file_caps(bprm, file, &effective, &has_fcap);
        if (ret < 0)
                return ret;

        root_uid = make_kuid(new->user_ns, 0);

        handle_privileged_root(bprm, has_fcap, &effective, root_uid);

        /* if we have fs caps, clear dangerous personality flags */
        if (__cap_gained(permitted, new, old))
                bprm->per_clear |= PER_CLEAR_ON_SETID;

        /* Don't let someone trace a set[ug]id/setpcap binary with the revised
         * credentials unless they have the appropriate permit.
         *
         * In addition, if NO_NEW_PRIVS, then ensure we get no new privs.
         */
        id_changed = !uid_eq(new->euid, old->euid) || !in_group_p(new->egid);

        if ((id_changed || __cap_gained(permitted, new, old)) &&
            ((bprm->unsafe & ~LSM_UNSAFE_PTRACE) ||
             !ptracer_capable(current, new->user_ns))) {
                /* downgrade; they get no more than they had, and maybe less */
                if (!ns_capable(new->user_ns, CAP_SETUID) ||
                    (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) {
                        new->euid = new->uid;
                        new->egid = new->gid;
                }
                new->cap_permitted = cap_intersect(new->cap_permitted,
                                                   old->cap_permitted);
        }

        new->suid = new->fsuid = new->euid;
        new->sgid = new->fsgid = new->egid;

        /* File caps or setid cancels ambient. */
        if (has_fcap || id_changed)
                cap_clear(new->cap_ambient);

        /*
         * Now that we've computed pA', update pP' to give:
         *   pP' = (X & fP) | (pI & fI) | pA'
         */
        new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient);

        /*
         * Set pE' = (fE ? pP' : pA').  Because pA' is zero if fE is set,
         * this is the same as pE' = (fE ? pP' : 0) | pA'.
         */
        if (effective)
                new->cap_effective = new->cap_permitted;
        else
                new->cap_effective = new->cap_ambient;

        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EPERM;

        if (nonroot_raised_pE(new, old, root_uid, has_fcap)) {
                ret = audit_log_bprm_fcaps(bprm, new, old);
                if (ret < 0)
                        return ret;
        }

        new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);

        if (WARN_ON(!cap_ambient_invariant_ok(new)))
                return -EPERM;

        /* Check for privilege-elevated exec. */
        if (id_changed ||
            !uid_eq(new->euid, old->uid) ||
            !gid_eq(new->egid, old->gid) ||
            (!__is_real(root_uid, new) &&
             (effective ||
              __cap_grew(permitted, ambient, new))))
                bprm->secureexec = 1;

        return 0;
}

/**
 * cap_inode_setxattr - Determine whether an xattr may be altered
 * @dentry: The inode/dentry being altered
 * @name: The name of the xattr to be changed
 * @value: The value that the xattr will be changed to
 * @size: The size of value
 * @flags: The replacement flag
 *
 * Determine whether an xattr may be altered or set on an inode, returning 0 if
 * permission is granted, -ve if denied.
 *
 * This is used to make sure security xattrs don't get updated or set by those
 * who aren't privileged to do so.
 */
int cap_inode_setxattr(struct dentry *dentry, const char *name,
                       const void *value, size_t size, int flags)
{
        struct user_namespace *user_ns = dentry->d_sb->s_user_ns;

        /* Ignore non-security xattrs */
        if (strncmp(name, XATTR_SECURITY_PREFIX,
                        XATTR_SECURITY_PREFIX_LEN) != 0)
                return 0;

        /*
         * For XATTR_NAME_CAPS the check will be done in
         * cap_convert_nscap(), called by setxattr()
         */
        if (strcmp(name, XATTR_NAME_CAPS) == 0)
                return 0;

        if (!ns_capable(user_ns, CAP_SYS_ADMIN))
                return -EPERM;
        return 0;
}

/**
 * cap_inode_removexattr - Determine whether an xattr may be removed
 *
 * @idmap:        idmap of the mount the inode was found from
 * @dentry:        The inode/dentry being altered
 * @name:        The name of the xattr to be changed
 *
 * Determine whether an xattr may be removed from an inode, returning 0 if
 * permission is granted, -ve if denied.
 *
 * If the inode has been found through an idmapped mount the idmap of
 * the vfsmount must be passed through @idmap. This function will then
 * take care to map the inode according to @idmap before checking
 * permissions. On non-idmapped mounts or if permission checking is to be
 * performed on the raw inode simply pass @nop_mnt_idmap.
 *
 * This is used to make sure security xattrs don't get removed by those who
 * aren't privileged to remove them.
 */
int cap_inode_removexattr(struct mnt_idmap *idmap,
                          struct dentry *dentry, const char *name)
{
        struct user_namespace *user_ns = dentry->d_sb->s_user_ns;

        /* Ignore non-security xattrs */
        if (strncmp(name, XATTR_SECURITY_PREFIX,
                        XATTR_SECURITY_PREFIX_LEN) != 0)
                return 0;

        if (strcmp(name, XATTR_NAME_CAPS) == 0) {
                /* security.capability gets namespaced */
                struct inode *inode = d_backing_inode(dentry);
                if (!inode)
                        return -EINVAL;
                if (!capable_wrt_inode_uidgid(idmap, inode, CAP_SETFCAP))
                        return -EPERM;
                return 0;
        }

        if (!ns_capable(user_ns, CAP_SYS_ADMIN))
                return -EPERM;
        return 0;
}

/*
 * cap_emulate_setxuid() fixes the effective / permitted capabilities of
 * a process after a call to setuid, setreuid, or setresuid.
 *
 *  1) When set*uiding _from_ one of {r,e,s}uid == 0 _to_ all of
 *  {r,e,s}uid != 0, the permitted and effective capabilities are
 *  cleared.
 *
 *  2) When set*uiding _from_ euid == 0 _to_ euid != 0, the effective
 *  capabilities of the process are cleared.
 *
 *  3) When set*uiding _from_ euid != 0 _to_ euid == 0, the effective
 *  capabilities are set to the permitted capabilities.
 *
 *  fsuid is handled elsewhere. fsuid == 0 and {r,e,s}uid!= 0 should
 *  never happen.
 *
 *  -astor
 *
 * cevans - New behaviour, Oct '99
 * A process may, via prctl(), elect to keep its capabilities when it
 * calls setuid() and switches away from uid==0. Both permitted and
 * effective sets will be retained.
 * Without this change, it was impossible for a daemon to drop only some
 * of its privilege. The call to setuid(!=0) would drop all privileges!
 * Keeping uid 0 is not an option because uid 0 owns too many vital
 * files..
 * Thanks to Olaf Kirch and Peter Benie for spotting this.
 */
static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old)
{
        kuid_t root_uid = make_kuid(old->user_ns, 0);

        if ((uid_eq(old->uid, root_uid) ||
             uid_eq(old->euid, root_uid) ||
             uid_eq(old->suid, root_uid)) &&
            (!uid_eq(new->uid, root_uid) &&
             !uid_eq(new->euid, root_uid) &&
             !uid_eq(new->suid, root_uid))) {
                if (!issecure(SECURE_KEEP_CAPS)) {
                        cap_clear(new->cap_permitted);
                        cap_clear(new->cap_effective);
                }

                /*
                 * Pre-ambient programs expect setresuid to nonroot followed
                 * by exec to drop capabilities.  We should make sure that
                 * this remains the case.
                 */
                cap_clear(new->cap_ambient);
        }
        if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid))
                cap_clear(new->cap_effective);
        if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid))
                new->cap_effective = new->cap_permitted;
}

/**
 * cap_task_fix_setuid - Fix up the results of setuid() call
 * @new: The proposed credentials
 * @old: The current task's current credentials
 * @flags: Indications of what has changed
 *
 * Fix up the results of setuid() call before the credential changes are
 * actually applied.
 *
 * Return: 0 to grant the changes, -ve to deny them.
 */
int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags)
{
        switch (flags) {
        case LSM_SETID_RE:
        case LSM_SETID_ID:
        case LSM_SETID_RES:
                /* juggle the capabilities to follow [RES]UID changes unless
                 * otherwise suppressed */
                if (!issecure(SECURE_NO_SETUID_FIXUP))
                        cap_emulate_setxuid(new, old);
                break;

        case LSM_SETID_FS:
                /* juggle the capabilities to follow FSUID changes, unless
                 * otherwise suppressed
                 *
                 * FIXME - is fsuser used for all CAP_FS_MASK capabilities?
                 *          if not, we might be a bit too harsh here.
                 */
                if (!issecure(SECURE_NO_SETUID_FIXUP)) {
                        kuid_t root_uid = make_kuid(old->user_ns, 0);
                        if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid))
                                new->cap_effective =
                                        cap_drop_fs_set(new->cap_effective);

                        if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid))
                                new->cap_effective =
                                        cap_raise_fs_set(new->cap_effective,
                                                         new->cap_permitted);
                }
                break;

        default:
                return -EINVAL;
        }

        return 0;
}

/*
 * Rationale: code calling task_setscheduler, task_setioprio, and
 * task_setnice, assumes that
 *   . if capable(cap_sys_nice), then those actions should be allowed
 *   . if not capable(cap_sys_nice), but acting on your own processes,
 *           then those actions should be allowed
 * This is insufficient now since you can call code without suid, but
 * yet with increased caps.
 * So we check for increased caps on the target process.
 */
static int cap_safe_nice(struct task_struct *p)
{
        int is_subset, ret = 0;

        rcu_read_lock();
        is_subset = cap_issubset(__task_cred(p)->cap_permitted,
                                 current_cred()->cap_permitted);
        if (!is_subset && !ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE))
                ret = -EPERM;
        rcu_read_unlock();

        return ret;
}

/**
 * cap_task_setscheduler - Determine if scheduler policy change is permitted
 * @p: The task to affect
 *
 * Determine if the requested scheduler policy change is permitted for the
 * specified task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setscheduler(struct task_struct *p)
{
        return cap_safe_nice(p);
}

/**
 * cap_task_setioprio - Determine if I/O priority change is permitted
 * @p: The task to affect
 * @ioprio: The I/O priority to set
 *
 * Determine if the requested I/O priority change is permitted for the specified
 * task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setioprio(struct task_struct *p, int ioprio)
{
        return cap_safe_nice(p);
}

/**
 * cap_task_setnice - Determine if task priority change is permitted
 * @p: The task to affect
 * @nice: The nice value to set
 *
 * Determine if the requested task priority change is permitted for the
 * specified task.
 *
 * Return: 0 if permission is granted, -ve if denied.
 */
int cap_task_setnice(struct task_struct *p, int nice)
{
        return cap_safe_nice(p);
}

/*
 * Implement PR_CAPBSET_DROP.  Attempt to remove the specified capability from
 * the current task's bounding set.  Returns 0 on success, -ve on error.
 */
static int cap_prctl_drop(unsigned long cap)
{
        struct cred *new;

        if (!ns_capable(current_user_ns(), CAP_SETPCAP))
                return -EPERM;
        if (!cap_valid(cap))
                return -EINVAL;

        new = prepare_creds();
        if (!new)
                return -ENOMEM;
        cap_lower(new->cap_bset, cap);
        return commit_creds(new);
}

/**
 * cap_task_prctl - Implement process control functions for this security module
 * @option: The process control function requested
 * @arg2: The argument data for this function
 * @arg3: The argument data for this function
 * @arg4: The argument data for this function
 * @arg5: The argument data for this function
 *
 * Allow process control functions (sys_prctl()) to alter capabilities; may
 * also deny access to other functions not otherwise implemented here.
 *
 * Return: 0 or +ve on success, -ENOSYS if this function is not implemented
 * here, other -ve on error.  If -ENOSYS is returned, sys_prctl() and other LSM
 * modules will consider performing the function.
 */
int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3,
                   unsigned long arg4, unsigned long arg5)
{
        const struct cred *old = current_cred();
        struct cred *new;

        switch (option) {
        case PR_CAPBSET_READ:
                if (!cap_valid(arg2))
                        return -EINVAL;
                return !!cap_raised(old->cap_bset, arg2);

        case PR_CAPBSET_DROP:
                return cap_prctl_drop(arg2);

        /*
         * The next four prctl's remain to assist with transitioning a
         * system from legacy UID=0 based privilege (when filesystem
         * capabilities are not in use) to a system using filesystem
         * capabilities only - as the POSIX.1e draft intended.
         *
         * Note:
         *
         *  PR_SET_SECUREBITS =
         *      issecure_mask(SECURE_KEEP_CAPS_LOCKED)
         *    | issecure_mask(SECURE_NOROOT)
         *    | issecure_mask(SECURE_NOROOT_LOCKED)
         *    | issecure_mask(SECURE_NO_SETUID_FIXUP)
         *    | issecure_mask(SECURE_NO_SETUID_FIXUP_LOCKED)
         *
         * will ensure that the current process and all of its
         * children will be locked into a pure
         * capability-based-privilege environment.
         */
        case PR_SET_SECUREBITS:
                if ((((old->securebits & SECURE_ALL_LOCKS) >> 1)
                     & (old->securebits ^ arg2))                        /*[1]*/
                    || ((old->securebits & SECURE_ALL_LOCKS & ~arg2))        /*[2]*/
                    || (arg2 & ~(SECURE_ALL_LOCKS | SECURE_ALL_BITS))        /*[3]*/
                        /*
                         * [1] no changing of bits that are locked
                         * [2] no unlocking of locks
                         * [3] no setting of unsupported bits
                         */
                    )
                        /* cannot change a locked bit */
                        return -EPERM;

                /*
                 * Doing anything requires privilege (go read about the
                 * "sendmail capabilities bug"), except for unprivileged bits.
                 * Indeed, the SECURE_ALL_UNPRIVILEGED bits are not
                 * restrictions enforced by the kernel but by user space on
                 * itself.
                 */
                if (cap_capable(current_cred(), current_cred()->user_ns,
                                CAP_SETPCAP, CAP_OPT_NONE) != 0) {
                        const unsigned long unpriv_and_locks =
                                SECURE_ALL_UNPRIVILEGED |
                                SECURE_ALL_UNPRIVILEGED << 1;
                        const unsigned long changed = old->securebits ^ arg2;

                        /* For legacy reason, denies non-change. */
                        if (!changed)
                                return -EPERM;

                        /* Denies privileged changes. */
                        if (changed & ~unpriv_and_locks)
                                return -EPERM;
                }

                new = prepare_creds();
                if (!new)
                        return -ENOMEM;
                new->securebits = arg2;
                return commit_creds(new);

        case PR_GET_SECUREBITS:
                return old->securebits;

        case PR_GET_KEEPCAPS:
                return !!issecure(SECURE_KEEP_CAPS);

        case PR_SET_KEEPCAPS:
                if (arg2 > 1) /* Note, we rely on arg2 being unsigned here */
                        return -EINVAL;
                if (issecure(SECURE_KEEP_CAPS_LOCKED))
                        return -EPERM;

                new = prepare_creds();
                if (!new)
                        return -ENOMEM;
                if (arg2)
                        new->securebits |= issecure_mask(SECURE_KEEP_CAPS);
                else
                        new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS);
                return commit_creds(new);

        case PR_CAP_AMBIENT:
                if (arg2 == PR_CAP_AMBIENT_CLEAR_ALL) {
                        if (arg3 | arg4 | arg5)
                                return -EINVAL;

                        new = prepare_creds();
                        if (!new)
                                return -ENOMEM;
                        cap_clear(new->cap_ambient);
                        return commit_creds(new);
                }

                if (((!cap_valid(arg3)) | arg4 | arg5))
                        return -EINVAL;

                if (arg2 == PR_CAP_AMBIENT_IS_SET) {
                        return !!cap_raised(current_cred()->cap_ambient, arg3);
                } else if (arg2 != PR_CAP_AMBIENT_RAISE &&
                           arg2 != PR_CAP_AMBIENT_LOWER) {
                        return -EINVAL;
                } else {
                        if (arg2 == PR_CAP_AMBIENT_RAISE &&
                            (!cap_raised(current_cred()->cap_permitted, arg3) ||
                             !cap_raised(current_cred()->cap_inheritable,
                                         arg3) ||
                             issecure(SECURE_NO_CAP_AMBIENT_RAISE)))
                                return -EPERM;

                        new = prepare_creds();
                        if (!new)
                                return -ENOMEM;
                        if (arg2 == PR_CAP_AMBIENT_RAISE)
                                cap_raise(new->cap_ambient, arg3);
                        else
                                cap_lower(new->cap_ambient, arg3);
                        return commit_creds(new);
                }

        default:
                /* No functionality available - continue with default */
                return -ENOSYS;
        }
}

/**
 * cap_vm_enough_memory - Determine whether a new virtual mapping is permitted
 * @mm: The VM space in which the new mapping is to be made
 * @pages: The size of the mapping
 *
 * Determine whether the allocation of a new virtual mapping by the current
 * task is permitted.
 *
 * Return: 0 if permission granted, negative error code if not.
 */
int cap_vm_enough_memory(struct mm_struct *mm, long pages)
{
        return cap_capable(current_cred(), &init_user_ns, CAP_SYS_ADMIN,
                           CAP_OPT_NOAUDIT);
}

/**
 * cap_mmap_addr - check if able to map given addr
 * @addr: address attempting to be mapped
 *
 * If the process is attempting to map memory below dac_mmap_min_addr they need
 * CAP_SYS_RAWIO.  The other parameters to this function are unused by the
 * capability security module.
 *
 * Return: 0 if this mapping should be allowed or -EPERM if not.
 */
int cap_mmap_addr(unsigned long addr)
{
        int ret = 0;

        if (addr < dac_mmap_min_addr) {
                ret = cap_capable(current_cred(), &init_user_ns, CAP_SYS_RAWIO,
                                  CAP_OPT_NONE);
                /* set PF_SUPERPRIV if it turns out we allow the low mmap */
                if (ret == 0)
                        current->flags |= PF_SUPERPRIV;
        }
        return ret;
}

#ifdef CONFIG_SECURITY

static const struct lsm_id capability_lsmid = {
        .name = "capability",
        .id = LSM_ID_CAPABILITY,
};

static struct security_hook_list capability_hooks[] __ro_after_init = {
        LSM_HOOK_INIT(capable, cap_capable),
        LSM_HOOK_INIT(settime, cap_settime),
        LSM_HOOK_INIT(ptrace_access_check, cap_ptrace_access_check),
        LSM_HOOK_INIT(ptrace_traceme, cap_ptrace_traceme),
        LSM_HOOK_INIT(capget, cap_capget),
        LSM_HOOK_INIT(capset, cap_capset),
        LSM_HOOK_INIT(bprm_creds_from_file, cap_bprm_creds_from_file),
        LSM_HOOK_INIT(inode_need_killpriv, cap_inode_need_killpriv),
        LSM_HOOK_INIT(inode_killpriv, cap_inode_killpriv),
        LSM_HOOK_INIT(inode_getsecurity, cap_inode_getsecurity),
        LSM_HOOK_INIT(mmap_addr, cap_mmap_addr),
        LSM_HOOK_INIT(task_fix_setuid, cap_task_fix_setuid),
        LSM_HOOK_INIT(task_prctl, cap_task_prctl),
        LSM_HOOK_INIT(task_setscheduler, cap_task_setscheduler),
        LSM_HOOK_INIT(task_setioprio, cap_task_setioprio),
        LSM_HOOK_INIT(task_setnice, cap_task_setnice),
        LSM_HOOK_INIT(vm_enough_memory, cap_vm_enough_memory),
};

static int __init capability_init(void)
{
        security_add_hooks(capability_hooks, ARRAY_SIZE(capability_hooks),
                           &capability_lsmid);
        return 0;
}

DEFINE_LSM(capability) = {
        .name = "capability",
        .order = LSM_ORDER_FIRST,
        .init = capability_init,
};

#endif /* CONFIG_SECURITY */























   27 




    9 






   27 














   27 

   27 
   27 















   27 

   19 





   19 





    7 
    5 




   27 







   27 
   27 
   27 



   16 




   17 
   17 
   13 







    8 




    8 
    1 




    7 





    3 






    3 
    3 







    3 






    3 




    3 






    3 




    3 
    2 




    2 


























   15 









   15 


   15 



    9 



    9 



    9 
    9 







    9 




























































































































































































































   10 



    6 
    6 


    9 
    1 


   10 




   10 



   12 





   12 
   12 




   12 









    7 





    7 
    7 





    1 











    3 








    3 
    3 






    3 











    3 



















    1 





    1 
    1 


    1 















































   24 




























 1101 






























   11 













    9 



    2 


































   11 



    8 

    3 


    1 





    5 



    5 


    2 
    2 



    3 




    1 















    1 
    8 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
// SPDX-License-Identifier: GPL-2.0
/* Copyright 2011-2014 Autronica Fire and Security AS
 *
 * Author(s):
 *        2011-2014 Arvid Brodin, arvid.brodin@alten.se
 * This file contains device methods for creating, using and destroying
 * virtual HSR or PRP devices.
 */

#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/pkt_sched.h>
#include "hsr_device.h"
#include "hsr_slave.h"
#include "hsr_framereg.h"
#include "hsr_main.h"
#include "hsr_forward.h"

static bool is_admin_up(struct net_device *dev)
{
        return dev && (dev->flags & IFF_UP);
}

static bool is_slave_up(struct net_device *dev)
{
        return dev && is_admin_up(dev) && netif_oper_up(dev);
}

static void hsr_set_operstate(struct hsr_port *master, bool has_carrier)
{
        struct net_device *dev = master->dev;

        if (!is_admin_up(dev)) {
                netif_set_operstate(dev, IF_OPER_DOWN);
                return;
        }

        if (has_carrier)
                netif_set_operstate(dev, IF_OPER_UP);
        else
                netif_set_operstate(dev, IF_OPER_LOWERLAYERDOWN);
}

static bool hsr_check_carrier(struct hsr_port *master)
{
        struct hsr_port *port;

        ASSERT_RTNL();

        hsr_for_each_port_rtnl(master->hsr, port) {
                if (port->type != HSR_PT_MASTER && is_slave_up(port->dev)) {
                        netif_carrier_on(master->dev);
                        return true;
                }
        }

        netif_carrier_off(master->dev);

        return false;
}

static void hsr_check_announce(struct net_device *hsr_dev)
{
        struct hsr_priv *hsr;

        hsr = netdev_priv(hsr_dev);
        if (netif_running(hsr_dev) && netif_oper_up(hsr_dev)) {
                /* Enable announce timer and start sending supervisory frames */
                if (!timer_pending(&hsr->announce_timer)) {
                        hsr->announce_count = 0;
                        mod_timer(&hsr->announce_timer, jiffies +
                                  msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL));
                }

                if (hsr->redbox && !timer_pending(&hsr->announce_proxy_timer))
                        mod_timer(&hsr->announce_proxy_timer, jiffies +
                                  msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL) / 2);
        } else {
                /* Deactivate the announce timer  */
                timer_delete(&hsr->announce_timer);
                if (hsr->redbox)
                        timer_delete(&hsr->announce_proxy_timer);
        }
}

void hsr_check_carrier_and_operstate(struct hsr_priv *hsr)
{
        struct hsr_port *master;
        bool has_carrier;

        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        /* netif_stacked_transfer_operstate() cannot be used here since
         * it doesn't set IF_OPER_LOWERLAYERDOWN (?)
         */
        has_carrier = hsr_check_carrier(master);
        hsr_set_operstate(master, has_carrier);
        hsr_check_announce(master->dev);
}

int hsr_get_max_mtu(struct hsr_priv *hsr)
{
        unsigned int mtu_max;
        struct hsr_port *port;

        mtu_max = ETH_DATA_LEN;
        hsr_for_each_port_rtnl(hsr, port)
                if (port->type != HSR_PT_MASTER)
                        mtu_max = min(port->dev->mtu, mtu_max);

        if (mtu_max < HSR_HLEN)
                return 0;
        return mtu_max - HSR_HLEN;
}

static int hsr_dev_change_mtu(struct net_device *dev, int new_mtu)
{
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        if (new_mtu > hsr_get_max_mtu(hsr)) {
                netdev_info(dev, "A HSR master's MTU cannot be greater than the smallest MTU of its slaves minus the HSR Tag length (%d octets).\n",
                            HSR_HLEN);
                return -EINVAL;
        }

        WRITE_ONCE(dev->mtu, new_mtu);

        return 0;
}

static int hsr_dev_open(struct net_device *dev)
{
        struct hsr_priv *hsr;
        struct hsr_port *port;
        const char *designation = NULL;

        hsr = netdev_priv(dev);

        hsr_for_each_port_rtnl(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                        designation = "Slave A";
                        break;
                case HSR_PT_SLAVE_B:
                        designation = "Slave B";
                        break;
                case HSR_PT_INTERLINK:
                        designation = "Interlink";
                        break;
                default:
                        designation = "Unknown";
                }
                if (!is_slave_up(port->dev))
                        netdev_warn(dev, "%s (%s) is not up; please bring it up to get a fully working HSR network\n",
                                    designation, port->dev->name);
        }

        if (!designation)
                netdev_warn(dev, "No slave devices configured\n");

        return 0;
}

static int hsr_dev_close(struct net_device *dev)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);
        hsr_for_each_port_rtnl(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        dev_uc_unsync(port->dev, dev);
                        dev_mc_unsync(port->dev, dev);
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static netdev_features_t hsr_features_recompute(struct hsr_priv *hsr,
                                                netdev_features_t features)
{
        netdev_features_t mask;
        struct hsr_port *port;

        mask = features;

        /* Mask out all features that, if supported by one device, should be
         * enabled for all devices (see NETIF_F_ONE_FOR_ALL).
         *
         * Anything that's off in mask will not be enabled - so only things
         * that were in features originally, and also is in NETIF_F_ONE_FOR_ALL,
         * may become enabled.
         */
        features &= ~NETIF_F_ONE_FOR_ALL;
        hsr_for_each_port_rtnl(hsr, port)
                features = netdev_increment_features(features,
                                                     port->dev->features,
                                                     mask);

        return features;
}

static netdev_features_t hsr_fix_features(struct net_device *dev,
                                          netdev_features_t features)
{
        struct hsr_priv *hsr = netdev_priv(dev);

        return hsr_features_recompute(hsr, features);
}

static netdev_tx_t hsr_dev_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct hsr_priv *hsr = netdev_priv(dev);
        struct hsr_port *master;

        rcu_read_lock();
        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        if (master) {
                skb->dev = master->dev;
                skb_reset_mac_header(skb);
                skb_reset_mac_len(skb);
                spin_lock_bh(&hsr->seqnr_lock);
                hsr_forward_skb(skb, master);
                spin_unlock_bh(&hsr->seqnr_lock);
        } else {
                dev_core_stats_tx_dropped_inc(dev);
                dev_kfree_skb_any(skb);
        }
        rcu_read_unlock();

        return NETDEV_TX_OK;
}

static const struct header_ops hsr_header_ops = {
        .create         = eth_header,
        .parse         = eth_header_parse,
};

static struct sk_buff *hsr_init_skb(struct hsr_port *master, int extra)
{
        struct hsr_priv *hsr = master->hsr;
        struct sk_buff *skb;
        int hlen, tlen;
        int len;

        hlen = LL_RESERVED_SPACE(master->dev);
        tlen = master->dev->needed_tailroom;
        len = sizeof(struct hsr_sup_tag) + sizeof(struct hsr_sup_payload);
        /* skb size is same for PRP/HSR frames, only difference
         * being, for PRP it is a trailer and for HSR it is a
         * header.
         * RedBox might use @extra more bytes.
         */
        skb = dev_alloc_skb(len + extra + hlen + tlen);

        if (!skb)
                return skb;

        skb_reserve(skb, hlen);
        skb->dev = master->dev;
        skb->priority = TC_PRIO_CONTROL;

        skb_reset_network_header(skb);
        skb_reset_transport_header(skb);
        if (dev_hard_header(skb, skb->dev, ETH_P_PRP,
                            hsr->sup_multicast_addr,
                            skb->dev->dev_addr, skb->len) <= 0)
                goto out;

        skb_reset_mac_header(skb);
        skb_reset_mac_len(skb);

        return skb;
out:
        kfree_skb(skb);

        return NULL;
}

static void send_hsr_supervision_frame(struct hsr_port *port,
                                       unsigned long *interval,
                                       const unsigned char *addr)
{
        struct hsr_priv *hsr = port->hsr;
        __u8 type = HSR_TLV_LIFE_CHECK;
        struct hsr_sup_payload *hsr_sp;
        struct hsr_sup_tlv *hsr_stlv;
        struct hsr_sup_tag *hsr_stag;
        struct sk_buff *skb;
        int extra = 0;

        *interval = msecs_to_jiffies(HSR_LIFE_CHECK_INTERVAL);
        if (hsr->announce_count < 3 && hsr->prot_version == 0) {
                type = HSR_TLV_ANNOUNCE;
                *interval = msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL);
                hsr->announce_count++;
        }

        if (hsr->redbox)
                extra = sizeof(struct hsr_sup_tlv) +
                        sizeof(struct hsr_sup_payload);

        skb = hsr_init_skb(port, extra);
        if (!skb) {
                netdev_warn_once(port->dev, "HSR: Could not send supervision frame\n");
                return;
        }

        hsr_stag = skb_put(skb, sizeof(struct hsr_sup_tag));
        set_hsr_stag_path(hsr_stag, (hsr->prot_version ? 0x0 : 0xf));
        set_hsr_stag_HSR_ver(hsr_stag, hsr->prot_version);

        /* From HSRv1 on we have separate supervision sequence numbers. */
        spin_lock_bh(&hsr->seqnr_lock);
        if (hsr->prot_version > 0) {
                hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr);
                hsr->sup_sequence_nr++;
        } else {
                hsr_stag->sequence_nr = htons(hsr->sequence_nr);
                hsr->sequence_nr++;
        }

        hsr_stag->tlv.HSR_TLV_type = type;
        /* TODO: Why 12 in HSRv0? */
        hsr_stag->tlv.HSR_TLV_length = hsr->prot_version ?
                                sizeof(struct hsr_sup_payload) : 12;

        /* Payload: MacAddressA / SAN MAC from ProxyNodeTable */
        hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
        ether_addr_copy(hsr_sp->macaddress_A, addr);

        if (hsr->redbox &&
            hsr_is_node_in_db(&hsr->proxy_node_db, addr)) {
                hsr_stlv = skb_put(skb, sizeof(struct hsr_sup_tlv));
                hsr_stlv->HSR_TLV_type = PRP_TLV_REDBOX_MAC;
                hsr_stlv->HSR_TLV_length = sizeof(struct hsr_sup_payload);

                /* Payload: MacAddressRedBox */
                hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
                ether_addr_copy(hsr_sp->macaddress_A, hsr->macaddress_redbox);
        }

        if (skb_put_padto(skb, ETH_ZLEN)) {
                spin_unlock_bh(&hsr->seqnr_lock);
                return;
        }

        hsr_forward_skb(skb, port);
        spin_unlock_bh(&hsr->seqnr_lock);
        return;
}

static void send_prp_supervision_frame(struct hsr_port *master,
                                       unsigned long *interval,
                                       const unsigned char *addr)
{
        struct hsr_priv *hsr = master->hsr;
        struct hsr_sup_payload *hsr_sp;
        struct hsr_sup_tag *hsr_stag;
        struct sk_buff *skb;

        skb = hsr_init_skb(master, 0);
        if (!skb) {
                netdev_warn_once(master->dev, "PRP: Could not send supervision frame\n");
                return;
        }

        *interval = msecs_to_jiffies(HSR_LIFE_CHECK_INTERVAL);
        hsr_stag = skb_put(skb, sizeof(struct hsr_sup_tag));
        set_hsr_stag_path(hsr_stag, (hsr->prot_version ? 0x0 : 0xf));
        set_hsr_stag_HSR_ver(hsr_stag, (hsr->prot_version ? 1 : 0));

        /* From HSRv1 on we have separate supervision sequence numbers. */
        spin_lock_bh(&hsr->seqnr_lock);
        hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr);
        hsr->sup_sequence_nr++;
        hsr_stag->tlv.HSR_TLV_type = PRP_TLV_LIFE_CHECK_DD;
        hsr_stag->tlv.HSR_TLV_length = sizeof(struct hsr_sup_payload);

        /* Payload: MacAddressA */
        hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
        ether_addr_copy(hsr_sp->macaddress_A, master->dev->dev_addr);

        if (skb_put_padto(skb, ETH_ZLEN)) {
                spin_unlock_bh(&hsr->seqnr_lock);
                return;
        }

        hsr_forward_skb(skb, master);
        spin_unlock_bh(&hsr->seqnr_lock);
}

/* Announce (supervision frame) timer function
 */
static void hsr_announce(struct timer_list *t)
{
        struct hsr_priv *hsr;
        struct hsr_port *master;
        unsigned long interval;

        hsr = timer_container_of(hsr, t, announce_timer);

        rcu_read_lock();
        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        hsr->proto_ops->send_sv_frame(master, &interval, master->dev->dev_addr);

        if (is_admin_up(master->dev))
                mod_timer(&hsr->announce_timer, jiffies + interval);

        rcu_read_unlock();
}

/* Announce (supervision frame) timer function for RedBox
 */
static void hsr_proxy_announce(struct timer_list *t)
{
        struct hsr_priv *hsr = timer_container_of(hsr, t,
                                                  announce_proxy_timer);
        struct hsr_port *interlink;
        unsigned long interval = 0;
        struct hsr_node *node;

        rcu_read_lock();
        /* RedBOX sends supervisory frames to HSR network with MAC addresses
         * of SAN nodes stored in ProxyNodeTable.
         */
        interlink = hsr_port_get_hsr(hsr, HSR_PT_INTERLINK);
        if (!interlink)
                goto done;

        list_for_each_entry_rcu(node, &hsr->proxy_node_db, mac_list) {
                if (hsr_addr_is_redbox(hsr, node->macaddress_A))
                        continue;
                hsr->proto_ops->send_sv_frame(interlink, &interval,
                                              node->macaddress_A);
        }

        if (is_admin_up(interlink->dev)) {
                if (!interval)
                        interval = msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL);

                mod_timer(&hsr->announce_proxy_timer, jiffies + interval);
        }

done:
        rcu_read_unlock();
}

void hsr_del_ports(struct hsr_priv *hsr)
{
        struct hsr_port *port;

        port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_A);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_B);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_INTERLINK);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        if (port)
                hsr_del_port(port);
}

static void hsr_set_rx_mode(struct net_device *dev)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port_rtnl(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        dev_mc_sync_multiple(port->dev, dev);
                        dev_uc_sync_multiple(port->dev, dev);
                        break;
                default:
                        break;
                }
        }
}

static void hsr_change_rx_flags(struct net_device *dev, int change)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port_rtnl(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        if (change & IFF_ALLMULTI)
                                dev_set_allmulti(port->dev,
                                                 dev->flags &
                                                 IFF_ALLMULTI ? 1 : -1);
                        break;
                default:
                        break;
                }
        }
}

static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev,
                                   __be16 proto, u16 vid)
{
        bool is_slave_a_added = false;
        bool is_slave_b_added = false;
        struct hsr_port *port;
        struct hsr_priv *hsr;
        int ret = 0;

        hsr = netdev_priv(dev);

        hsr_for_each_port_rtnl(hsr, port) {
                if (port->type == HSR_PT_MASTER ||
                    port->type == HSR_PT_INTERLINK)
                        continue;

                ret = vlan_vid_add(port->dev, proto, vid);
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                        if (ret) {
                                /* clean up Slave-B */
                                netdev_err(dev, "add vid failed for Slave-A\n");
                                if (is_slave_b_added)
                                        vlan_vid_del(port->dev, proto, vid);
                                return ret;
                        }

                        is_slave_a_added = true;
                        break;

                case HSR_PT_SLAVE_B:
                        if (ret) {
                                /* clean up Slave-A */
                                netdev_err(dev, "add vid failed for Slave-B\n");
                                if (is_slave_a_added)
                                        vlan_vid_del(port->dev, proto, vid);
                                return ret;
                        }

                        is_slave_b_added = true;
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static int hsr_ndo_vlan_rx_kill_vid(struct net_device *dev,
                                    __be16 proto, u16 vid)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port_rtnl(hsr, port) {
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        vlan_vid_del(port->dev, proto, vid);
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static const struct net_device_ops hsr_device_ops = {
        .ndo_change_mtu = hsr_dev_change_mtu,
        .ndo_open = hsr_dev_open,
        .ndo_stop = hsr_dev_close,
        .ndo_start_xmit = hsr_dev_xmit,
        .ndo_change_rx_flags = hsr_change_rx_flags,
        .ndo_fix_features = hsr_fix_features,
        .ndo_set_rx_mode = hsr_set_rx_mode,
        .ndo_vlan_rx_add_vid = hsr_ndo_vlan_rx_add_vid,
        .ndo_vlan_rx_kill_vid = hsr_ndo_vlan_rx_kill_vid,
};

static const struct device_type hsr_type = {
        .name = "hsr",
};

static struct hsr_proto_ops hsr_ops = {
        .send_sv_frame = send_hsr_supervision_frame,
        .create_tagged_frame = hsr_create_tagged_frame,
        .get_untagged_frame = hsr_get_untagged_frame,
        .drop_frame = hsr_drop_frame,
        .fill_frame_info = hsr_fill_frame_info,
        .invalid_dan_ingress_frame = hsr_invalid_dan_ingress_frame,
        .register_frame_out = hsr_register_frame_out,
};

static struct hsr_proto_ops prp_ops = {
        .send_sv_frame = send_prp_supervision_frame,
        .create_tagged_frame = prp_create_tagged_frame,
        .get_untagged_frame = prp_get_untagged_frame,
        .drop_frame = prp_drop_frame,
        .fill_frame_info = prp_fill_frame_info,
        .handle_san_frame = prp_handle_san_frame,
        .update_san_info = prp_update_san_info,
        .register_frame_out = prp_register_frame_out,
};

void hsr_dev_setup(struct net_device *dev)
{
        eth_hw_addr_random(dev);

        ether_setup(dev);
        dev->min_mtu = 0;
        dev->header_ops = &hsr_header_ops;
        dev->netdev_ops = &hsr_device_ops;
        SET_NETDEV_DEVTYPE(dev, &hsr_type);
        dev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL;
        /* Prevent recursive tx locking */
        dev->lltx = true;
        /* Not sure about this. Taken from bridge code. netdevice.h says
         * it means "Does not change network namespaces".
         */
        dev->netns_immutable = true;

        dev->needs_free_netdev = true;

        dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA |
                           NETIF_F_GSO_MASK | NETIF_F_HW_CSUM |
                           NETIF_F_HW_VLAN_CTAG_TX |
                           NETIF_F_HW_VLAN_CTAG_FILTER;

        dev->features = dev->hw_features;
}

/* Return true if dev is a HSR master; return false otherwise.
 */
bool is_hsr_master(struct net_device *dev)
{
        return (dev->netdev_ops->ndo_start_xmit == hsr_dev_xmit);
}
EXPORT_SYMBOL(is_hsr_master);

struct net_device *hsr_get_port_ndev(struct net_device *ndev,
                                     enum hsr_port_type pt)
{
        struct hsr_priv *hsr = netdev_priv(ndev);
        struct hsr_port *port;

        rcu_read_lock();
        hsr_for_each_port(hsr, port)
                if (port->type == pt) {
                        dev_hold(port->dev);
                        rcu_read_unlock();
                        return port->dev;
                }
        rcu_read_unlock();
        return NULL;
}
EXPORT_SYMBOL(hsr_get_port_ndev);

/* Default multicast address for HSR Supervision frames */
static const unsigned char def_multicast_addr[ETH_ALEN] __aligned(2) = {
        0x01, 0x15, 0x4e, 0x00, 0x01, 0x00
};

int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2],
                     struct net_device *interlink, unsigned char multicast_spec,
                     u8 protocol_version, struct netlink_ext_ack *extack)
{
        bool unregister = false;
        struct hsr_priv *hsr;
        int res;

        hsr = netdev_priv(hsr_dev);
        INIT_LIST_HEAD(&hsr->ports);
        INIT_LIST_HEAD(&hsr->node_db);
        INIT_LIST_HEAD(&hsr->proxy_node_db);
        spin_lock_init(&hsr->list_lock);

        eth_hw_addr_set(hsr_dev, slave[0]->dev_addr);

        /* initialize protocol specific functions */
        if (protocol_version == PRP_V1) {
                /* For PRP, lan_id has most significant 3 bits holding
                 * the net_id of PRP_LAN_ID
                 */
                hsr->net_id = PRP_LAN_ID << 1;
                hsr->proto_ops = &prp_ops;
        } else {
                hsr->proto_ops = &hsr_ops;
        }

        /* Make sure we recognize frames from ourselves in hsr_rcv() */
        res = hsr_create_self_node(hsr, hsr_dev->dev_addr,
                                   slave[1]->dev_addr);
        if (res < 0)
                return res;

        spin_lock_init(&hsr->seqnr_lock);
        /* Overflow soon to find bugs easier: */
        hsr->sequence_nr = HSR_SEQNR_START;
        hsr->sup_sequence_nr = HSR_SUP_SEQNR_START;

        timer_setup(&hsr->announce_timer, hsr_announce, 0);
        timer_setup(&hsr->prune_timer, hsr_prune_nodes, 0);
        timer_setup(&hsr->prune_proxy_timer, hsr_prune_proxy_nodes, 0);
        timer_setup(&hsr->announce_proxy_timer, hsr_proxy_announce, 0);

        ether_addr_copy(hsr->sup_multicast_addr, def_multicast_addr);
        hsr->sup_multicast_addr[ETH_ALEN - 1] = multicast_spec;

        hsr->prot_version = protocol_version;

        /* Make sure the 1st call to netif_carrier_on() gets through */
        netif_carrier_off(hsr_dev);

        res = hsr_add_port(hsr, hsr_dev, HSR_PT_MASTER, extack);
        if (res)
                goto err_add_master;

        /* HSR forwarding offload supported in lower device? */
        if ((slave[0]->features & NETIF_F_HW_HSR_FWD) &&
            (slave[1]->features & NETIF_F_HW_HSR_FWD))
                hsr->fwd_offloaded = true;

        if ((slave[0]->features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
            (slave[1]->features & NETIF_F_HW_VLAN_CTAG_FILTER))
                hsr_dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;

        res = register_netdevice(hsr_dev);
        if (res)
                goto err_unregister;

        unregister = true;

        res = hsr_add_port(hsr, slave[0], HSR_PT_SLAVE_A, extack);
        if (res)
                goto err_unregister;

        res = hsr_add_port(hsr, slave[1], HSR_PT_SLAVE_B, extack);
        if (res)
                goto err_unregister;

        if (protocol_version == PRP_V1) {
                eth_hw_addr_set(slave[1], slave[0]->dev_addr);
                call_netdevice_notifiers(NETDEV_CHANGEADDR, slave[1]);
        }

        if (interlink) {
                res = hsr_add_port(hsr, interlink, HSR_PT_INTERLINK, extack);
                if (res)
                        goto err_unregister;

                hsr->redbox = true;
                ether_addr_copy(hsr->macaddress_redbox, interlink->dev_addr);
                mod_timer(&hsr->prune_proxy_timer,
                          jiffies + msecs_to_jiffies(PRUNE_PROXY_PERIOD));
        }

        hsr_debugfs_init(hsr, hsr_dev);
        mod_timer(&hsr->prune_timer, jiffies + msecs_to_jiffies(PRUNE_PERIOD));

        return 0;

err_unregister:
        hsr_del_ports(hsr);
err_add_master:
        hsr_del_self_node(hsr);

        if (unregister)
                unregister_netdevice(hsr_dev);
        return res;
}




























































    1 
    1 























































































































    1 




    1 





























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Handshake request lifetime events
 *
 * Author: Chuck Lever <chuck.lever@oracle.com>
 *
 * Copyright (c) 2023, Oracle and/or its affiliates.
 */

#include <linux/types.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/inet.h>
#include <linux/rhashtable.h>

#include <net/sock.h>
#include <net/genetlink.h>
#include <net/netns/generic.h>

#include <kunit/visibility.h>

#include <uapi/linux/handshake.h>
#include "handshake.h"

#include <trace/events/handshake.h>

/*
 * We need both a handshake_req -> sock mapping, and a sock ->
 * handshake_req mapping. Both are one-to-one.
 *
 * To avoid adding another pointer field to struct sock, net/handshake
 * maintains a hash table, indexed by the memory address of @sock, to
 * find the struct handshake_req outstanding for that socket. The
 * reverse direction uses a simple pointer field in the handshake_req
 * struct.
 */

static struct rhashtable handshake_rhashtbl ____cacheline_aligned_in_smp;

static const struct rhashtable_params handshake_rhash_params = {
        .key_len                = sizeof_field(struct handshake_req, hr_sk),
        .key_offset                = offsetof(struct handshake_req, hr_sk),
        .head_offset                = offsetof(struct handshake_req, hr_rhash),
        .automatic_shrinking        = true,
};

int handshake_req_hash_init(void)
{
        return rhashtable_init(&handshake_rhashtbl, &handshake_rhash_params);
}

void handshake_req_hash_destroy(void)
{
        rhashtable_destroy(&handshake_rhashtbl);
}

struct handshake_req *handshake_req_hash_lookup(struct sock *sk)
{
        return rhashtable_lookup_fast(&handshake_rhashtbl, &sk,
                                      handshake_rhash_params);
}
EXPORT_SYMBOL_IF_KUNIT(handshake_req_hash_lookup);

static bool handshake_req_hash_add(struct handshake_req *req)
{
        int ret;

        ret = rhashtable_lookup_insert_fast(&handshake_rhashtbl,
                                            &req->hr_rhash,
                                            handshake_rhash_params);
        return ret == 0;
}

static void handshake_req_destroy(struct handshake_req *req)
{
        if (req->hr_proto->hp_destroy)
                req->hr_proto->hp_destroy(req);
        rhashtable_remove_fast(&handshake_rhashtbl, &req->hr_rhash,
                               handshake_rhash_params);
        kfree(req);
}

static void handshake_sk_destruct(struct sock *sk)
{
        void (*sk_destruct)(struct sock *sk);
        struct handshake_req *req;

        req = handshake_req_hash_lookup(sk);
        if (!req)
                return;

        trace_handshake_destruct(sock_net(sk), req, sk);
        sk_destruct = req->hr_odestruct;
        handshake_req_destroy(req);
        if (sk_destruct)
                sk_destruct(sk);
}

/**
 * handshake_req_alloc - Allocate a handshake request
 * @proto: security protocol
 * @flags: memory allocation flags
 *
 * Returns an initialized handshake_req or NULL.
 */
struct handshake_req *handshake_req_alloc(const struct handshake_proto *proto,
                                          gfp_t flags)
{
        struct handshake_req *req;

        if (!proto)
                return NULL;
        if (proto->hp_handler_class <= HANDSHAKE_HANDLER_CLASS_NONE)
                return NULL;
        if (proto->hp_handler_class >= HANDSHAKE_HANDLER_CLASS_MAX)
                return NULL;
        if (!proto->hp_accept || !proto->hp_done)
                return NULL;

        req = kzalloc(struct_size(req, hr_priv, proto->hp_privsize), flags);
        if (!req)
                return NULL;

        INIT_LIST_HEAD(&req->hr_list);
        req->hr_proto = proto;
        return req;
}
EXPORT_SYMBOL(handshake_req_alloc);

/**
 * handshake_req_private - Get per-handshake private data
 * @req: handshake arguments
 *
 */
void *handshake_req_private(struct handshake_req *req)
{
        return (void *)&req->hr_priv;
}
EXPORT_SYMBOL(handshake_req_private);

static bool __add_pending_locked(struct handshake_net *hn,
                                 struct handshake_req *req)
{
        if (WARN_ON_ONCE(!list_empty(&req->hr_list)))
                return false;
        hn->hn_pending++;
        list_add_tail(&req->hr_list, &hn->hn_requests);
        return true;
}

static void __remove_pending_locked(struct handshake_net *hn,
                                    struct handshake_req *req)
{
        hn->hn_pending--;
        list_del_init(&req->hr_list);
}

/*
 * Returns %true if the request was found on @net's pending list,
 * otherwise %false.
 *
 * If @req was on a pending list, it has not yet been accepted.
 */
static bool remove_pending(struct handshake_net *hn, struct handshake_req *req)
{
        bool ret = false;

        spin_lock(&hn->hn_lock);
        if (!list_empty(&req->hr_list)) {
                __remove_pending_locked(hn, req);
                ret = true;
        }
        spin_unlock(&hn->hn_lock);

        return ret;
}

struct handshake_req *handshake_req_next(struct handshake_net *hn, int class)
{
        struct handshake_req *req, *pos;

        req = NULL;
        spin_lock(&hn->hn_lock);
        list_for_each_entry(pos, &hn->hn_requests, hr_list) {
                if (pos->hr_proto->hp_handler_class != class)
                        continue;
                __remove_pending_locked(hn, pos);
                req = pos;
                break;
        }
        spin_unlock(&hn->hn_lock);

        return req;
}
EXPORT_SYMBOL_IF_KUNIT(handshake_req_next);

/**
 * handshake_req_submit - Submit a handshake request
 * @sock: open socket on which to perform the handshake
 * @req: handshake arguments
 * @flags: memory allocation flags
 *
 * Return values:
 *   %0: Request queued
 *   %-EINVAL: Invalid argument
 *   %-EBUSY: A handshake is already under way for this socket
 *   %-ESRCH: No handshake agent is available
 *   %-EAGAIN: Too many pending handshake requests
 *   %-ENOMEM: Failed to allocate memory
 *   %-EMSGSIZE: Failed to construct notification message
 *   %-EOPNOTSUPP: Handshake module not initialized
 *
 * A zero return value from handshake_req_submit() means that
 * exactly one subsequent completion callback is guaranteed.
 *
 * A negative return value from handshake_req_submit() means that
 * no completion callback will be done and that @req has been
 * destroyed.
 */
int handshake_req_submit(struct socket *sock, struct handshake_req *req,
                         gfp_t flags)
{
        struct handshake_net *hn;
        struct net *net;
        int ret;

        if (!sock || !req || !sock->file) {
                kfree(req);
                return -EINVAL;
        }

        req->hr_sk = sock->sk;
        if (!req->hr_sk) {
                kfree(req);
                return -EINVAL;
        }
        req->hr_odestruct = req->hr_sk->sk_destruct;
        req->hr_sk->sk_destruct = handshake_sk_destruct;

        ret = -EOPNOTSUPP;
        net = sock_net(req->hr_sk);
        hn = handshake_pernet(net);
        if (!hn)
                goto out_err;

        ret = -EAGAIN;
        if (READ_ONCE(hn->hn_pending) >= hn->hn_pending_max)
                goto out_err;

        spin_lock(&hn->hn_lock);
        ret = -EOPNOTSUPP;
        if (test_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags))
                goto out_unlock;
        ret = -EBUSY;
        if (!handshake_req_hash_add(req))
                goto out_unlock;
        if (!__add_pending_locked(hn, req))
                goto out_unlock;
        spin_unlock(&hn->hn_lock);

        ret = handshake_genl_notify(net, req->hr_proto, flags);
        if (ret) {
                trace_handshake_notify_err(net, req, req->hr_sk, ret);
                if (remove_pending(hn, req))
                        goto out_err;
        }

        /* Prevent socket release while a handshake request is pending */
        sock_hold(req->hr_sk);

        trace_handshake_submit(net, req, req->hr_sk);
        return 0;

out_unlock:
        spin_unlock(&hn->hn_lock);
out_err:
        trace_handshake_submit_err(net, req, req->hr_sk, ret);
        handshake_req_destroy(req);
        return ret;
}
EXPORT_SYMBOL(handshake_req_submit);

void handshake_complete(struct handshake_req *req, unsigned int status,
                        struct genl_info *info)
{
        struct sock *sk = req->hr_sk;
        struct net *net = sock_net(sk);

        if (!test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
                trace_handshake_complete(net, req, sk, status);
                req->hr_proto->hp_done(req, status, info);

                /* Handshake request is no longer pending */
                sock_put(sk);
        }
}
EXPORT_SYMBOL_IF_KUNIT(handshake_complete);

/**
 * handshake_req_cancel - Cancel an in-progress handshake
 * @sk: socket on which there is an ongoing handshake
 *
 * Request cancellation races with request completion. To determine
 * who won, callers examine the return value from this function.
 *
 * Return values:
 *   %true - Uncompleted handshake request was canceled
 *   %false - Handshake request already completed or not found
 */
bool handshake_req_cancel(struct sock *sk)
{
        struct handshake_req *req;
        struct handshake_net *hn;
        struct net *net;

        net = sock_net(sk);
        req = handshake_req_hash_lookup(sk);
        if (!req) {
                trace_handshake_cancel_none(net, req, sk);
                return false;
        }

        hn = handshake_pernet(net);
        if (hn && remove_pending(hn, req)) {
                /* Request hadn't been accepted */
                goto out_true;
        }
        if (test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) {
                /* Request already completed */
                trace_handshake_cancel_busy(net, req, sk);
                return false;
        }

out_true:
        trace_handshake_cancel(net, req, sk);

        /* Handshake request is no longer pending */
        sock_put(sk);
        return true;
}
EXPORT_SYMBOL(handshake_req_cancel);












































































































  620 































  606 

   15 


  591 



  592 


  591 









  269 
    5 


  265 



  266 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  698 



   44 









  696 




  697 
    2 
  697 
  693 
  700 
  697 
   25 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/base/power/main.c - Where the driver meets power management.
 *
 * Copyright (c) 2003 Patrick Mochel
 * Copyright (c) 2003 Open Source Development Lab
 *
 * The driver model core calls device_pm_add() when a device is registered.
 * This will initialize the embedded device_pm_info object in the device
 * and add it to the list of power-controlled devices. sysfs entries for
 * controlling device power management will also be added.
 *
 * A separate list is used for keeping track of power info, because the power
 * domain dependencies may differ from the ancestral dependencies that the
 * subsystem list maintains.
 */

#define pr_fmt(fmt) "PM: " fmt
#define dev_fmt pr_fmt

#include <linux/device.h>
#include <linux/export.h>
#include <linux/mutex.h>
#include <linux/pm.h>
#include <linux/pm_runtime.h>
#include <linux/pm-trace.h>
#include <linux/pm_wakeirq.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/async.h>
#include <linux/suspend.h>
#include <trace/events/power.h>
#include <linux/cpufreq.h>
#include <linux/devfreq.h>
#include <linux/timer.h>

#include "../base.h"
#include "power.h"

typedef int (*pm_callback_t)(struct device *);

/*
 * The entries in the dpm_list list are in a depth first order, simply
 * because children are guaranteed to be discovered after parents, and
 * are inserted at the back of the list on discovery.
 *
 * Since device_pm_add() may be called with a device lock held,
 * we must never try to acquire a device lock while holding
 * dpm_list_mutex.
 */

LIST_HEAD(dpm_list);
static LIST_HEAD(dpm_prepared_list);
static LIST_HEAD(dpm_suspended_list);
static LIST_HEAD(dpm_late_early_list);
static LIST_HEAD(dpm_noirq_list);

static DEFINE_MUTEX(dpm_list_mtx);
static pm_message_t pm_transition;

static DEFINE_MUTEX(async_wip_mtx);
static int async_error;

/**
 * pm_hibernate_is_recovering - if recovering from hibernate due to error.
 *
 * Used to query if dev_pm_ops.thaw() is called for normal hibernation case or
 * recovering from some error.
 *
 * Return: true for error case, false for normal case.
 */
bool pm_hibernate_is_recovering(void)
{
        return pm_transition.event == PM_EVENT_RECOVER;
}
EXPORT_SYMBOL_GPL(pm_hibernate_is_recovering);

static const char *pm_verb(int event)
{
        switch (event) {
        case PM_EVENT_SUSPEND:
                return "suspend";
        case PM_EVENT_RESUME:
                return "resume";
        case PM_EVENT_FREEZE:
                return "freeze";
        case PM_EVENT_QUIESCE:
                return "quiesce";
        case PM_EVENT_HIBERNATE:
                return "hibernate";
        case PM_EVENT_THAW:
                return "thaw";
        case PM_EVENT_RESTORE:
                return "restore";
        case PM_EVENT_RECOVER:
                return "recover";
        default:
                return "(unknown PM event)";
        }
}

/**
 * device_pm_sleep_init - Initialize system suspend-related device fields.
 * @dev: Device object being initialized.
 */
void device_pm_sleep_init(struct device *dev)
{
        dev->power.is_prepared = false;
        dev->power.is_suspended = false;
        dev->power.is_noirq_suspended = false;
        dev->power.is_late_suspended = false;
        init_completion(&dev->power.completion);
        complete_all(&dev->power.completion);
        dev->power.wakeup = NULL;
        INIT_LIST_HEAD(&dev->power.entry);
}

/**
 * device_pm_lock - Lock the list of active devices used by the PM core.
 */
void device_pm_lock(void)
{
        mutex_lock(&dpm_list_mtx);
}

/**
 * device_pm_unlock - Unlock the list of active devices used by the PM core.
 */
void device_pm_unlock(void)
{
        mutex_unlock(&dpm_list_mtx);
}

/**
 * device_pm_add - Add a device to the PM core's list of active devices.
 * @dev: Device to add to the list.
 */
void device_pm_add(struct device *dev)
{
        /* Skip PM setup/initialization. */
        if (device_pm_not_required(dev))
                return;

        pr_debug("Adding info for %s:%s\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        device_pm_check_callbacks(dev);
        mutex_lock(&dpm_list_mtx);
        if (dev->parent && dev->parent->power.is_prepared)
                dev_warn(dev, "parent %s should not be sleeping\n",
                        dev_name(dev->parent));
        list_add_tail(&dev->power.entry, &dpm_list);
        dev->power.in_dpm_list = true;
        mutex_unlock(&dpm_list_mtx);
}

/**
 * device_pm_remove - Remove a device from the PM core's list of active devices.
 * @dev: Device to be removed from the list.
 */
void device_pm_remove(struct device *dev)
{
        if (device_pm_not_required(dev))
                return;

        pr_debug("Removing info for %s:%s\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        complete_all(&dev->power.completion);
        mutex_lock(&dpm_list_mtx);
        list_del_init(&dev->power.entry);
        dev->power.in_dpm_list = false;
        mutex_unlock(&dpm_list_mtx);
        device_wakeup_disable(dev);
        pm_runtime_remove(dev);
        device_pm_check_callbacks(dev);
}

/**
 * device_pm_move_before - Move device in the PM core's list of active devices.
 * @deva: Device to move in dpm_list.
 * @devb: Device @deva should come before.
 */
void device_pm_move_before(struct device *deva, struct device *devb)
{
        pr_debug("Moving %s:%s before %s:%s\n",
                 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
                 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
        /* Delete deva from dpm_list and reinsert before devb. */
        list_move_tail(&deva->power.entry, &devb->power.entry);
}

/**
 * device_pm_move_after - Move device in the PM core's list of active devices.
 * @deva: Device to move in dpm_list.
 * @devb: Device @deva should come after.
 */
void device_pm_move_after(struct device *deva, struct device *devb)
{
        pr_debug("Moving %s:%s after %s:%s\n",
                 deva->bus ? deva->bus->name : "No Bus", dev_name(deva),
                 devb->bus ? devb->bus->name : "No Bus", dev_name(devb));
        /* Delete deva from dpm_list and reinsert after devb. */
        list_move(&deva->power.entry, &devb->power.entry);
}

/**
 * device_pm_move_last - Move device to end of the PM core's list of devices.
 * @dev: Device to move in dpm_list.
 */
void device_pm_move_last(struct device *dev)
{
        pr_debug("Moving %s:%s to end of list\n",
                 dev->bus ? dev->bus->name : "No Bus", dev_name(dev));
        list_move_tail(&dev->power.entry, &dpm_list);
}

static ktime_t initcall_debug_start(struct device *dev, void *cb)
{
        if (!pm_print_times_enabled)
                return 0;

        dev_info(dev, "calling %ps @ %i, parent: %s\n", cb,
                 task_pid_nr(current),
                 dev->parent ? dev_name(dev->parent) : "none");
        return ktime_get();
}

static void initcall_debug_report(struct device *dev, ktime_t calltime,
                                  void *cb, int error)
{
        ktime_t rettime;

        if (!pm_print_times_enabled)
                return;

        rettime = ktime_get();
        dev_info(dev, "%ps returned %d after %Ld usecs\n", cb, error,
                 (unsigned long long)ktime_us_delta(rettime, calltime));
}

/**
 * dpm_wait - Wait for a PM operation to complete.
 * @dev: Device to wait for.
 * @async: If unset, wait only if the device's power.async_suspend flag is set.
 */
static void dpm_wait(struct device *dev, bool async)
{
        if (!dev)
                return;

        if (async || (pm_async_enabled && dev->power.async_suspend))
                wait_for_completion(&dev->power.completion);
}

static int dpm_wait_fn(struct device *dev, void *async_ptr)
{
        dpm_wait(dev, *((bool *)async_ptr));
        return 0;
}

static void dpm_wait_for_children(struct device *dev, bool async)
{
        device_for_each_child(dev, &async, dpm_wait_fn);
}

static void dpm_wait_for_suppliers(struct device *dev, bool async)
{
        struct device_link *link;
        int idx;

        idx = device_links_read_lock();

        /*
         * If the supplier goes away right after we've checked the link to it,
         * we'll wait for its completion to change the state, but that's fine,
         * because the only things that will block as a result are the SRCU
         * callbacks freeing the link objects for the links in the list we're
         * walking.
         */
        dev_for_each_link_to_supplier(link, dev)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT &&
                    !device_link_flag_is_sync_state_only(link->flags))
                        dpm_wait(link->supplier, async);

        device_links_read_unlock(idx);
}

static bool dpm_wait_for_superior(struct device *dev, bool async)
{
        struct device *parent;

        /*
         * If the device is resumed asynchronously and the parent's callback
         * deletes both the device and the parent itself, the parent object may
         * be freed while this function is running, so avoid that by reference
         * counting the parent once more unless the device has been deleted
         * already (in which case return right away).
         */
        mutex_lock(&dpm_list_mtx);

        if (!device_pm_initialized(dev)) {
                mutex_unlock(&dpm_list_mtx);
                return false;
        }

        parent = get_device(dev->parent);

        mutex_unlock(&dpm_list_mtx);

        dpm_wait(parent, async);
        put_device(parent);

        dpm_wait_for_suppliers(dev, async);

        /*
         * If the parent's callback has deleted the device, attempting to resume
         * it would be invalid, so avoid doing that then.
         */
        return device_pm_initialized(dev);
}

static void dpm_wait_for_consumers(struct device *dev, bool async)
{
        struct device_link *link;
        int idx;

        idx = device_links_read_lock();

        /*
         * The status of a device link can only be changed from "dormant" by a
         * probe, but that cannot happen during system suspend/resume.  In
         * theory it can change to "dormant" at that time, but then it is
         * reasonable to wait for the target device anyway (eg. if it goes
         * away, it's better to wait for it to go away completely and then
         * continue instead of trying to continue in parallel with its
         * unregistration).
         */
        dev_for_each_link_to_consumer(link, dev)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT &&
                    !device_link_flag_is_sync_state_only(link->flags))
                        dpm_wait(link->consumer, async);

        device_links_read_unlock(idx);
}

static void dpm_wait_for_subordinate(struct device *dev, bool async)
{
        dpm_wait_for_children(dev, async);
        dpm_wait_for_consumers(dev, async);
}

/**
 * pm_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 */
static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend;
        case PM_EVENT_RESUME:
                return ops->resume;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw;
        case PM_EVENT_RESTORE:
                return ops->restore;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

/**
 * pm_late_early_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops,
                                      pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend_late;
        case PM_EVENT_RESUME:
                return ops->resume_early;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze_late;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff_late;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw_early;
        case PM_EVENT_RESTORE:
                return ops->restore_early;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

/**
 * pm_noirq_op - Return the PM operation appropriate for given PM event.
 * @ops: PM operations to choose from.
 * @state: PM transition of the system being carried out.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t state)
{
        switch (state.event) {
#ifdef CONFIG_SUSPEND
        case PM_EVENT_SUSPEND:
                return ops->suspend_noirq;
        case PM_EVENT_RESUME:
                return ops->resume_noirq;
#endif /* CONFIG_SUSPEND */
#ifdef CONFIG_HIBERNATE_CALLBACKS
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return ops->freeze_noirq;
        case PM_EVENT_HIBERNATE:
                return ops->poweroff_noirq;
        case PM_EVENT_THAW:
        case PM_EVENT_RECOVER:
                return ops->thaw_noirq;
        case PM_EVENT_RESTORE:
                return ops->restore_noirq;
#endif /* CONFIG_HIBERNATE_CALLBACKS */
        }

        return NULL;
}

static void pm_dev_dbg(struct device *dev, pm_message_t state, const char *info)
{
        dev_dbg(dev, "%s%s%s driver flags: %x\n", info, pm_verb(state.event),
                ((state.event & PM_EVENT_SLEEP) && device_may_wakeup(dev)) ?
                ", may wakeup" : "", dev->power.driver_flags);
}

static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
                        int error)
{
        dev_err(dev, "failed to %s%s: error %d\n", pm_verb(state.event), info,
                error);
}

static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
                          const char *info)
{
        ktime_t calltime;
        u64 usecs64;
        int usecs;

        calltime = ktime_get();
        usecs64 = ktime_to_ns(ktime_sub(calltime, starttime));
        do_div(usecs64, NSEC_PER_USEC);
        usecs = usecs64;
        if (usecs == 0)
                usecs = 1;

        pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
                  info ?: "", info ? " " : "", pm_verb(state.event),
                  error ? "aborted" : "complete",
                  usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
}

static int dpm_run_callback(pm_callback_t cb, struct device *dev,
                            pm_message_t state, const char *info)
{
        ktime_t calltime;
        int error;

        if (!cb)
                return 0;

        calltime = initcall_debug_start(dev, cb);

        pm_dev_dbg(dev, state, info);
        trace_device_pm_callback_start(dev, info, state.event);
        error = cb(dev);
        trace_device_pm_callback_end(dev, error);
        suspend_report_result(dev, cb, error);

        initcall_debug_report(dev, calltime, cb, error);

        return error;
}

#ifdef CONFIG_DPM_WATCHDOG
struct dpm_watchdog {
        struct device                *dev;
        struct task_struct        *tsk;
        struct timer_list        timer;
        bool                        fatal;
};

#define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \
        struct dpm_watchdog wd

/**
 * dpm_watchdog_handler - Driver suspend / resume watchdog handler.
 * @t: The timer that PM watchdog depends on.
 *
 * Called when a driver has timed out suspending or resuming.
 * There's not much we can do here to recover so panic() to
 * capture a crash-dump in pstore.
 */
static void dpm_watchdog_handler(struct timer_list *t)
{
        struct dpm_watchdog *wd = timer_container_of(wd, t, timer);
        struct timer_list *timer = &wd->timer;
        unsigned int time_left;

        if (wd->fatal) {
                dev_emerg(wd->dev, "**** DPM device timeout ****\n");
                show_stack(wd->tsk, NULL, KERN_EMERG);
                panic("%s %s: unrecoverable failure\n",
                        dev_driver_string(wd->dev), dev_name(wd->dev));
        }

        time_left = CONFIG_DPM_WATCHDOG_TIMEOUT - CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT;
        dev_warn(wd->dev, "**** DPM device timeout after %u seconds; %u seconds until panic ****\n",
                 CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT, time_left);
        show_stack(wd->tsk, NULL, KERN_WARNING);

        wd->fatal = true;
        mod_timer(timer, jiffies + HZ * time_left);
}

/**
 * dpm_watchdog_set - Enable pm watchdog for given device.
 * @wd: Watchdog. Must be allocated on the stack.
 * @dev: Device to handle.
 */
static void dpm_watchdog_set(struct dpm_watchdog *wd, struct device *dev)
{
        struct timer_list *timer = &wd->timer;

        wd->dev = dev;
        wd->tsk = current;
        wd->fatal = CONFIG_DPM_WATCHDOG_TIMEOUT == CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT;

        timer_setup_on_stack(timer, dpm_watchdog_handler, 0);
        /* use same timeout value for both suspend and resume */
        timer->expires = jiffies + HZ * CONFIG_DPM_WATCHDOG_WARNING_TIMEOUT;
        add_timer(timer);
}

/**
 * dpm_watchdog_clear - Disable suspend/resume watchdog.
 * @wd: Watchdog to disable.
 */
static void dpm_watchdog_clear(struct dpm_watchdog *wd)
{
        struct timer_list *timer = &wd->timer;

        timer_delete_sync(timer);
        timer_destroy_on_stack(timer);
}
#else
#define DECLARE_DPM_WATCHDOG_ON_STACK(wd)
#define dpm_watchdog_set(x, y)
#define dpm_watchdog_clear(x)
#endif

/*------------------------- Resume routines -------------------------*/

/**
 * dev_pm_skip_resume - System-wide device resume optimization check.
 * @dev: Target device.
 *
 * Return:
 * - %false if the transition under way is RESTORE.
 * - Return value of dev_pm_skip_suspend() if the transition under way is THAW.
 * - The logical negation of %power.must_resume otherwise (that is, when the
 *   transition under way is RESUME).
 */
bool dev_pm_skip_resume(struct device *dev)
{
        if (pm_transition.event == PM_EVENT_RESTORE)
                return false;

        if (pm_transition.event == PM_EVENT_THAW)
                return dev_pm_skip_suspend(dev);

        return !dev->power.must_resume;
}

static bool is_async(struct device *dev)
{
        return dev->power.async_suspend && pm_async_enabled
                && !pm_trace_is_enabled();
}

static bool __dpm_async(struct device *dev, async_func_t func)
{
        if (dev->power.work_in_progress)
                return true;

        if (!is_async(dev))
                return false;

        dev->power.work_in_progress = true;

        get_device(dev);

        if (async_schedule_dev_nocall(func, dev))
                return true;

        put_device(dev);

        return false;
}

static bool dpm_async_fn(struct device *dev, async_func_t func)
{
        guard(mutex)(&async_wip_mtx);

        return __dpm_async(dev, func);
}

static int dpm_async_with_cleanup(struct device *dev, void *fn)
{
        guard(mutex)(&async_wip_mtx);

        if (!__dpm_async(dev, fn))
                dev->power.work_in_progress = false;

        return 0;
}

static void dpm_async_resume_children(struct device *dev, async_func_t func)
{
        /*
         * Prevent racing with dpm_clear_async_state() during initial list
         * walks in dpm_noirq_resume_devices(), dpm_resume_early(), and
         * dpm_resume().
         */
        guard(mutex)(&dpm_list_mtx);

        /*
         * Start processing "async" children of the device unless it's been
         * started already for them.
         */
        device_for_each_child(dev, func, dpm_async_with_cleanup);
}

static void dpm_async_resume_subordinate(struct device *dev, async_func_t func)
{
        struct device_link *link;
        int idx;

        dpm_async_resume_children(dev, func);

        idx = device_links_read_lock();

        /* Start processing the device's "async" consumers. */
        dev_for_each_link_to_consumer(link, dev)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT)
                        dpm_async_with_cleanup(link->consumer, func);

        device_links_read_unlock(idx);
}

static void dpm_clear_async_state(struct device *dev)
{
        reinit_completion(&dev->power.completion);
        dev->power.work_in_progress = false;
}

static bool dpm_root_device(struct device *dev)
{
        lockdep_assert_held(&dpm_list_mtx);

        /*
         * Since this function is required to run under dpm_list_mtx, the
         * list_empty() below will only return true if the device's list of
         * consumers is actually empty before calling it.
         */
        return !dev->parent && list_empty(&dev->links.suppliers);
}

static void async_resume_noirq(void *data, async_cookie_t cookie);

/**
 * device_resume_noirq - Execute a "noirq resume" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static void device_resume_noirq(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        bool skip_resume;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore || dev->power.direct_complete)
                goto Out;

        if (!dev->power.is_noirq_suspended) {
                /*
                 * This means that system suspend has been aborted in the noirq
                 * phase before invoking the noirq suspend callback for the
                 * device, so if device_suspend_late() has left it in suspend,
                 * device_resume_early() should leave it in suspend either in
                 * case the early resume of it depends on the noirq resume that
                 * has not run.
                 */
                if (dev_pm_skip_suspend(dev))
                        dev->power.must_resume = false;

                goto Out;
        }

        if (!dpm_wait_for_superior(dev, async))
                goto Out;

        skip_resume = dev_pm_skip_resume(dev);
        /*
         * If the driver callback is skipped below or by the middle layer
         * callback and device_resume_early() also skips the driver callback for
         * this device later, it needs to appear as "suspended" to PM-runtime,
         * so change its status accordingly.
         *
         * Otherwise, the device is going to be resumed, so set its PM-runtime
         * status to "active" unless its power.smart_suspend flag is clear, in
         * which case it is not necessary to update its PM-runtime status.
         */
        if (skip_resume)
                pm_runtime_set_suspended(dev);
        else if (dev_pm_smart_suspend(dev))
                pm_runtime_set_active(dev);

        if (dev->pm_domain) {
                info = "noirq power domain ";
                callback = pm_noirq_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "noirq type ";
                callback = pm_noirq_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "noirq class ";
                callback = pm_noirq_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "noirq bus ";
                callback = pm_noirq_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (skip_resume)
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "noirq driver ";
                callback = pm_noirq_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);

Skip:
        dev->power.is_noirq_suspended = false;

Out:
        complete_all(&dev->power.completion);
        TRACE_RESUME(error);

        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
        }

        dpm_async_resume_subordinate(dev, async_resume_noirq);
}

static void async_resume_noirq(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_resume_noirq(dev, pm_transition, true);
        put_device(dev);
}

static void dpm_noirq_resume_devices(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, true);

        async_error = 0;
        pm_transition = state;

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" root devices upfront so they don't wait for
         * the "sync" devices they don't depend on.
         */
        list_for_each_entry(dev, &dpm_noirq_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_root_device(dev))
                        dpm_async_with_cleanup(dev, async_resume_noirq);
        }

        while (!list_empty(&dpm_noirq_list)) {
                dev = to_device(dpm_noirq_list.next);
                list_move_tail(&dev->power.entry, &dpm_late_early_list);

                if (!dpm_async_fn(dev, async_resume_noirq)) {
                        get_device(dev);

                        mutex_unlock(&dpm_list_mtx);

                        device_resume_noirq(dev, state, false);

                        put_device(dev);

                        mutex_lock(&dpm_list_mtx);
                }
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, "noirq");
        if (READ_ONCE(async_error))
                dpm_save_failed_step(SUSPEND_RESUME_NOIRQ);

        trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
}

/**
 * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 *
 * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
 * allow device drivers' interrupt handlers to be called.
 */
void dpm_resume_noirq(pm_message_t state)
{
        dpm_noirq_resume_devices(state);

        resume_device_irqs();
        device_wakeup_disarm_wake_irqs();
}

static void async_resume_early(void *data, async_cookie_t cookie);

/**
 * device_resume_early - Execute an "early resume" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static void device_resume_early(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore || dev->power.direct_complete)
                goto Out;

        if (!dev->power.is_late_suspended)
                goto Out;

        if (!dpm_wait_for_superior(dev, async))
                goto Out;

        if (dev->pm_domain) {
                info = "early power domain ";
                callback = pm_late_early_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "early type ";
                callback = pm_late_early_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "early class ";
                callback = pm_late_early_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "early bus ";
                callback = pm_late_early_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_resume(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "early driver ";
                callback = pm_late_early_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);

Skip:
        dev->power.is_late_suspended = false;

Out:
        TRACE_RESUME(error);

        pm_runtime_enable(dev);
        complete_all(&dev->power.completion);

        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async early" : " early", error);
        }

        dpm_async_resume_subordinate(dev, async_resume_early);
}

static void async_resume_early(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_resume_early(dev, pm_transition, true);
        put_device(dev);
}

/**
 * dpm_resume_early - Execute "early resume" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 */
void dpm_resume_early(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume_early"), state.event, true);

        async_error = 0;
        pm_transition = state;

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" root devices upfront so they don't wait for
         * the "sync" devices they don't depend on.
         */
        list_for_each_entry(dev, &dpm_late_early_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_root_device(dev))
                        dpm_async_with_cleanup(dev, async_resume_early);
        }

        while (!list_empty(&dpm_late_early_list)) {
                dev = to_device(dpm_late_early_list.next);
                list_move_tail(&dev->power.entry, &dpm_suspended_list);

                if (!dpm_async_fn(dev, async_resume_early)) {
                        get_device(dev);

                        mutex_unlock(&dpm_list_mtx);

                        device_resume_early(dev, state, false);

                        put_device(dev);

                        mutex_lock(&dpm_list_mtx);
                }
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, "early");
        if (READ_ONCE(async_error))
                dpm_save_failed_step(SUSPEND_RESUME_EARLY);

        trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
}

/**
 * dpm_resume_start - Execute "noirq" and "early" device callbacks.
 * @state: PM transition of the system being carried out.
 */
void dpm_resume_start(pm_message_t state)
{
        dpm_resume_noirq(state);
        dpm_resume_early(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_start);

static void async_resume(void *data, async_cookie_t cookie);

/**
 * device_resume - Execute "resume" callbacks for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being resumed asynchronously.
 */
static void device_resume(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;
        DECLARE_DPM_WATCHDOG_ON_STACK(wd);

        TRACE_DEVICE(dev);
        TRACE_RESUME(0);

        if (dev->power.syscore)
                goto Complete;

        if (!dev->power.is_suspended)
                goto Complete;

        dev->power.is_suspended = false;

        if (dev->power.direct_complete) {
                /*
                 * Allow new children to be added under the device after this
                 * point if it has no PM callbacks.
                 */
                if (dev->power.no_pm_callbacks)
                        dev->power.is_prepared = false;

                /* Match the pm_runtime_disable() in device_suspend(). */
                pm_runtime_enable(dev);
                goto Complete;
        }

        if (!dpm_wait_for_superior(dev, async))
                goto Complete;

        dpm_watchdog_set(&wd, dev);
        device_lock(dev);

        /*
         * This is a fib.  But we'll allow new children to be added below
         * a resumed device, even if the device hasn't been completed yet.
         */
        dev->power.is_prepared = false;

        if (dev->pm_domain) {
                info = "power domain ";
                callback = pm_op(&dev->pm_domain->ops, state);
                goto Driver;
        }

        if (dev->type && dev->type->pm) {
                info = "type ";
                callback = pm_op(dev->type->pm, state);
                goto Driver;
        }

        if (dev->class && dev->class->pm) {
                info = "class ";
                callback = pm_op(dev->class->pm, state);
                goto Driver;
        }

        if (dev->bus) {
                if (dev->bus->pm) {
                        info = "bus ";
                        callback = pm_op(dev->bus->pm, state);
                } else if (dev->bus->resume) {
                        info = "legacy bus ";
                        callback = dev->bus->resume;
                        goto End;
                }
        }

 Driver:
        if (!callback && dev->driver && dev->driver->pm) {
                info = "driver ";
                callback = pm_op(dev->driver->pm, state);
        }

 End:
        error = dpm_run_callback(callback, dev, state, info);

        device_unlock(dev);
        dpm_watchdog_clear(&wd);

 Complete:
        complete_all(&dev->power.completion);

        TRACE_RESUME(error);

        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async" : "", error);
        }

        dpm_async_resume_subordinate(dev, async_resume);
}

static void async_resume(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_resume(dev, pm_transition, true);
        put_device(dev);
}

/**
 * dpm_resume - Execute "resume" callbacks for non-sysdev devices.
 * @state: PM transition of the system being carried out.
 *
 * Execute the appropriate "resume" callback for all devices whose status
 * indicates that they are suspended.
 */
void dpm_resume(pm_message_t state)
{
        struct device *dev;
        ktime_t starttime = ktime_get();

        trace_suspend_resume(TPS("dpm_resume"), state.event, true);

        pm_transition = state;
        async_error = 0;

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" root devices upfront so they don't wait for
         * the "sync" devices they don't depend on.
         */
        list_for_each_entry(dev, &dpm_suspended_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_root_device(dev))
                        dpm_async_with_cleanup(dev, async_resume);
        }

        while (!list_empty(&dpm_suspended_list)) {
                dev = to_device(dpm_suspended_list.next);
                list_move_tail(&dev->power.entry, &dpm_prepared_list);

                if (!dpm_async_fn(dev, async_resume)) {
                        get_device(dev);

                        mutex_unlock(&dpm_list_mtx);

                        device_resume(dev, state, false);

                        put_device(dev);

                        mutex_lock(&dpm_list_mtx);
                }
        }
        mutex_unlock(&dpm_list_mtx);
        async_synchronize_full();
        dpm_show_time(starttime, state, 0, NULL);
        if (READ_ONCE(async_error))
                dpm_save_failed_step(SUSPEND_RESUME);

        cpufreq_resume();
        devfreq_resume();
        trace_suspend_resume(TPS("dpm_resume"), state.event, false);
}

/**
 * device_complete - Complete a PM transition for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 */
static void device_complete(struct device *dev, pm_message_t state)
{
        void (*callback)(struct device *) = NULL;
        const char *info = NULL;

        if (dev->power.syscore)
                goto out;

        device_lock(dev);

        if (dev->pm_domain) {
                info = "completing power domain ";
                callback = dev->pm_domain->ops.complete;
        } else if (dev->type && dev->type->pm) {
                info = "completing type ";
                callback = dev->type->pm->complete;
        } else if (dev->class && dev->class->pm) {
                info = "completing class ";
                callback = dev->class->pm->complete;
        } else if (dev->bus && dev->bus->pm) {
                info = "completing bus ";
                callback = dev->bus->pm->complete;
        }

        if (!callback && dev->driver && dev->driver->pm) {
                info = "completing driver ";
                callback = dev->driver->pm->complete;
        }

        if (callback) {
                pm_dev_dbg(dev, state, info);
                callback(dev);
        }

        device_unlock(dev);

out:
        /* If enabling runtime PM for the device is blocked, unblock it. */
        pm_runtime_unblock(dev);
        pm_runtime_put(dev);
}

/**
 * dpm_complete - Complete a PM transition for all non-sysdev devices.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->complete() callbacks for all devices whose PM status is not
 * DPM_ON (this allows new devices to be registered).
 */
void dpm_complete(pm_message_t state)
{
        struct list_head list;

        trace_suspend_resume(TPS("dpm_complete"), state.event, true);

        INIT_LIST_HEAD(&list);
        mutex_lock(&dpm_list_mtx);
        while (!list_empty(&dpm_prepared_list)) {
                struct device *dev = to_device(dpm_prepared_list.prev);

                get_device(dev);
                dev->power.is_prepared = false;
                list_move(&dev->power.entry, &list);

                mutex_unlock(&dpm_list_mtx);

                trace_device_pm_callback_start(dev, "", state.event);
                device_complete(dev, state);
                trace_device_pm_callback_end(dev, 0);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        list_splice(&list, &dpm_list);
        mutex_unlock(&dpm_list_mtx);

        /* Allow device probing and trigger re-probing of deferred devices */
        device_unblock_probing();
        trace_suspend_resume(TPS("dpm_complete"), state.event, false);
}

/**
 * dpm_resume_end - Execute "resume" callbacks and complete system transition.
 * @state: PM transition of the system being carried out.
 *
 * Execute "resume" callbacks for all devices and complete the PM transition of
 * the system.
 */
void dpm_resume_end(pm_message_t state)
{
        dpm_resume(state);
        pm_restore_gfp_mask();
        dpm_complete(state);
}
EXPORT_SYMBOL_GPL(dpm_resume_end);


/*------------------------- Suspend routines -------------------------*/

static bool dpm_leaf_device(struct device *dev)
{
        struct device *child;

        lockdep_assert_held(&dpm_list_mtx);

        child = device_find_any_child(dev);
        if (child) {
                put_device(child);

                return false;
        }

        /*
         * Since this function is required to run under dpm_list_mtx, the
         * list_empty() below will only return true if the device's list of
         * consumers is actually empty before calling it.
         */
        return list_empty(&dev->links.consumers);
}

static bool dpm_async_suspend_parent(struct device *dev, async_func_t func)
{
        guard(mutex)(&dpm_list_mtx);

        /*
         * If the device is suspended asynchronously and the parent's callback
         * deletes both the device and the parent itself, the parent object may
         * be freed while this function is running, so avoid that by checking
         * if the device has been deleted already as the parent cannot be
         * deleted before it.
         */
        if (!device_pm_initialized(dev))
                return false;

        /* Start processing the device's parent if it is "async". */
        if (dev->parent)
                dpm_async_with_cleanup(dev->parent, func);

        return true;
}

static void dpm_async_suspend_superior(struct device *dev, async_func_t func)
{
        struct device_link *link;
        int idx;

        if (!dpm_async_suspend_parent(dev, func))
                return;

        idx = device_links_read_lock();

        /* Start processing the device's "async" suppliers. */
        dev_for_each_link_to_supplier(link, dev)
                if (READ_ONCE(link->status) != DL_STATE_DORMANT)
                        dpm_async_with_cleanup(link->supplier, func);

        device_links_read_unlock(idx);
}

static void dpm_async_suspend_complete_all(struct list_head *device_list)
{
        struct device *dev;

        guard(mutex)(&async_wip_mtx);

        list_for_each_entry_reverse(dev, device_list, power.entry) {
                /*
                 * In case the device is being waited for and async processing
                 * has not started for it yet, let the waiters make progress.
                 */
                if (!dev->power.work_in_progress)
                        complete_all(&dev->power.completion);
        }
}

/**
 * resume_event - Return a "resume" message for given "suspend" sleep state.
 * @sleep_state: PM message representing a sleep state.
 *
 * Return a PM message representing the resume event corresponding to given
 * sleep state.
 */
static pm_message_t resume_event(pm_message_t sleep_state)
{
        switch (sleep_state.event) {
        case PM_EVENT_SUSPEND:
                return PMSG_RESUME;
        case PM_EVENT_FREEZE:
        case PM_EVENT_QUIESCE:
                return PMSG_RECOVER;
        case PM_EVENT_HIBERNATE:
                return PMSG_RESTORE;
        }
        return PMSG_ON;
}

static void dpm_superior_set_must_resume(struct device *dev)
{
        struct device_link *link;
        int idx;

        if (dev->parent)
                dev->parent->power.must_resume = true;

        idx = device_links_read_lock();

        dev_for_each_link_to_supplier(link, dev)
                link->supplier->power.must_resume = true;

        device_links_read_unlock(idx);
}

static void async_suspend_noirq(void *data, async_cookie_t cookie);

/**
 * device_suspend_noirq - Execute a "noirq suspend" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 *
 * The driver of @dev will not receive interrupts while this function is being
 * executed.
 */
static void device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        dpm_wait_for_subordinate(dev, async);

        if (READ_ONCE(async_error))
                goto Complete;

        if (dev->power.syscore || dev->power.direct_complete)
                goto Complete;

        if (dev->pm_domain) {
                info = "noirq power domain ";
                callback = pm_noirq_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "noirq type ";
                callback = pm_noirq_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "noirq class ";
                callback = pm_noirq_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "noirq bus ";
                callback = pm_noirq_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_suspend(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "noirq driver ";
                callback = pm_noirq_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);
        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
                goto Complete;
        }

Skip:
        dev->power.is_noirq_suspended = true;

        /*
         * Devices must be resumed unless they are explicitly allowed to be left
         * in suspend, but even in that case skipping the resume of devices that
         * were in use right before the system suspend (as indicated by their
         * runtime PM usage counters and child counters) would be suboptimal.
         */
        if (!(dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME) &&
              dev->power.may_skip_resume) || !pm_runtime_need_not_resume(dev))
                dev->power.must_resume = true;

        if (dev->power.must_resume)
                dpm_superior_set_must_resume(dev);

Complete:
        complete_all(&dev->power.completion);
        TRACE_SUSPEND(error);

        if (error || READ_ONCE(async_error))
                return;

        dpm_async_suspend_superior(dev, async_suspend_noirq);
}

static void async_suspend_noirq(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_suspend_noirq(dev, pm_transition, true);
        put_device(dev);
}

static int dpm_noirq_suspend_devices(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        struct device *dev;
        int error;

        trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);

        pm_transition = state;
        async_error = 0;

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" leaf devices upfront so they don't need to
         * wait for the "sync" devices they don't depend on.
         */
        list_for_each_entry_reverse(dev, &dpm_late_early_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_leaf_device(dev))
                        dpm_async_with_cleanup(dev, async_suspend_noirq);
        }

        while (!list_empty(&dpm_late_early_list)) {
                dev = to_device(dpm_late_early_list.prev);

                list_move(&dev->power.entry, &dpm_noirq_list);

                if (dpm_async_fn(dev, async_suspend_noirq))
                        continue;

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                device_suspend_noirq(dev, state, false);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (READ_ONCE(async_error)) {
                        dpm_async_suspend_complete_all(&dpm_late_early_list);
                        /*
                         * Move all devices to the target list to resume them
                         * properly.
                         */
                        list_splice_init(&dpm_late_early_list, &dpm_noirq_list);
                        break;
                }
        }

        mutex_unlock(&dpm_list_mtx);

        async_synchronize_full();

        error = READ_ONCE(async_error);
        if (error)
                dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);

        dpm_show_time(starttime, state, error, "noirq");
        trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
        return error;
}

/**
 * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 *
 * Prevent device drivers' interrupt handlers from being called and invoke
 * "noirq" suspend callbacks for all non-sysdev devices.
 */
int dpm_suspend_noirq(pm_message_t state)
{
        int ret;

        device_wakeup_arm_wake_irqs();
        suspend_device_irqs();

        ret = dpm_noirq_suspend_devices(state);
        if (ret)
                dpm_resume_noirq(resume_event(state));

        return ret;
}

static void dpm_propagate_wakeup_to_parent(struct device *dev)
{
        struct device *parent = dev->parent;

        if (!parent)
                return;

        spin_lock_irq(&parent->power.lock);

        if (device_wakeup_path(dev) && !parent->power.ignore_children)
                parent->power.wakeup_path = true;

        spin_unlock_irq(&parent->power.lock);
}

static void async_suspend_late(void *data, async_cookie_t cookie);

/**
 * device_suspend_late - Execute a "late suspend" callback for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 *
 * Runtime PM is disabled for @dev while this function is being executed.
 */
static void device_suspend_late(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        /*
         * Disable runtime PM for the device without checking if there is a
         * pending resume request for it.
         */
        __pm_runtime_disable(dev, false);

        dpm_wait_for_subordinate(dev, async);

        if (READ_ONCE(async_error))
                goto Complete;

        if (pm_wakeup_pending()) {
                WRITE_ONCE(async_error, -EBUSY);
                goto Complete;
        }

        if (dev->power.syscore || dev->power.direct_complete)
                goto Complete;

        if (dev->pm_domain) {
                info = "late power domain ";
                callback = pm_late_early_op(&dev->pm_domain->ops, state);
        } else if (dev->type && dev->type->pm) {
                info = "late type ";
                callback = pm_late_early_op(dev->type->pm, state);
        } else if (dev->class && dev->class->pm) {
                info = "late class ";
                callback = pm_late_early_op(dev->class->pm, state);
        } else if (dev->bus && dev->bus->pm) {
                info = "late bus ";
                callback = pm_late_early_op(dev->bus->pm, state);
        }
        if (callback)
                goto Run;

        if (dev_pm_skip_suspend(dev))
                goto Skip;

        if (dev->driver && dev->driver->pm) {
                info = "late driver ";
                callback = pm_late_early_op(dev->driver->pm, state);
        }

Run:
        error = dpm_run_callback(callback, dev, state, info);
        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async late" : " late", error);
                goto Complete;
        }
        dpm_propagate_wakeup_to_parent(dev);

Skip:
        dev->power.is_late_suspended = true;

Complete:
        TRACE_SUSPEND(error);
        complete_all(&dev->power.completion);

        if (error || READ_ONCE(async_error))
                return;

        dpm_async_suspend_superior(dev, async_suspend_late);
}

static void async_suspend_late(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_suspend_late(dev, pm_transition, true);
        put_device(dev);
}

/**
 * dpm_suspend_late - Execute "late suspend" callbacks for all devices.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend_late(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        struct device *dev;
        int error;

        trace_suspend_resume(TPS("dpm_suspend_late"), state.event, true);

        pm_transition = state;
        async_error = 0;

        wake_up_all_idle_cpus();

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" leaf devices upfront so they don't need to
         * wait for the "sync" devices they don't depend on.
         */
        list_for_each_entry_reverse(dev, &dpm_suspended_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_leaf_device(dev))
                        dpm_async_with_cleanup(dev, async_suspend_late);
        }

        while (!list_empty(&dpm_suspended_list)) {
                dev = to_device(dpm_suspended_list.prev);

                list_move(&dev->power.entry, &dpm_late_early_list);

                if (dpm_async_fn(dev, async_suspend_late))
                        continue;

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                device_suspend_late(dev, state, false);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (READ_ONCE(async_error)) {
                        dpm_async_suspend_complete_all(&dpm_suspended_list);
                        /*
                         * Move all devices to the target list to resume them
                         * properly.
                         */
                        list_splice_init(&dpm_suspended_list, &dpm_late_early_list);
                        break;
                }
        }

        mutex_unlock(&dpm_list_mtx);

        async_synchronize_full();

        error = READ_ONCE(async_error);
        if (error) {
                dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
                dpm_resume_early(resume_event(state));
        }
        dpm_show_time(starttime, state, error, "late");
        trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
        return error;
}

/**
 * dpm_suspend_end - Execute "late" and "noirq" device suspend callbacks.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend_end(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error;

        error = dpm_suspend_late(state);
        if (error)
                goto out;

        error = dpm_suspend_noirq(state);
        if (error)
                dpm_resume_early(resume_event(state));

out:
        dpm_show_time(starttime, state, error, "end");
        return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_end);

/**
 * legacy_suspend - Execute a legacy (bus or class) suspend callback for device.
 * @dev: Device to suspend.
 * @state: PM transition of the system being carried out.
 * @cb: Suspend callback to execute.
 * @info: string description of caller.
 */
static int legacy_suspend(struct device *dev, pm_message_t state,
                          int (*cb)(struct device *dev, pm_message_t state),
                          const char *info)
{
        int error;
        ktime_t calltime;

        calltime = initcall_debug_start(dev, cb);

        trace_device_pm_callback_start(dev, info, state.event);
        error = cb(dev, state);
        trace_device_pm_callback_end(dev, error);
        suspend_report_result(dev, cb, error);

        initcall_debug_report(dev, calltime, cb, error);

        return error;
}

static void dpm_clear_superiors_direct_complete(struct device *dev)
{
        struct device_link *link;
        int idx;

        if (dev->parent) {
                spin_lock_irq(&dev->parent->power.lock);
                dev->parent->power.direct_complete = false;
                spin_unlock_irq(&dev->parent->power.lock);
        }

        idx = device_links_read_lock();

        dev_for_each_link_to_supplier(link, dev) {
                spin_lock_irq(&link->supplier->power.lock);
                link->supplier->power.direct_complete = false;
                spin_unlock_irq(&link->supplier->power.lock);
        }

        device_links_read_unlock(idx);
}

static void async_suspend(void *data, async_cookie_t cookie);

/**
 * device_suspend - Execute "suspend" callbacks for given device.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 * @async: If true, the device is being suspended asynchronously.
 */
static void device_suspend(struct device *dev, pm_message_t state, bool async)
{
        pm_callback_t callback = NULL;
        const char *info = NULL;
        int error = 0;
        DECLARE_DPM_WATCHDOG_ON_STACK(wd);

        TRACE_DEVICE(dev);
        TRACE_SUSPEND(0);

        dpm_wait_for_subordinate(dev, async);

        if (READ_ONCE(async_error)) {
                dev->power.direct_complete = false;
                goto Complete;
        }

        /*
         * Wait for possible runtime PM transitions of the device in progress
         * to complete and if there's a runtime resume request pending for it,
         * resume it before proceeding with invoking the system-wide suspend
         * callbacks for it.
         *
         * If the system-wide suspend callbacks below change the configuration
         * of the device, they must disable runtime PM for it or otherwise
         * ensure that its runtime-resume callbacks will not be confused by that
         * change in case they are invoked going forward.
         */
        pm_runtime_barrier(dev);

        if (pm_wakeup_pending()) {
                dev->power.direct_complete = false;
                WRITE_ONCE(async_error, -EBUSY);
                goto Complete;
        }

        if (dev->power.syscore)
                goto Complete;

        /* Avoid direct_complete to let wakeup_path propagate. */
        if (device_may_wakeup(dev) || device_wakeup_path(dev))
                dev->power.direct_complete = false;

        if (dev->power.direct_complete) {
                if (pm_runtime_status_suspended(dev)) {
                        pm_runtime_disable(dev);
                        if (pm_runtime_status_suspended(dev)) {
                                pm_dev_dbg(dev, state, "direct-complete ");
                                dev->power.is_suspended = true;
                                goto Complete;
                        }

                        pm_runtime_enable(dev);
                }
                dev->power.direct_complete = false;
        }

        dev->power.may_skip_resume = true;
        dev->power.must_resume = !dev_pm_test_driver_flags(dev, DPM_FLAG_MAY_SKIP_RESUME);

        dpm_watchdog_set(&wd, dev);
        device_lock(dev);

        if (dev->pm_domain) {
                info = "power domain ";
                callback = pm_op(&dev->pm_domain->ops, state);
                goto Run;
        }

        if (dev->type && dev->type->pm) {
                info = "type ";
                callback = pm_op(dev->type->pm, state);
                goto Run;
        }

        if (dev->class && dev->class->pm) {
                info = "class ";
                callback = pm_op(dev->class->pm, state);
                goto Run;
        }

        if (dev->bus) {
                if (dev->bus->pm) {
                        info = "bus ";
                        callback = pm_op(dev->bus->pm, state);
                } else if (dev->bus->suspend) {
                        pm_dev_dbg(dev, state, "legacy bus ");
                        error = legacy_suspend(dev, state, dev->bus->suspend,
                                                "legacy bus ");
                        goto End;
                }
        }

 Run:
        if (!callback && dev->driver && dev->driver->pm) {
                info = "driver ";
                callback = pm_op(dev->driver->pm, state);
        }

        error = dpm_run_callback(callback, dev, state, info);

 End:
        if (!error) {
                dev->power.is_suspended = true;
                if (device_may_wakeup(dev))
                        dev->power.wakeup_path = true;

                dpm_propagate_wakeup_to_parent(dev);
                dpm_clear_superiors_direct_complete(dev);
        }

        device_unlock(dev);
        dpm_watchdog_clear(&wd);

 Complete:
        if (error) {
                WRITE_ONCE(async_error, error);
                dpm_save_failed_dev(dev_name(dev));
                pm_dev_err(dev, state, async ? " async" : "", error);
        }

        complete_all(&dev->power.completion);
        TRACE_SUSPEND(error);

        if (error || READ_ONCE(async_error))
                return;

        dpm_async_suspend_superior(dev, async_suspend);
}

static void async_suspend(void *data, async_cookie_t cookie)
{
        struct device *dev = data;

        device_suspend(dev, pm_transition, true);
        put_device(dev);
}

/**
 * dpm_suspend - Execute "suspend" callbacks for all non-sysdev devices.
 * @state: PM transition of the system being carried out.
 */
int dpm_suspend(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        struct device *dev;
        int error;

        trace_suspend_resume(TPS("dpm_suspend"), state.event, true);
        might_sleep();

        devfreq_suspend();
        cpufreq_suspend();

        pm_transition = state;
        async_error = 0;

        mutex_lock(&dpm_list_mtx);

        /*
         * Start processing "async" leaf devices upfront so they don't need to
         * wait for the "sync" devices they don't depend on.
         */
        list_for_each_entry_reverse(dev, &dpm_prepared_list, power.entry) {
                dpm_clear_async_state(dev);
                if (dpm_leaf_device(dev))
                        dpm_async_with_cleanup(dev, async_suspend);
        }

        while (!list_empty(&dpm_prepared_list)) {
                dev = to_device(dpm_prepared_list.prev);

                list_move(&dev->power.entry, &dpm_suspended_list);

                if (dpm_async_fn(dev, async_suspend))
                        continue;

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                device_suspend(dev, state, false);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);

                if (READ_ONCE(async_error)) {
                        dpm_async_suspend_complete_all(&dpm_prepared_list);
                        /*
                         * Move all devices to the target list to resume them
                         * properly.
                         */
                        list_splice_init(&dpm_prepared_list, &dpm_suspended_list);
                        break;
                }
        }

        mutex_unlock(&dpm_list_mtx);

        async_synchronize_full();

        error = READ_ONCE(async_error);
        if (error)
                dpm_save_failed_step(SUSPEND_SUSPEND);

        dpm_show_time(starttime, state, error, NULL);
        trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
        return error;
}

static bool device_prepare_smart_suspend(struct device *dev)
{
        struct device_link *link;
        bool ret = true;
        int idx;

        /*
         * The "smart suspend" feature is enabled for devices whose drivers ask
         * for it and for devices without PM callbacks.
         *
         * However, if "smart suspend" is not enabled for the device's parent
         * or any of its suppliers that take runtime PM into account, it cannot
         * be enabled for the device either.
         */
        if (!dev->power.no_pm_callbacks &&
            !dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND))
                return false;

        if (dev->parent && !dev_pm_smart_suspend(dev->parent) &&
            !dev->parent->power.ignore_children && !pm_runtime_blocked(dev->parent))
                return false;

        idx = device_links_read_lock();

        dev_for_each_link_to_supplier(link, dev) {
                if (!device_link_test(link, DL_FLAG_PM_RUNTIME))
                        continue;

                if (!dev_pm_smart_suspend(link->supplier) &&
                    !pm_runtime_blocked(link->supplier)) {
                        ret = false;
                        break;
                }
        }

        device_links_read_unlock(idx);

        return ret;
}

/**
 * device_prepare - Prepare a device for system power transition.
 * @dev: Device to handle.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->prepare() callback(s) for given device.  No new children of the
 * device may be registered after this function has returned.
 */
static int device_prepare(struct device *dev, pm_message_t state)
{
        int (*callback)(struct device *) = NULL;
        bool smart_suspend;
        int ret = 0;

        /*
         * If a device's parent goes into runtime suspend at the wrong time,
         * it won't be possible to resume the device.  To prevent this we
         * block runtime suspend here, during the prepare phase, and allow
         * it again during the complete phase.
         */
        pm_runtime_get_noresume(dev);
        /*
         * If runtime PM is disabled for the device at this point and it has
         * never been enabled so far, it should not be enabled until this system
         * suspend-resume cycle is complete, so prepare to trigger a warning on
         * subsequent attempts to enable it.
         */
        smart_suspend = !pm_runtime_block_if_disabled(dev);

        if (dev->power.syscore)
                return 0;

        device_lock(dev);

        dev->power.wakeup_path = false;

        if (dev->power.no_pm_callbacks)
                goto unlock;

        if (dev->pm_domain)
                callback = dev->pm_domain->ops.prepare;
        else if (dev->type && dev->type->pm)
                callback = dev->type->pm->prepare;
        else if (dev->class && dev->class->pm)
                callback = dev->class->pm->prepare;
        else if (dev->bus && dev->bus->pm)
                callback = dev->bus->pm->prepare;

        if (!callback && dev->driver && dev->driver->pm)
                callback = dev->driver->pm->prepare;

        if (callback)
                ret = callback(dev);

unlock:
        device_unlock(dev);

        if (ret < 0) {
                suspend_report_result(dev, callback, ret);
                pm_runtime_put(dev);
                return ret;
        }
        /* Do not enable "smart suspend" for devices with disabled runtime PM. */
        if (smart_suspend)
                smart_suspend = device_prepare_smart_suspend(dev);

        spin_lock_irq(&dev->power.lock);

        dev->power.smart_suspend = smart_suspend;
        /*
         * A positive return value from ->prepare() means "this device appears
         * to be runtime-suspended and its state is fine, so if it really is
         * runtime-suspended, you can leave it in that state provided that you
         * will do the same thing with all of its descendants".  This only
         * applies to suspend transitions, however.
         */
        dev->power.direct_complete = state.event == PM_EVENT_SUSPEND &&
                (ret > 0 || dev->power.no_pm_callbacks) &&
                !dev_pm_test_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE);

        spin_unlock_irq(&dev->power.lock);

        return 0;
}

/**
 * dpm_prepare - Prepare all non-sysdev devices for a system PM transition.
 * @state: PM transition of the system being carried out.
 *
 * Execute the ->prepare() callback(s) for all devices.
 */
int dpm_prepare(pm_message_t state)
{
        int error = 0;

        trace_suspend_resume(TPS("dpm_prepare"), state.event, true);

        /*
         * Give a chance for the known devices to complete their probes, before
         * disable probing of devices. This sync point is important at least
         * at boot time + hibernation restore.
         */
        wait_for_device_probe();
        /*
         * It is unsafe if probing of devices will happen during suspend or
         * hibernation and system behavior will be unpredictable in this case.
         * So, let's prohibit device's probing here and defer their probes
         * instead. The normal behavior will be restored in dpm_complete().
         */
        device_block_probing();

        mutex_lock(&dpm_list_mtx);
        while (!list_empty(&dpm_list) && !error) {
                struct device *dev = to_device(dpm_list.next);

                get_device(dev);

                mutex_unlock(&dpm_list_mtx);

                trace_device_pm_callback_start(dev, "", state.event);
                error = device_prepare(dev, state);
                trace_device_pm_callback_end(dev, error);

                mutex_lock(&dpm_list_mtx);

                if (!error) {
                        dev->power.is_prepared = true;
                        if (!list_empty(&dev->power.entry))
                                list_move_tail(&dev->power.entry, &dpm_prepared_list);
                } else if (error == -EAGAIN) {
                        error = 0;
                } else {
                        dev_info(dev, "not prepared for power transition: code %d\n",
                                 error);
                }

                mutex_unlock(&dpm_list_mtx);

                put_device(dev);

                mutex_lock(&dpm_list_mtx);
        }
        mutex_unlock(&dpm_list_mtx);
        trace_suspend_resume(TPS("dpm_prepare"), state.event, false);
        return error;
}

/**
 * dpm_suspend_start - Prepare devices for PM transition and suspend them.
 * @state: PM transition of the system being carried out.
 *
 * Prepare all non-sysdev devices for system PM transition and execute "suspend"
 * callbacks for them.
 */
int dpm_suspend_start(pm_message_t state)
{
        ktime_t starttime = ktime_get();
        int error;

        error = dpm_prepare(state);
        if (error)
                dpm_save_failed_step(SUSPEND_PREPARE);
        else {
                pm_restrict_gfp_mask();
                error = dpm_suspend(state);
        }

        dpm_show_time(starttime, state, error, "start");
        return error;
}
EXPORT_SYMBOL_GPL(dpm_suspend_start);

void __suspend_report_result(const char *function, struct device *dev, void *fn, int ret)
{
        if (ret)
                dev_err(dev, "%s(): %ps returns %d\n", function, fn, ret);
}
EXPORT_SYMBOL_GPL(__suspend_report_result);

/**
 * device_pm_wait_for_dev - Wait for suspend/resume of a device to complete.
 * @subordinate: Device that needs to wait for @dev.
 * @dev: Device to wait for.
 */
int device_pm_wait_for_dev(struct device *subordinate, struct device *dev)
{
        dpm_wait(dev, subordinate->power.async_suspend);
        return async_error;
}
EXPORT_SYMBOL_GPL(device_pm_wait_for_dev);

/**
 * dpm_for_each_dev - device iterator.
 * @data: data for the callback.
 * @fn: function to be called for each device.
 *
 * Iterate over devices in dpm_list, and call @fn for each device,
 * passing it @data.
 */
void dpm_for_each_dev(void *data, void (*fn)(struct device *, void *))
{
        struct device *dev;

        if (!fn)
                return;

        device_pm_lock();
        list_for_each_entry(dev, &dpm_list, power.entry)
                fn(dev, data);
        device_pm_unlock();
}
EXPORT_SYMBOL_GPL(dpm_for_each_dev);

static bool pm_ops_is_empty(const struct dev_pm_ops *ops)
{
        if (!ops)
                return true;

        return !ops->prepare &&
               !ops->suspend &&
               !ops->suspend_late &&
               !ops->suspend_noirq &&
               !ops->resume_noirq &&
               !ops->resume_early &&
               !ops->resume &&
               !ops->complete;
}

void device_pm_check_callbacks(struct device *dev)
{
        unsigned long flags;

        spin_lock_irqsave(&dev->power.lock, flags);
        dev->power.no_pm_callbacks =
                (!dev->bus || (pm_ops_is_empty(dev->bus->pm) &&
                 !dev->bus->suspend && !dev->bus->resume)) &&
                (!dev->class || pm_ops_is_empty(dev->class->pm)) &&
                (!dev->type || pm_ops_is_empty(dev->type->pm)) &&
                (!dev->pm_domain || pm_ops_is_empty(&dev->pm_domain->ops)) &&
                (!dev->driver || (pm_ops_is_empty(dev->driver->pm) &&
                 !dev->driver->suspend && !dev->driver->resume));
        spin_unlock_irqrestore(&dev->power.lock, flags);
}

bool dev_pm_skip_suspend(struct device *dev)
{
        return dev_pm_smart_suspend(dev) && pm_runtime_status_suspended(dev);
}
























































































    4 
    1 


    2 
    4 
    1 
    5 

    3 





    5 





    3 
    5 

    5 
    5 






    3 






    5 
    4 





    4 


    4 
    4 
    4 














    2 
    1 





    2 

















    4 





    4 




























    2 
    2 


    1 
    2 



    2 



    2 




    5 



    5 



    4 






    4 

    4 

    4 






    1 










    3 
    2 





    1 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
// SPDX-License-Identifier: GPL-2.0-only
/*
 * H-TCP congestion control. The algorithm is detailed in:
 * R.N.Shorten, D.J.Leith:
 *   "H-TCP: TCP for high-speed and long-distance networks"
 *   Proc. PFLDnet, Argonne, 2004.
 * https://www.hamilton.ie/net/htcp3.pdf
 */

#include <linux/mm.h>
#include <linux/module.h>
#include <net/tcp.h>

#define ALPHA_BASE        (1<<7)        /* 1.0 with shift << 7 */
#define BETA_MIN        (1<<6)        /* 0.5 with shift << 7 */
#define BETA_MAX        102        /* 0.8 with shift << 7 */

static int use_rtt_scaling __read_mostly = 1;
module_param(use_rtt_scaling, int, 0644);
MODULE_PARM_DESC(use_rtt_scaling, "turn on/off RTT scaling");

static int use_bandwidth_switch __read_mostly = 1;
module_param(use_bandwidth_switch, int, 0644);
MODULE_PARM_DESC(use_bandwidth_switch, "turn on/off bandwidth switcher");

struct htcp {
        u32        alpha;                /* Fixed point arith, << 7 */
        u8        beta;           /* Fixed point arith, << 7 */
        u8        modeswitch;        /* Delay modeswitch
                                   until we had at least one congestion event */
        u16        pkts_acked;
        u32        packetcount;
        u32        minRTT;
        u32        maxRTT;
        u32        last_cong;        /* Time since last congestion event end */
        u32        undo_last_cong;

        u32        undo_maxRTT;
        u32        undo_old_maxB;

        /* Bandwidth estimation */
        u32        minB;
        u32        maxB;
        u32        old_maxB;
        u32        Bi;
        u32        lasttime;
};

static inline u32 htcp_cong_time(const struct htcp *ca)
{
        return jiffies - ca->last_cong;
}

static inline u32 htcp_ccount(const struct htcp *ca)
{
        return htcp_cong_time(ca) / ca->minRTT;
}

static inline void htcp_reset(struct htcp *ca)
{
        ca->undo_last_cong = ca->last_cong;
        ca->undo_maxRTT = ca->maxRTT;
        ca->undo_old_maxB = ca->old_maxB;

        ca->last_cong = jiffies;
}

static u32 htcp_cwnd_undo(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);

        if (ca->undo_last_cong) {
                ca->last_cong = ca->undo_last_cong;
                ca->maxRTT = ca->undo_maxRTT;
                ca->old_maxB = ca->undo_old_maxB;
                ca->undo_last_cong = 0;
        }

        return tcp_reno_undo_cwnd(sk);
}

static inline void measure_rtt(struct sock *sk, u32 srtt)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        struct htcp *ca = inet_csk_ca(sk);

        /* keep track of minimum RTT seen so far, minRTT is zero at first */
        if (ca->minRTT > srtt || !ca->minRTT)
                ca->minRTT = srtt;

        /* max RTT */
        if (icsk->icsk_ca_state == TCP_CA_Open) {
                if (ca->maxRTT < ca->minRTT)
                        ca->maxRTT = ca->minRTT;
                if (ca->maxRTT < srtt &&
                    srtt <= ca->maxRTT + msecs_to_jiffies(20))
                        ca->maxRTT = srtt;
        }
}

static void measure_achieved_throughput(struct sock *sk,
                                        const struct ack_sample *sample)
{
        const struct inet_connection_sock *icsk = inet_csk(sk);
        const struct tcp_sock *tp = tcp_sk(sk);
        struct htcp *ca = inet_csk_ca(sk);
        u32 now = tcp_jiffies32;

        if (icsk->icsk_ca_state == TCP_CA_Open)
                ca->pkts_acked = sample->pkts_acked;

        if (sample->rtt_us > 0)
                measure_rtt(sk, usecs_to_jiffies(sample->rtt_us));

        if (!use_bandwidth_switch)
                return;

        /* achieved throughput calculations */
        if (!((1 << icsk->icsk_ca_state) & (TCPF_CA_Open | TCPF_CA_Disorder))) {
                ca->packetcount = 0;
                ca->lasttime = now;
                return;
        }

        ca->packetcount += sample->pkts_acked;

        if (ca->packetcount >= tcp_snd_cwnd(tp) - (ca->alpha >> 7 ? : 1) &&
            now - ca->lasttime >= ca->minRTT &&
            ca->minRTT > 0) {
                __u32 cur_Bi = ca->packetcount * HZ / (now - ca->lasttime);

                if (htcp_ccount(ca) <= 3) {
                        /* just after backoff */
                        ca->minB = ca->maxB = ca->Bi = cur_Bi;
                } else {
                        ca->Bi = (3 * ca->Bi + cur_Bi) / 4;
                        if (ca->Bi > ca->maxB)
                                ca->maxB = ca->Bi;
                        if (ca->minB > ca->maxB)
                                ca->minB = ca->maxB;
                }
                ca->packetcount = 0;
                ca->lasttime = now;
        }
}

static inline void htcp_beta_update(struct htcp *ca, u32 minRTT, u32 maxRTT)
{
        if (use_bandwidth_switch) {
                u32 maxB = ca->maxB;
                u32 old_maxB = ca->old_maxB;

                ca->old_maxB = ca->maxB;
                if (!between(5 * maxB, 4 * old_maxB, 6 * old_maxB)) {
                        ca->beta = BETA_MIN;
                        ca->modeswitch = 0;
                        return;
                }
        }

        if (ca->modeswitch && minRTT > msecs_to_jiffies(10) && maxRTT) {
                ca->beta = (minRTT << 7) / maxRTT;
                if (ca->beta < BETA_MIN)
                        ca->beta = BETA_MIN;
                else if (ca->beta > BETA_MAX)
                        ca->beta = BETA_MAX;
        } else {
                ca->beta = BETA_MIN;
                ca->modeswitch = 1;
        }
}

static inline void htcp_alpha_update(struct htcp *ca)
{
        u32 minRTT = ca->minRTT;
        u32 factor = 1;
        u32 diff = htcp_cong_time(ca);

        if (diff > HZ) {
                diff -= HZ;
                factor = 1 + (10 * diff + ((diff / 2) * (diff / 2) / HZ)) / HZ;
        }

        if (use_rtt_scaling && minRTT) {
                u32 scale = (HZ << 3) / (10 * minRTT);

                /* clamping ratio to interval [0.5,10]<<3 */
                scale = clamp(scale, 1U << 2, 10U << 3);
                factor = (factor << 3) / scale;
                if (!factor)
                        factor = 1;
        }

        ca->alpha = 2 * factor * ((1 << 7) - ca->beta);
        if (!ca->alpha)
                ca->alpha = ALPHA_BASE;
}

/*
 * After we have the rtt data to calculate beta, we'd still prefer to wait one
 * rtt before we adjust our beta to ensure we are working from a consistent
 * data.
 *
 * This function should be called when we hit a congestion event since only at
 * that point do we really have a real sense of maxRTT (the queues en route
 * were getting just too full now).
 */
static void htcp_param_update(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);
        u32 minRTT = ca->minRTT;
        u32 maxRTT = ca->maxRTT;

        htcp_beta_update(ca, minRTT, maxRTT);
        htcp_alpha_update(ca);

        /* add slowly fading memory for maxRTT to accommodate routing changes */
        if (minRTT > 0 && maxRTT > minRTT)
                ca->maxRTT = minRTT + ((maxRTT - minRTT) * 95) / 100;
}

static u32 htcp_recalc_ssthresh(struct sock *sk)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        const struct htcp *ca = inet_csk_ca(sk);

        htcp_param_update(sk);
        return max((tcp_snd_cwnd(tp) * ca->beta) >> 7, 2U);
}

static void htcp_cong_avoid(struct sock *sk, u32 ack, u32 acked)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct htcp *ca = inet_csk_ca(sk);

        if (!tcp_is_cwnd_limited(sk))
                return;

        if (tcp_in_slow_start(tp))
                tcp_slow_start(tp, acked);
        else {
                /* In dangerous area, increase slowly.
                 * In theory this is tp->snd_cwnd += alpha / tp->snd_cwnd
                 */
                if ((tp->snd_cwnd_cnt * ca->alpha)>>7 >= tcp_snd_cwnd(tp)) {
                        if (tcp_snd_cwnd(tp) < tp->snd_cwnd_clamp)
                                tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) + 1);
                        tp->snd_cwnd_cnt = 0;
                        htcp_alpha_update(ca);
                } else
                        tp->snd_cwnd_cnt += ca->pkts_acked;

                ca->pkts_acked = 1;
        }
}

static void htcp_init(struct sock *sk)
{
        struct htcp *ca = inet_csk_ca(sk);

        memset(ca, 0, sizeof(struct htcp));
        ca->alpha = ALPHA_BASE;
        ca->beta = BETA_MIN;
        ca->pkts_acked = 1;
        ca->last_cong = jiffies;
}

static void htcp_state(struct sock *sk, u8 new_state)
{
        switch (new_state) {
        case TCP_CA_Open:
                {
                        struct htcp *ca = inet_csk_ca(sk);

                        if (ca->undo_last_cong) {
                                ca->last_cong = jiffies;
                                ca->undo_last_cong = 0;
                        }
                }
                break;
        case TCP_CA_CWR:
        case TCP_CA_Recovery:
        case TCP_CA_Loss:
                htcp_reset(inet_csk_ca(sk));
                break;
        }
}

static struct tcp_congestion_ops htcp __read_mostly = {
        .init                = htcp_init,
        .ssthresh        = htcp_recalc_ssthresh,
        .cong_avoid        = htcp_cong_avoid,
        .set_state        = htcp_state,
        .undo_cwnd        = htcp_cwnd_undo,
        .pkts_acked        = measure_achieved_throughput,
        .owner                = THIS_MODULE,
        .name                = "htcp",
};

static int __init htcp_register(void)
{
        BUILD_BUG_ON(sizeof(struct htcp) > ICSK_CA_PRIV_SIZE);
        BUILD_BUG_ON(BETA_MIN >= BETA_MAX);
        return tcp_register_congestion_control(&htcp);
}

static void __exit htcp_unregister(void)
{
        tcp_unregister_congestion_control(&htcp);
}

module_init(htcp_register);
module_exit(htcp_unregister);

MODULE_AUTHOR("Baruch Even");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("H-TCP");

































































































































































































































    3 













    1 




    1 




















































































































































































































































































































































































































    8 



    2 
    2 



    4 



























































































































































































































































































    7 








    1 



    2 
    4 
    4 





    2 

    2 



    1 


    3 


    3 




    1 


    1 






























































   16 




    1 
    2 




    1 










    7 




    1 




    1 





    3 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/if_vlan.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_offload.h>
/* For layer 4 checksum field offset. */
#include <linux/tcp.h>
#include <linux/udp.h>
#include <net/gre.h>
#include <linux/icmpv6.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <net/sctp/checksum.h>

static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off,
                                         struct vlan_ethhdr *veth)
{
        if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN))
                return false;

        veth->h_vlan_proto = skb->vlan_proto;
        veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
        veth->h_vlan_encapsulated_proto = skb->protocol;

        return true;
}

/* add vlan header into the user buffer for if tag was removed by offloads */
static bool
nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u16 offset, u8 len)
{
        int mac_off = skb_mac_header(skb) - skb->data;
        u8 *vlanh, *dst_u8 = (u8 *) d;
        struct vlan_ethhdr veth;

        vlanh = (u8 *) &veth;
        if (offset < VLAN_ETH_HLEN) {
                u8 ethlen = len;

                if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth))
                        return false;

                if (offset + len > VLAN_ETH_HLEN)
                        ethlen -= offset + len - VLAN_ETH_HLEN;

                memcpy(dst_u8, vlanh + offset, ethlen);

                len -= ethlen;
                if (len == 0)
                        return true;

                dst_u8 += ethlen;
                offset = ETH_HLEN;
        } else {
                offset -= VLAN_HLEN;
        }

        return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
}

static int __nft_payload_inner_offset(struct nft_pktinfo *pkt)
{
        unsigned int thoff = nft_thoff(pkt);

        if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
                return -1;

        switch (pkt->tprot) {
        case IPPROTO_UDP:
                pkt->inneroff = thoff + sizeof(struct udphdr);
                break;
        case IPPROTO_TCP: {
                struct tcphdr *th, _tcph;

                th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph);
                if (!th)
                        return -1;

                pkt->inneroff = thoff + __tcp_hdrlen(th);
                }
                break;
        case IPPROTO_GRE: {
                u32 offset = sizeof(struct gre_base_hdr);
                struct gre_base_hdr *gre, _gre;
                __be16 version;

                gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre);
                if (!gre)
                        return -1;

                version = gre->flags & GRE_VERSION;
                switch (version) {
                case GRE_VERSION_0:
                        if (gre->flags & GRE_ROUTING)
                                return -1;

                        if (gre->flags & GRE_CSUM) {
                                offset += sizeof_field(struct gre_full_hdr, csum) +
                                          sizeof_field(struct gre_full_hdr, reserved1);
                        }
                        if (gre->flags & GRE_KEY)
                                offset += sizeof_field(struct gre_full_hdr, key);

                        if (gre->flags & GRE_SEQ)
                                offset += sizeof_field(struct gre_full_hdr, seq);
                        break;
                default:
                        return -1;
                }

                pkt->inneroff = thoff + offset;
                }
                break;
        case IPPROTO_IPIP:
                pkt->inneroff = thoff;
                break;
        default:
                return -1;
        }

        pkt->flags |= NFT_PKTINFO_INNER;

        return 0;
}

int nft_payload_inner_offset(const struct nft_pktinfo *pkt)
{
        if (!(pkt->flags & NFT_PKTINFO_INNER) &&
            __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0)
                return -1;

        return pkt->inneroff;
}

static bool nft_payload_need_vlan_adjust(u32 offset, u32 len)
{
        unsigned int boundary = offset + len;

        /* data past ether src/dst requested, copy needed */
        if (boundary > offsetof(struct ethhdr, h_proto))
                return true;

        return false;
}

void nft_payload_eval(const struct nft_expr *expr,
                      struct nft_regs *regs,
                      const struct nft_pktinfo *pkt)
{
        const struct nft_payload *priv = nft_expr_priv(expr);
        const struct sk_buff *skb = pkt->skb;
        u32 *dest = &regs->data[priv->dreg];
        int offset;

        if (priv->len % NFT_REG32_SIZE)
                dest[priv->len / NFT_REG32_SIZE] = 0;

        switch (priv->base) {
        case NFT_PAYLOAD_LL_HEADER:
                if (!skb_mac_header_was_set(skb) || skb_mac_header_len(skb) == 0)
                        goto err;

                if (skb_vlan_tag_present(skb) &&
                    nft_payload_need_vlan_adjust(priv->offset, priv->len)) {
                        if (!nft_payload_copy_vlan(dest, skb,
                                                   priv->offset, priv->len))
                                goto err;
                        return;
                }
                offset = skb_mac_header(skb) - skb->data;
                break;
        case NFT_PAYLOAD_NETWORK_HEADER:
                offset = skb_network_offset(skb);
                break;
        case NFT_PAYLOAD_TRANSPORT_HEADER:
                if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
                        goto err;
                offset = nft_thoff(pkt);
                break;
        case NFT_PAYLOAD_INNER_HEADER:
                offset = nft_payload_inner_offset(pkt);
                if (offset < 0)
                        goto err;
                break;
        default:
                WARN_ON_ONCE(1);
                goto err;
        }
        offset += priv->offset;

        if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
                goto err;
        return;
err:
        regs->verdict.code = NFT_BREAK;
}

static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
        [NFTA_PAYLOAD_SREG]                = { .type = NLA_U32 },
        [NFTA_PAYLOAD_DREG]                = { .type = NLA_U32 },
        [NFTA_PAYLOAD_BASE]                = { .type = NLA_U32 },
        [NFTA_PAYLOAD_OFFSET]                = { .type = NLA_BE32 },
        [NFTA_PAYLOAD_LEN]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_PAYLOAD_CSUM_TYPE]        = { .type = NLA_U32 },
        [NFTA_PAYLOAD_CSUM_OFFSET]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_PAYLOAD_CSUM_FLAGS]        = { .type = NLA_U32 },
};

static int nft_payload_init(const struct nft_ctx *ctx,
                            const struct nft_expr *expr,
                            const struct nlattr * const tb[])
{
        struct nft_payload *priv = nft_expr_priv(expr);

        priv->base   = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
        priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
        priv->len    = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));

        return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
                                        &priv->dreg, NULL, NFT_DATA_VALUE,
                                        priv->len);
}

static int nft_payload_dump(struct sk_buff *skb,
                            const struct nft_expr *expr, bool reset)
{
        const struct nft_payload *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
            nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static bool nft_payload_reduce(struct nft_regs_track *track,
                               const struct nft_expr *expr)
{
        const struct nft_payload *priv = nft_expr_priv(expr);
        const struct nft_payload *payload;

        if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        payload = nft_expr_priv(track->regs[priv->dreg].selector);
        if (priv->base != payload->base ||
            priv->offset != payload->offset ||
            priv->len != payload->len) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        if (!track->regs[priv->dreg].bitwise)
                return true;

        return nft_expr_reduce_bitwise(track, expr);
}

static bool nft_payload_offload_mask(struct nft_offload_reg *reg,
                                     u32 priv_len, u32 field_len)
{
        unsigned int remainder, delta, k;
        struct nft_data mask = {};
        __be32 remainder_mask;

        if (priv_len == field_len) {
                memset(&reg->mask, 0xff, priv_len);
                return true;
        } else if (priv_len > field_len) {
                return false;
        }

        memset(&mask, 0xff, field_len);
        remainder = priv_len % sizeof(u32);
        if (remainder) {
                k = priv_len / sizeof(u32);
                delta = field_len - priv_len;
                remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1));
                mask.data[k] = (__force u32)remainder_mask;
        }

        memcpy(&reg->mask, &mask, field_len);

        return true;
}

static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
                                  struct nft_flow_rule *flow,
                                  const struct nft_payload *priv)
{
        struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

        switch (priv->offset) {
        case offsetof(struct ethhdr, h_source):
                if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
                                  src, ETH_ALEN, reg);
                break;
        case offsetof(struct ethhdr, h_dest):
                if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
                                  dst, ETH_ALEN, reg);
                break;
        case offsetof(struct ethhdr, h_proto):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic,
                                  n_proto, sizeof(__be16), reg);
                nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
                break;
        case offsetof(struct vlan_ethhdr, h_vlan_TCI):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan,
                                        vlan_tci, sizeof(__be16), reg,
                                        NFT_OFFLOAD_F_NETWORK2HOST);
                break;
        case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan,
                                  vlan_tpid, sizeof(__be16), reg);
                nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
                break;
        case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
                                        vlan_tci, sizeof(__be16), reg,
                                        NFT_OFFLOAD_F_NETWORK2HOST);
                break;
        case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) +
                                                        sizeof(struct vlan_hdr):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
                                  vlan_tpid, sizeof(__be16), reg);
                nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
                                  struct nft_flow_rule *flow,
                                  const struct nft_payload *priv)
{
        struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

        switch (priv->offset) {
        case offsetof(struct iphdr, saddr):
                if (!nft_payload_offload_mask(reg, priv->len,
                                              sizeof(struct in_addr)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
                                  sizeof(struct in_addr), reg);
                nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
                break;
        case offsetof(struct iphdr, daddr):
                if (!nft_payload_offload_mask(reg, priv->len,
                                              sizeof(struct in_addr)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
                                  sizeof(struct in_addr), reg);
                nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
                break;
        case offsetof(struct iphdr, protocol):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
                                  sizeof(__u8), reg);
                nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
                                  struct nft_flow_rule *flow,
                                  const struct nft_payload *priv)
{
        struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

        switch (priv->offset) {
        case offsetof(struct ipv6hdr, saddr):
                if (!nft_payload_offload_mask(reg, priv->len,
                                              sizeof(struct in6_addr)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
                                  sizeof(struct in6_addr), reg);
                nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
                break;
        case offsetof(struct ipv6hdr, daddr):
                if (!nft_payload_offload_mask(reg, priv->len,
                                              sizeof(struct in6_addr)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
                                  sizeof(struct in6_addr), reg);
                nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
                break;
        case offsetof(struct ipv6hdr, nexthdr):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
                                  sizeof(__u8), reg);
                nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
                                  struct nft_flow_rule *flow,
                                  const struct nft_payload *priv)
{
        int err;

        switch (ctx->dep.l3num) {
        case htons(ETH_P_IP):
                err = nft_payload_offload_ip(ctx, flow, priv);
                break;
        case htons(ETH_P_IPV6):
                err = nft_payload_offload_ip6(ctx, flow, priv);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return err;
}

static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
                                   struct nft_flow_rule *flow,
                                   const struct nft_payload *priv)
{
        struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

        switch (priv->offset) {
        case offsetof(struct tcphdr, source):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
                                  sizeof(__be16), reg);
                break;
        case offsetof(struct tcphdr, dest):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
                                  sizeof(__be16), reg);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
                                   struct nft_flow_rule *flow,
                                   const struct nft_payload *priv)
{
        struct nft_offload_reg *reg = &ctx->regs[priv->dreg];

        switch (priv->offset) {
        case offsetof(struct udphdr, source):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
                                  sizeof(__be16), reg);
                break;
        case offsetof(struct udphdr, dest):
                if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
                        return -EOPNOTSUPP;

                NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
                                  sizeof(__be16), reg);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
                                  struct nft_flow_rule *flow,
                                  const struct nft_payload *priv)
{
        int err;

        switch (ctx->dep.protonum) {
        case IPPROTO_TCP:
                err = nft_payload_offload_tcp(ctx, flow, priv);
                break;
        case IPPROTO_UDP:
                err = nft_payload_offload_udp(ctx, flow, priv);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return err;
}

static int nft_payload_offload(struct nft_offload_ctx *ctx,
                               struct nft_flow_rule *flow,
                               const struct nft_expr *expr)
{
        const struct nft_payload *priv = nft_expr_priv(expr);
        int err;

        switch (priv->base) {
        case NFT_PAYLOAD_LL_HEADER:
                err = nft_payload_offload_ll(ctx, flow, priv);
                break;
        case NFT_PAYLOAD_NETWORK_HEADER:
                err = nft_payload_offload_nh(ctx, flow, priv);
                break;
        case NFT_PAYLOAD_TRANSPORT_HEADER:
                err = nft_payload_offload_th(ctx, flow, priv);
                break;
        default:
                err = -EOPNOTSUPP;
                break;
        }
        return err;
}

static const struct nft_expr_ops nft_payload_ops = {
        .type                = &nft_payload_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
        .eval                = nft_payload_eval,
        .init                = nft_payload_init,
        .dump                = nft_payload_dump,
        .reduce                = nft_payload_reduce,
        .offload        = nft_payload_offload,
};

const struct nft_expr_ops nft_payload_fast_ops = {
        .type                = &nft_payload_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
        .eval                = nft_payload_eval,
        .init                = nft_payload_init,
        .dump                = nft_payload_dump,
        .reduce                = nft_payload_reduce,
        .offload        = nft_payload_offload,
};

void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs,
                            const struct nft_pktinfo *pkt,
                            struct nft_inner_tun_ctx *tun_ctx)
{
        const struct nft_payload *priv = nft_expr_priv(expr);
        const struct sk_buff *skb = pkt->skb;
        u32 *dest = &regs->data[priv->dreg];
        int offset;

        if (priv->len % NFT_REG32_SIZE)
                dest[priv->len / NFT_REG32_SIZE] = 0;

        switch (priv->base) {
        case NFT_PAYLOAD_TUN_HEADER:
                if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN))
                        goto err;

                offset = tun_ctx->inner_tunoff;
                break;
        case NFT_PAYLOAD_LL_HEADER:
                if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL))
                        goto err;

                offset = tun_ctx->inner_lloff;
                break;
        case NFT_PAYLOAD_NETWORK_HEADER:
                if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH))
                        goto err;

                offset = tun_ctx->inner_nhoff;
                break;
        case NFT_PAYLOAD_TRANSPORT_HEADER:
                if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH))
                        goto err;

                offset = tun_ctx->inner_thoff;
                break;
        default:
                WARN_ON_ONCE(1);
                goto err;
        }
        offset += priv->offset;

        if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
                goto err;

        return;
err:
        regs->verdict.code = NFT_BREAK;
}

static int nft_payload_inner_init(const struct nft_ctx *ctx,
                                  const struct nft_expr *expr,
                                  const struct nlattr * const tb[])
{
        struct nft_payload *priv = nft_expr_priv(expr);
        u32 base;

        if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] ||
            !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG])
                return -EINVAL;

        base   = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
        switch (base) {
        case NFT_PAYLOAD_TUN_HEADER:
        case NFT_PAYLOAD_LL_HEADER:
        case NFT_PAYLOAD_NETWORK_HEADER:
        case NFT_PAYLOAD_TRANSPORT_HEADER:
                break;
        default:
                return -EOPNOTSUPP;
        }

        priv->base   = base;
        priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
        priv->len    = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));

        return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
                                        &priv->dreg, NULL, NFT_DATA_VALUE,
                                        priv->len);
}

static const struct nft_expr_ops nft_payload_inner_ops = {
        .type                = &nft_payload_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
        .init                = nft_payload_inner_init,
        .dump                = nft_payload_dump,
        /* direct call to nft_payload_inner_eval(). */
};

static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
{
        csum_replace4(sum, (__force __be32)fsum, (__force __be32)tsum);
        if (*sum == 0)
                *sum = CSUM_MANGLED_0;
}

static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
{
        struct udphdr *uh, _uh;

        uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
        if (!uh)
                return false;

        return (__force bool)uh->check;
}

static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
                                     struct sk_buff *skb,
                                     unsigned int *l4csum_offset)
{
        if (pkt->fragoff)
                return -1;

        switch (pkt->tprot) {
        case IPPROTO_TCP:
                *l4csum_offset = offsetof(struct tcphdr, check);
                break;
        case IPPROTO_UDP:
                if (!nft_payload_udp_checksum(skb, nft_thoff(pkt)))
                        return -1;
                fallthrough;
        case IPPROTO_UDPLITE:
                *l4csum_offset = offsetof(struct udphdr, check);
                break;
        case IPPROTO_ICMPV6:
                *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
                break;
        default:
                return -1;
        }

        *l4csum_offset += nft_thoff(pkt);
        return 0;
}

static int nft_payload_csum_sctp(struct sk_buff *skb, int offset)
{
        struct sctphdr *sh;

        if (skb_ensure_writable(skb, offset + sizeof(*sh)))
                return -1;

        sh = (struct sctphdr *)(skb->data + offset);
        sh->checksum = sctp_compute_cksum(skb, offset);
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        return 0;
}

static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
                                     struct sk_buff *skb,
                                     __wsum fsum, __wsum tsum)
{
        int l4csum_offset;
        __sum16 sum;

        /* If we cannot determine layer 4 checksum offset or this packet doesn't
         * require layer 4 checksum recalculation, skip this packet.
         */
        if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
                return 0;

        if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
                return -1;

        /* Checksum mangling for an arbitrary amount of bytes, based on
         * inet_proto_csum_replace*() functions.
         */
        if (skb->ip_summed != CHECKSUM_PARTIAL) {
                nft_csum_replace(&sum, fsum, tsum);
                if (skb->ip_summed == CHECKSUM_COMPLETE) {
                        skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
                                              tsum);
                }
        } else {
                sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
                                          tsum));
        }

        if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
            skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
                return -1;

        return 0;
}

static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
                                 __wsum fsum, __wsum tsum, int csum_offset)
{
        __sum16 sum;

        if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
                return -1;

        nft_csum_replace(&sum, fsum, tsum);
        if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
            skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
                return -1;

        return 0;
}

struct nft_payload_set {
        enum nft_payload_bases        base:8;
        u16                        offset;
        u8                        len;
        u8                        sreg;
        u8                        csum_type;
        u8                        csum_offset;
        u8                        csum_flags;
};

/* This is not struct vlan_hdr. */
struct nft_payload_vlan_hdr {
        __be16                        h_vlan_proto;
        __be16                        h_vlan_TCI;
};

static bool
nft_payload_set_vlan(const u32 *src, struct sk_buff *skb, u16 offset, u8 len,
                     int *vlan_hlen)
{
        struct nft_payload_vlan_hdr *vlanh;
        __be16 vlan_proto;
        u16 vlan_tci;

        if (offset >= offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto)) {
                *vlan_hlen = VLAN_HLEN;
                return true;
        }

        switch (offset) {
        case offsetof(struct vlan_ethhdr, h_vlan_proto):
                if (len == 2) {
                        vlan_proto = nft_reg_load_be16(src);
                        skb->vlan_proto = vlan_proto;
                } else if (len == 4) {
                        vlanh = (struct nft_payload_vlan_hdr *)src;
                        __vlan_hwaccel_put_tag(skb, vlanh->h_vlan_proto,
                                               ntohs(vlanh->h_vlan_TCI));
                } else {
                        return false;
                }
                break;
        case offsetof(struct vlan_ethhdr, h_vlan_TCI):
                if (len != 2)
                        return false;

                vlan_tci = ntohs(nft_reg_load_be16(src));
                skb->vlan_tci = vlan_tci;
                break;
        default:
                return false;
        }

        return true;
}

static void nft_payload_set_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        const struct nft_payload_set *priv = nft_expr_priv(expr);
        const u32 *src = &regs->data[priv->sreg];
        int offset, csum_offset, vlan_hlen = 0;
        struct sk_buff *skb = pkt->skb;
        __wsum fsum, tsum;

        switch (priv->base) {
        case NFT_PAYLOAD_LL_HEADER:
                if (!skb_mac_header_was_set(skb))
                        goto err;

                if (skb_vlan_tag_present(skb) &&
                    nft_payload_need_vlan_adjust(priv->offset, priv->len)) {
                        if (!nft_payload_set_vlan(src, skb,
                                                  priv->offset, priv->len,
                                                  &vlan_hlen))
                                goto err;

                        if (!vlan_hlen)
                                return;
                }

                offset = skb_mac_header(skb) - skb->data - vlan_hlen;
                break;
        case NFT_PAYLOAD_NETWORK_HEADER:
                offset = skb_network_offset(skb);
                break;
        case NFT_PAYLOAD_TRANSPORT_HEADER:
                if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
                        goto err;
                offset = nft_thoff(pkt);
                break;
        case NFT_PAYLOAD_INNER_HEADER:
                offset = nft_payload_inner_offset(pkt);
                if (offset < 0)
                        goto err;
                break;
        default:
                WARN_ON_ONCE(1);
                goto err;
        }

        csum_offset = offset + priv->csum_offset;
        offset += priv->offset;

        if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
            ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER &&
              priv->base != NFT_PAYLOAD_INNER_HEADER) ||
             skb->ip_summed != CHECKSUM_PARTIAL)) {
                if (offset + priv->len > skb->len)
                        goto err;

                fsum = skb_checksum(skb, offset, priv->len, 0);
                tsum = csum_partial(src, priv->len, 0);

                if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
                    nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
                        goto err;

                if (priv->csum_flags &&
                    nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
                        goto err;
        }

        if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
            skb_store_bits(skb, offset, src, priv->len) < 0)
                goto err;

        if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP &&
            pkt->tprot == IPPROTO_SCTP &&
            skb->ip_summed != CHECKSUM_PARTIAL) {
                if (pkt->fragoff == 0 &&
                    nft_payload_csum_sctp(skb, nft_thoff(pkt)))
                        goto err;
        }

        return;
err:
        regs->verdict.code = NFT_BREAK;
}

static int nft_payload_set_init(const struct nft_ctx *ctx,
                                const struct nft_expr *expr,
                                const struct nlattr * const tb[])
{
        u32 csum_offset, offset, csum_type = NFT_PAYLOAD_CSUM_NONE;
        struct nft_payload_set *priv = nft_expr_priv(expr);
        int err;

        priv->base        = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
        priv->len         = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));

        err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset);
        if (err < 0)
                return err;
        priv->offset = offset;

        if (tb[NFTA_PAYLOAD_CSUM_TYPE])
                csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
        if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) {
                err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX,
                                          &csum_offset);
                if (err < 0)
                        return err;

                priv->csum_offset = csum_offset;
        }
        if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
                u32 flags;

                flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
                if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
                        return -EINVAL;

                priv->csum_flags = flags;
        }

        switch (csum_type) {
        case NFT_PAYLOAD_CSUM_NONE:
        case NFT_PAYLOAD_CSUM_INET:
                break;
        case NFT_PAYLOAD_CSUM_SCTP:
                if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER)
                        return -EINVAL;

                if (priv->csum_offset != offsetof(struct sctphdr, checksum))
                        return -EINVAL;
                break;
        default:
                return -EOPNOTSUPP;
        }
        priv->csum_type = csum_type;

        return nft_parse_register_load(ctx, tb[NFTA_PAYLOAD_SREG], &priv->sreg,
                                       priv->len);
}

static int nft_payload_set_dump(struct sk_buff *skb,
                                const struct nft_expr *expr, bool reset)
{
        const struct nft_payload_set *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
            nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
                         htonl(priv->csum_offset)) ||
            nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static bool nft_payload_set_reduce(struct nft_regs_track *track,
                                   const struct nft_expr *expr)
{
        int i;

        for (i = 0; i < NFT_REG32_NUM; i++) {
                if (!track->regs[i].selector)
                        continue;

                if (track->regs[i].selector->ops != &nft_payload_ops &&
                    track->regs[i].selector->ops != &nft_payload_fast_ops)
                        continue;

                __nft_reg_track_cancel(track, i);
        }

        return false;
}

static const struct nft_expr_ops nft_payload_set_ops = {
        .type                = &nft_payload_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
        .eval                = nft_payload_set_eval,
        .init                = nft_payload_set_init,
        .dump                = nft_payload_set_dump,
        .reduce                = nft_payload_set_reduce,
};

static const struct nft_expr_ops *
nft_payload_select_ops(const struct nft_ctx *ctx,
                       const struct nlattr * const tb[])
{
        enum nft_payload_bases base;
        unsigned int offset, len;
        int err;

        if (tb[NFTA_PAYLOAD_BASE] == NULL ||
            tb[NFTA_PAYLOAD_OFFSET] == NULL ||
            tb[NFTA_PAYLOAD_LEN] == NULL)
                return ERR_PTR(-EINVAL);

        base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
        switch (base) {
        case NFT_PAYLOAD_LL_HEADER:
        case NFT_PAYLOAD_NETWORK_HEADER:
        case NFT_PAYLOAD_TRANSPORT_HEADER:
        case NFT_PAYLOAD_INNER_HEADER:
                break;
        default:
                return ERR_PTR(-EOPNOTSUPP);
        }

        if (tb[NFTA_PAYLOAD_SREG] != NULL) {
                if (tb[NFTA_PAYLOAD_DREG] != NULL)
                        return ERR_PTR(-EINVAL);
                return &nft_payload_set_ops;
        }

        if (tb[NFTA_PAYLOAD_DREG] == NULL)
                return ERR_PTR(-EINVAL);

        err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset);
        if (err < 0)
                return ERR_PTR(err);

        err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len);
        if (err < 0)
                return ERR_PTR(err);

        if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
            base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER)
                return &nft_payload_fast_ops;
        else
                return &nft_payload_ops;
}

struct nft_expr_type nft_payload_type __read_mostly = {
        .name                = "payload",
        .select_ops        = nft_payload_select_ops,
        .inner_ops        = &nft_payload_inner_ops,
        .policy                = nft_payload_policy,
        .maxattr        = NFTA_PAYLOAD_MAX,
        .owner                = THIS_MODULE,
};
























































































  211 

















   93 













    1 






















































  423 

























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMEKEEPING_H
#define _LINUX_TIMEKEEPING_H

#include <linux/errno.h>
#include <linux/clocksource_ids.h>
#include <linux/ktime.h>

/* Included from linux/ktime.h */

void timekeeping_init(void);
extern int timekeeping_suspended;

/* Architecture timer tick functions: */
extern void legacy_timer_tick(unsigned long ticks);

/*
 * Get and set timeofday
 */
extern int do_settimeofday64(const struct timespec64 *ts);
extern int do_sys_settimeofday64(const struct timespec64 *tv,
                                 const struct timezone *tz);

/*
 * ktime_get() family - read the current time in a multitude of ways.
 *
 * The default time reference is CLOCK_MONOTONIC, starting at
 * boot time but not counting the time spent in suspend.
 * For other references, use the functions with "real", "clocktai",
 * "boottime" and "raw" suffixes.
 *
 * To get the time in a different format, use the ones with
 * "ns", "ts64" and "seconds" suffix.
 *
 * See Documentation/core-api/timekeeping.rst for more details.
 */


/*
 * timespec64 based interfaces
 */
extern void ktime_get_raw_ts64(struct timespec64 *ts);
extern void ktime_get_ts64(struct timespec64 *ts);
extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);
extern void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts);

/* Multigrain timestamp interfaces */
extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts);
extern void ktime_get_real_ts64_mg(struct timespec64 *ts);
extern unsigned long timekeeping_get_mg_floor_swaps(void);

void getboottime64(struct timespec64 *ts);

/*
 * time64_t base interfaces
 */
extern time64_t ktime_get_seconds(void);
extern time64_t __ktime_get_real_seconds(void);
extern time64_t ktime_get_real_seconds(void);

/*
 * ktime_t based interfaces
 */

enum tk_offsets {
        TK_OFFS_REAL,
        TK_OFFS_BOOT,
        TK_OFFS_TAI,
        TK_OFFS_MAX,
};

extern ktime_t ktime_get(void);
extern ktime_t ktime_get_with_offset(enum tk_offsets offs);
extern ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs);
extern ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs);
extern ktime_t ktime_get_raw(void);
extern u32 ktime_get_resolution_ns(void);

/**
 * ktime_get_real - get the real (wall-) time in ktime_t format
 *
 * Returns: real (wall) time in ktime_t format
 */
static inline ktime_t ktime_get_real(void)
{
        return ktime_get_with_offset(TK_OFFS_REAL);
}

static inline ktime_t ktime_get_coarse_real(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_REAL);
}

/**
 * ktime_get_boottime - Get monotonic time since boot in ktime_t format
 *
 * This is similar to CLOCK_MONTONIC/ktime_get, but also includes the
 * time spent in suspend.
 *
 * Returns: monotonic time since boot in ktime_t format
 */
static inline ktime_t ktime_get_boottime(void)
{
        return ktime_get_with_offset(TK_OFFS_BOOT);
}

static inline ktime_t ktime_get_coarse_boottime(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_BOOT);
}

/**
 * ktime_get_clocktai - Get the TAI time of day in ktime_t format
 *
 * Returns: the TAI time of day in ktime_t format
 */
static inline ktime_t ktime_get_clocktai(void)
{
        return ktime_get_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse_clocktai(void)
{
        return ktime_get_coarse_with_offset(TK_OFFS_TAI);
}

static inline ktime_t ktime_get_coarse(void)
{
        struct timespec64 ts;

        ktime_get_coarse_ts64(&ts);
        return timespec64_to_ktime(ts);
}

static inline u64 ktime_get_coarse_ns(void)
{
        return ktime_to_ns(ktime_get_coarse());
}

static inline u64 ktime_get_coarse_real_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_real());
}

static inline u64 ktime_get_coarse_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_boottime());
}

static inline u64 ktime_get_coarse_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_coarse_clocktai());
}

/**
 * ktime_mono_to_real - Convert monotonic time to clock realtime
 * @mono: monotonic time to convert
 *
 * Returns: time converted to realtime clock
 */
static inline ktime_t ktime_mono_to_real(ktime_t mono)
{
        return ktime_mono_to_any(mono, TK_OFFS_REAL);
}

/**
 * ktime_get_ns - Get the current time in nanoseconds
 *
 * Returns: current time converted to nanoseconds
 */
static inline u64 ktime_get_ns(void)
{
        return ktime_to_ns(ktime_get());
}

/**
 * ktime_get_real_ns - Get the current real/wall time in nanoseconds
 *
 * Returns: current real time converted to nanoseconds
 */
static inline u64 ktime_get_real_ns(void)
{
        return ktime_to_ns(ktime_get_real());
}

/**
 * ktime_get_boottime_ns - Get the monotonic time since boot in nanoseconds
 *
 * Returns: current boottime converted to nanoseconds
 */
static inline u64 ktime_get_boottime_ns(void)
{
        return ktime_to_ns(ktime_get_boottime());
}

/**
 * ktime_get_clocktai_ns - Get the current TAI time of day in nanoseconds
 *
 * Returns: current TAI time converted to nanoseconds
 */
static inline u64 ktime_get_clocktai_ns(void)
{
        return ktime_to_ns(ktime_get_clocktai());
}

/**
 * ktime_get_raw_ns - Get the raw monotonic time in nanoseconds
 *
 * Returns: current raw monotonic time converted to nanoseconds
 */
static inline u64 ktime_get_raw_ns(void)
{
        return ktime_to_ns(ktime_get_raw());
}

extern u64 ktime_get_mono_fast_ns(void);
extern u64 ktime_get_raw_fast_ns(void);
extern u64 ktime_get_boot_fast_ns(void);
extern u64 ktime_get_tai_fast_ns(void);
extern u64 ktime_get_real_fast_ns(void);

/*
 * timespec64/time64_t interfaces utilizing the ktime based ones
 * for API completeness, these could be implemented more efficiently
 * if needed.
 */
static inline void ktime_get_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_boottime());
}

static inline void ktime_get_coarse_boottime_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_boottime());
}

static inline time64_t ktime_get_boottime_seconds(void)
{
        return ktime_divns(ktime_get_coarse_boottime(), NSEC_PER_SEC);
}

static inline void ktime_get_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_clocktai());
}

static inline void ktime_get_coarse_clocktai_ts64(struct timespec64 *ts)
{
        *ts = ktime_to_timespec64(ktime_get_coarse_clocktai());
}

static inline time64_t ktime_get_clocktai_seconds(void)
{
        return ktime_divns(ktime_get_coarse_clocktai(), NSEC_PER_SEC);
}

/*
 * RTC specific
 */
extern bool timekeeping_rtc_skipsuspend(void);
extern bool timekeeping_rtc_skipresume(void);

extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);

/*
 * Auxiliary clock interfaces
 */
#ifdef CONFIG_POSIX_AUX_CLOCKS
extern bool ktime_get_aux(clockid_t id, ktime_t *kt);
extern bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt);
#else
static inline bool ktime_get_aux(clockid_t id, ktime_t *kt) { return false; }
static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { return false; }
#endif

/**
 * struct system_time_snapshot - simultaneous raw/real time capture with
 *                                 counter value
 * @cycles:        Clocksource counter value to produce the system times
 * @real:        Realtime system time
 * @boot:        Boot time
 * @raw:        Monotonic raw system time
 * @cs_id:        Clocksource ID
 * @clock_was_set_seq:        The sequence number of clock-was-set events
 * @cs_was_changed_seq:        The sequence number of clocksource change events
 */
struct system_time_snapshot {
        u64                        cycles;
        ktime_t                        real;
        ktime_t                        boot;
        ktime_t                        raw;
        enum clocksource_ids        cs_id;
        unsigned int                clock_was_set_seq;
        u8                        cs_was_changed_seq;
};

/**
 * struct system_device_crosststamp - system/device cross-timestamp
 *                                      (synchronized capture)
 * @device:                Device time
 * @sys_realtime:        Realtime simultaneous with device time
 * @sys_monoraw:        Monotonic raw simultaneous with device time
 */
struct system_device_crosststamp {
        ktime_t device;
        ktime_t sys_realtime;
        ktime_t sys_monoraw;
};

/**
 * struct system_counterval_t - system counter value with the ID of the
 *                                corresponding clocksource
 * @cycles:        System counter value
 * @cs_id:        Clocksource ID corresponding to system counter value. Used by
 *                timekeeping code to verify comparability of two cycle values.
 *                The default ID, CSID_GENERIC, does not identify a specific
 *                clocksource.
 * @use_nsecs:        @cycles is in nanoseconds.
 */
struct system_counterval_t {
        u64                        cycles;
        enum clocksource_ids        cs_id;
        bool                        use_nsecs;
};

extern bool ktime_real_to_base_clock(ktime_t treal,
                                     enum clocksource_ids base_id, u64 *cycles);
extern bool timekeeping_clocksource_has_base(enum clocksource_ids id);

/*
 * Get cross timestamp between system clock and device clock
 */
extern int get_device_system_crosststamp(
                        int (*get_time_fn)(ktime_t *device_time,
                                struct system_counterval_t *system_counterval,
                                void *ctx),
                        void *ctx,
                        struct system_time_snapshot *history,
                        struct system_device_crosststamp *xtstamp);

/*
 * Simultaneously snapshot realtime and monotonic raw clocks
 */
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);

/*
 * Persistent clock related interfaces
 */
extern int persistent_clock_is_local;

extern void read_persistent_clock64(struct timespec64 *ts);
void read_persistent_wall_and_boot_offset(struct timespec64 *wall_clock,
                                          struct timespec64 *boot_offset);
#ifdef CONFIG_GENERIC_CMOS_UPDATE
extern int update_persistent_clock64(struct timespec64 now);
#endif

#endif





















    1 

























    1 







    2 






    1 

    1 





















    1 

    2 



































    2 








    2 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * CBC: Cipher Block Chaining mode
 *
 * Copyright (c) 2006-2016 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <crypto/internal/skcipher.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/log2.h>
#include <linux/module.h>

static int crypto_cbc_encrypt_segment(struct crypto_lskcipher *tfm,
                                      const u8 *src, u8 *dst, unsigned nbytes,
                                      u8 *iv)
{
        unsigned int bsize = crypto_lskcipher_blocksize(tfm);

        for (; nbytes >= bsize; src += bsize, dst += bsize, nbytes -= bsize) {
                crypto_xor(iv, src, bsize);
                crypto_lskcipher_encrypt(tfm, iv, dst, bsize, NULL);
                memcpy(iv, dst, bsize);
        }

        return nbytes;
}

static int crypto_cbc_encrypt_inplace(struct crypto_lskcipher *tfm,
                                      u8 *src, unsigned nbytes, u8 *oiv)
{
        unsigned int bsize = crypto_lskcipher_blocksize(tfm);
        u8 *iv = oiv;

        if (nbytes < bsize)
                goto out;

        do {
                crypto_xor(src, iv, bsize);
                crypto_lskcipher_encrypt(tfm, src, src, bsize, NULL);
                iv = src;

                src += bsize;
        } while ((nbytes -= bsize) >= bsize);

        memcpy(oiv, iv, bsize);

out:
        return nbytes;
}

static int crypto_cbc_encrypt(struct crypto_lskcipher *tfm, const u8 *src,
                              u8 *dst, unsigned len, u8 *iv, u32 flags)
{
        struct crypto_lskcipher **ctx = crypto_lskcipher_ctx(tfm);
        bool final = flags & CRYPTO_LSKCIPHER_FLAG_FINAL;
        struct crypto_lskcipher *cipher = *ctx;
        int rem;

        if (src == dst)
                rem = crypto_cbc_encrypt_inplace(cipher, dst, len, iv);
        else
                rem = crypto_cbc_encrypt_segment(cipher, src, dst, len, iv);

        return rem && final ? -EINVAL : rem;
}

static int crypto_cbc_decrypt_segment(struct crypto_lskcipher *tfm,
                                      const u8 *src, u8 *dst, unsigned nbytes,
                                      u8 *oiv)
{
        unsigned int bsize = crypto_lskcipher_blocksize(tfm);
        const u8 *iv = oiv;

        if (nbytes < bsize)
                goto out;

        do {
                crypto_lskcipher_decrypt(tfm, src, dst, bsize, NULL);
                crypto_xor(dst, iv, bsize);
                iv = src;

                src += bsize;
                dst += bsize;
        } while ((nbytes -= bsize) >= bsize);

        memcpy(oiv, iv, bsize);

out:
        return nbytes;
}

static int crypto_cbc_decrypt_inplace(struct crypto_lskcipher *tfm,
                                      u8 *src, unsigned nbytes, u8 *iv)
{
        unsigned int bsize = crypto_lskcipher_blocksize(tfm);
        u8 last_iv[MAX_CIPHER_BLOCKSIZE];

        if (nbytes < bsize)
                goto out;

        /* Start of the last block. */
        src += nbytes - (nbytes & (bsize - 1)) - bsize;
        memcpy(last_iv, src, bsize);

        for (;;) {
                crypto_lskcipher_decrypt(tfm, src, src, bsize, NULL);
                if ((nbytes -= bsize) < bsize)
                        break;
                crypto_xor(src, src - bsize, bsize);
                src -= bsize;
        }

        crypto_xor(src, iv, bsize);
        memcpy(iv, last_iv, bsize);

out:
        return nbytes;
}

static int crypto_cbc_decrypt(struct crypto_lskcipher *tfm, const u8 *src,
                              u8 *dst, unsigned len, u8 *iv, u32 flags)
{
        struct crypto_lskcipher **ctx = crypto_lskcipher_ctx(tfm);
        bool final = flags & CRYPTO_LSKCIPHER_FLAG_FINAL;
        struct crypto_lskcipher *cipher = *ctx;
        int rem;

        if (src == dst)
                rem = crypto_cbc_decrypt_inplace(cipher, dst, len, iv);
        else
                rem = crypto_cbc_decrypt_segment(cipher, src, dst, len, iv);

        return rem && final ? -EINVAL : rem;
}

static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb)
{
        struct lskcipher_instance *inst;
        int err;

        inst = lskcipher_alloc_instance_simple(tmpl, tb);
        if (IS_ERR(inst))
                return PTR_ERR(inst);

        err = -EINVAL;
        if (!is_power_of_2(inst->alg.co.base.cra_blocksize))
                goto out_free_inst;

        if (inst->alg.co.statesize)
                goto out_free_inst;

        inst->alg.encrypt = crypto_cbc_encrypt;
        inst->alg.decrypt = crypto_cbc_decrypt;

        err = lskcipher_register_instance(tmpl, inst);
        if (err) {
out_free_inst:
                inst->free(inst);
        }

        return err;
}

static struct crypto_template crypto_cbc_tmpl = {
        .name = "cbc",
        .create = crypto_cbc_create,
        .module = THIS_MODULE,
};

static int __init crypto_cbc_module_init(void)
{
        return crypto_register_template(&crypto_cbc_tmpl);
}

static void __exit crypto_cbc_module_exit(void)
{
        crypto_unregister_template(&crypto_cbc_tmpl);
}

module_init(crypto_cbc_module_init);
module_exit(crypto_cbc_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("CBC block cipher mode of operation");
MODULE_ALIAS_CRYPTO("cbc");






































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor file mediation function definitions.
 *
 * Copyright (C) 1998-2008 Novell/SUSE
 * Copyright 2009-2010 Canonical Ltd.
 */

#ifndef __AA_FILE_H
#define __AA_FILE_H

#include <linux/spinlock.h>

#include "domain.h"
#include "match.h"
#include "perms.h"

struct aa_policydb;
struct aa_profile;
struct path;

#define mask_mode_t(X) (X & (MAY_EXEC | MAY_WRITE | MAY_READ | MAY_APPEND))

#define AA_AUDIT_FILE_MASK        (MAY_READ | MAY_WRITE | MAY_EXEC | MAY_APPEND |\
                                 AA_MAY_CREATE | AA_MAY_DELETE |        \
                                 AA_MAY_GETATTR | AA_MAY_SETATTR | \
                                 AA_MAY_CHMOD | AA_MAY_CHOWN | AA_MAY_LOCK | \
                                 AA_EXEC_MMAP | AA_MAY_LINK)

static inline struct aa_file_ctx *file_ctx(struct file *file)
{
        return file->f_security + apparmor_blob_sizes.lbs_file;
}

/* struct aa_file_ctx - the AppArmor context the file was opened in
 * @lock: lock to update the ctx
 * @label: label currently cached on the ctx
 * @perms: the permission the file was opened with
 */
struct aa_file_ctx {
        spinlock_t lock;
        struct aa_label __rcu *label;
        u32 allow;
};

/*
 * The xindex is broken into 3 parts
 * - index - an index into either the exec name table or the variable table
 * - exec type - which determines how the executable name and index are used
 * - flags - which modify how the destination name is applied
 */
#define AA_X_INDEX_MASK                AA_INDEX_MASK

#define AA_X_TYPE_MASK                0x0c000000
#define AA_X_NONE                AA_INDEX_NONE
#define AA_X_NAME                0x04000000 /* use executable name px */
#define AA_X_TABLE                0x08000000 /* use a specified name ->n# */

#define AA_X_UNSAFE                0x10000000
#define AA_X_CHILD                0x20000000
#define AA_X_INHERIT                0x40000000
#define AA_X_UNCONFINED                0x80000000

/* need to make conditional which ones are being set */
struct path_cond {
        kuid_t uid;
        umode_t mode;
};

#define COMBINED_PERM_MASK(X) ((X).allow | (X).audit | (X).quiet | (X).kill)

int aa_audit_file(const struct cred *cred,
                  struct aa_profile *profile, struct aa_perms *perms,
                  const char *op, u32 request, const char *name,
                  const char *target, struct aa_label *tlabel, kuid_t ouid,
                  const char *info, int error);

struct aa_perms *aa_lookup_condperms(kuid_t subj_uid,
                                     struct aa_policydb *file_rules,
                                     aa_state_t state, struct path_cond *cond);
aa_state_t aa_str_perms(struct aa_policydb *file_rules, aa_state_t start,
                        const char *name, struct path_cond *cond,
                        struct aa_perms *perms);

int __aa_path_perm(const char *op, const struct cred *subj_cred,
                   struct aa_profile *profile, const char *name,
                   u32 request, struct path_cond *cond, int flags,
                   struct aa_perms *perms);
int aa_path_perm(const char *op, const struct cred *subj_cred,
                 struct aa_label *label, const struct path *path,
                 int flags, u32 request, struct path_cond *cond);

int aa_path_link(const struct cred *subj_cred, struct aa_label *label,
                 struct dentry *old_dentry, const struct path *new_dir,
                 struct dentry *new_dentry);

int aa_file_perm(const char *op, const struct cred *subj_cred,
                 struct aa_label *label, struct file *file,
                 u32 request, bool in_atomic);

void aa_inherit_files(const struct cred *cred, struct files_struct *files);


/**
 * aa_map_file_to_perms - map file flags to AppArmor permissions
 * @file: open file to map flags to AppArmor permissions
 *
 * Returns: apparmor permission set for the file
 */
static inline u32 aa_map_file_to_perms(struct file *file)
{
        int flags = file->f_flags;
        u32 perms = 0;

        if (file->f_mode & FMODE_WRITE)
                perms |= MAY_WRITE;
        if (file->f_mode & FMODE_READ)
                perms |= MAY_READ;

        if ((flags & O_APPEND) && (perms & MAY_WRITE))
                perms = (perms & ~MAY_WRITE) | MAY_APPEND;
        /* trunc implies write permission */
        if (flags & O_TRUNC)
                perms |= MAY_WRITE;
        if (flags & O_CREAT)
                perms |= AA_MAY_CREATE;

        return perms;
}

#endif /* __AA_FILE_H */



















































































































































































   28 
















    4 




    1 



    3 




    3 
    3 


    3 


    2 
    1 







    1 
    1 




    2 
    2 




    1 
    1 






































   24 




    1 
















    4 
    1 


    2 

    2 






    2 

    1 







    1 








    6 
    2 



    5 

    3 





    2 



    1 

    1 






   11 




    8 
   13 








    3 
   23 















   24 





   24 











    1 



    4 

   19 






    4 



   19 





    3 
    3 


















   37 















































   37 



























   29 


    1 


    6 



    3 
    2 



    1 









    2 
    2 







    1 
    1 





















































































































   37 



    9 



    4 




    1 















    1 



    1 
    1 





    1 



























    1 






































































   24 
   19 
    4 















    1 


   19 



   10 

    1 




    2 



    1 







    1 



    1 



    2 



    1 




































   37 





   37 










   37 








   37 


   36 







   37 




   37 
   37 


   37 




















































































































   41 





    3 




    1 




















   37 


   37 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2007-2014 Nicira, Inc.
 */

#include <linux/uaccess.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <net/llc_pdu.h>
#include <linux/kernel.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/llc.h>
#include <linux/module.h>
#include <linux/in.h>
#include <linux/rcupdate.h>
#include <linux/cpumask.h>
#include <linux/if_arp.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/mpls.h>
#include <linux/sctp.h>
#include <linux/smp.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/icmp.h>
#include <linux/icmpv6.h>
#include <linux/rculist.h>
#include <net/ip.h>
#include <net/ip_tunnels.h>
#include <net/ipv6.h>
#include <net/mpls.h>
#include <net/ndisc.h>
#include <net/nsh.h>
#include <net/pkt_cls.h>
#include <net/netfilter/nf_conntrack_zones.h>

#include "conntrack.h"
#include "datapath.h"
#include "flow.h"
#include "flow_netlink.h"
#include "vport.h"

u64 ovs_flow_used_time(unsigned long flow_jiffies)
{
        struct timespec64 cur_ts;
        u64 cur_ms, idle_ms;

        ktime_get_ts64(&cur_ts);
        idle_ms = jiffies_to_msecs(jiffies - flow_jiffies);
        cur_ms = (u64)(u32)cur_ts.tv_sec * MSEC_PER_SEC +
                 cur_ts.tv_nsec / NSEC_PER_MSEC;

        return cur_ms - idle_ms;
}

#define TCP_FLAGS_BE16(tp) (*(__be16 *)&tcp_flag_word(tp) & htons(0x0FFF))

void ovs_flow_stats_update(struct sw_flow *flow, __be16 tcp_flags,
                           const struct sk_buff *skb)
{
        struct sw_flow_stats *stats;
        unsigned int cpu = smp_processor_id();
        int len = skb->len + (skb_vlan_tag_present(skb) ? VLAN_HLEN : 0);

        stats = rcu_dereference(flow->stats[cpu]);

        /* Check if already have CPU-specific stats. */
        if (likely(stats)) {
                spin_lock(&stats->lock);
                /* Mark if we write on the pre-allocated stats. */
                if (cpu == 0 && unlikely(flow->stats_last_writer != cpu))
                        flow->stats_last_writer = cpu;
        } else {
                stats = rcu_dereference(flow->stats[0]); /* Pre-allocated. */
                spin_lock(&stats->lock);

                /* If the current CPU is the only writer on the
                 * pre-allocated stats keep using them.
                 */
                if (unlikely(flow->stats_last_writer != cpu)) {
                        /* A previous locker may have already allocated the
                         * stats, so we need to check again.  If CPU-specific
                         * stats were already allocated, we update the pre-
                         * allocated stats as we have already locked them.
                         */
                        if (likely(flow->stats_last_writer != -1) &&
                            likely(!rcu_access_pointer(flow->stats[cpu]))) {
                                /* Try to allocate CPU-specific stats. */
                                struct sw_flow_stats *new_stats;

                                new_stats =
                                        kmem_cache_alloc_node(flow_stats_cache,
                                                              GFP_NOWAIT |
                                                              __GFP_THISNODE |
                                                              __GFP_NOWARN |
                                                              __GFP_NOMEMALLOC,
                                                              numa_node_id());
                                if (likely(new_stats)) {
                                        new_stats->used = jiffies;
                                        new_stats->packet_count = 1;
                                        new_stats->byte_count = len;
                                        new_stats->tcp_flags = tcp_flags;
                                        spin_lock_init(&new_stats->lock);

                                        rcu_assign_pointer(flow->stats[cpu],
                                                           new_stats);
                                        cpumask_set_cpu(cpu,
                                                        flow->cpu_used_mask);
                                        goto unlock;
                                }
                        }
                        flow->stats_last_writer = cpu;
                }
        }

        stats->used = jiffies;
        stats->packet_count++;
        stats->byte_count += len;
        stats->tcp_flags |= tcp_flags;
unlock:
        spin_unlock(&stats->lock);
}

/* Must be called with rcu_read_lock or ovs_mutex. */
void ovs_flow_stats_get(const struct sw_flow *flow,
                        struct ovs_flow_stats *ovs_stats,
                        unsigned long *used, __be16 *tcp_flags)
{
        unsigned int cpu;

        *used = 0;
        *tcp_flags = 0;
        memset(ovs_stats, 0, sizeof(*ovs_stats));

        for_each_cpu(cpu, flow->cpu_used_mask) {
                struct sw_flow_stats *stats = rcu_dereference_ovsl(flow->stats[cpu]);

                if (stats) {
                        /* Local CPU may write on non-local stats, so we must
                         * block bottom-halves here.
                         */
                        spin_lock_bh(&stats->lock);
                        if (!*used || time_after(stats->used, *used))
                                *used = stats->used;
                        *tcp_flags |= stats->tcp_flags;
                        ovs_stats->n_packets += stats->packet_count;
                        ovs_stats->n_bytes += stats->byte_count;
                        spin_unlock_bh(&stats->lock);
                }
        }
}

/* Called with ovs_mutex. */
void ovs_flow_stats_clear(struct sw_flow *flow)
{
        unsigned int cpu;

        for_each_cpu(cpu, flow->cpu_used_mask) {
                struct sw_flow_stats *stats = ovsl_dereference(flow->stats[cpu]);

                if (stats) {
                        spin_lock_bh(&stats->lock);
                        stats->used = 0;
                        stats->packet_count = 0;
                        stats->byte_count = 0;
                        stats->tcp_flags = 0;
                        spin_unlock_bh(&stats->lock);
                }
        }
}

static int check_header(struct sk_buff *skb, int len)
{
        if (unlikely(skb->len < len))
                return -EINVAL;
        if (unlikely(!pskb_may_pull(skb, len)))
                return -ENOMEM;
        return 0;
}

static bool arphdr_ok(struct sk_buff *skb)
{
        return pskb_may_pull(skb, skb_network_offset(skb) +
                                  sizeof(struct arp_eth_header));
}

static int check_iphdr(struct sk_buff *skb)
{
        unsigned int nh_ofs = skb_network_offset(skb);
        unsigned int ip_len;
        int err;

        err = check_header(skb, nh_ofs + sizeof(struct iphdr));
        if (unlikely(err))
                return err;

        ip_len = ip_hdrlen(skb);
        if (unlikely(ip_len < sizeof(struct iphdr) ||
                     skb->len < nh_ofs + ip_len))
                return -EINVAL;

        skb_set_transport_header(skb, nh_ofs + ip_len);
        return 0;
}

static bool tcphdr_ok(struct sk_buff *skb)
{
        int th_ofs = skb_transport_offset(skb);
        int tcp_len;

        if (unlikely(!pskb_may_pull(skb, th_ofs + sizeof(struct tcphdr))))
                return false;

        tcp_len = tcp_hdrlen(skb);
        if (unlikely(tcp_len < sizeof(struct tcphdr) ||
                     skb->len < th_ofs + tcp_len))
                return false;

        return true;
}

static bool udphdr_ok(struct sk_buff *skb)
{
        return pskb_may_pull(skb, skb_transport_offset(skb) +
                                  sizeof(struct udphdr));
}

static bool sctphdr_ok(struct sk_buff *skb)
{
        return pskb_may_pull(skb, skb_transport_offset(skb) +
                                  sizeof(struct sctphdr));
}

static bool icmphdr_ok(struct sk_buff *skb)
{
        return pskb_may_pull(skb, skb_transport_offset(skb) +
                                  sizeof(struct icmphdr));
}

/**
 * get_ipv6_ext_hdrs() - Parses packet and sets IPv6 extension header flags.
 *
 * @skb: buffer where extension header data starts in packet
 * @nh: ipv6 header
 * @ext_hdrs: flags are stored here
 *
 * OFPIEH12_UNREP is set if more than one of a given IPv6 extension header
 * is unexpectedly encountered. (Two destination options headers may be
 * expected and would not cause this bit to be set.)
 *
 * OFPIEH12_UNSEQ is set if IPv6 extension headers were not in the order
 * preferred (but not required) by RFC 2460:
 *
 * When more than one extension header is used in the same packet, it is
 * recommended that those headers appear in the following order:
 *      IPv6 header
 *      Hop-by-Hop Options header
 *      Destination Options header
 *      Routing header
 *      Fragment header
 *      Authentication header
 *      Encapsulating Security Payload header
 *      Destination Options header
 *      upper-layer header
 */
static void get_ipv6_ext_hdrs(struct sk_buff *skb, struct ipv6hdr *nh,
                              u16 *ext_hdrs)
{
        u8 next_type = nh->nexthdr;
        unsigned int start = skb_network_offset(skb) + sizeof(struct ipv6hdr);
        int dest_options_header_count = 0;

        *ext_hdrs = 0;

        while (ipv6_ext_hdr(next_type)) {
                struct ipv6_opt_hdr _hdr, *hp;

                switch (next_type) {
                case IPPROTO_NONE:
                        *ext_hdrs |= OFPIEH12_NONEXT;
                        /* stop parsing */
                        return;

                case IPPROTO_ESP:
                        if (*ext_hdrs & OFPIEH12_ESP)
                                *ext_hdrs |= OFPIEH12_UNREP;
                        if ((*ext_hdrs & ~(OFPIEH12_HOP | OFPIEH12_DEST |
                                           OFPIEH12_ROUTER | IPPROTO_FRAGMENT |
                                           OFPIEH12_AUTH | OFPIEH12_UNREP)) ||
                            dest_options_header_count >= 2) {
                                *ext_hdrs |= OFPIEH12_UNSEQ;
                        }
                        *ext_hdrs |= OFPIEH12_ESP;
                        break;

                case IPPROTO_AH:
                        if (*ext_hdrs & OFPIEH12_AUTH)
                                *ext_hdrs |= OFPIEH12_UNREP;
                        if ((*ext_hdrs &
                             ~(OFPIEH12_HOP | OFPIEH12_DEST | OFPIEH12_ROUTER |
                               IPPROTO_FRAGMENT | OFPIEH12_UNREP)) ||
                            dest_options_header_count >= 2) {
                                *ext_hdrs |= OFPIEH12_UNSEQ;
                        }
                        *ext_hdrs |= OFPIEH12_AUTH;
                        break;

                case IPPROTO_DSTOPTS:
                        if (dest_options_header_count == 0) {
                                if (*ext_hdrs &
                                    ~(OFPIEH12_HOP | OFPIEH12_UNREP))
                                        *ext_hdrs |= OFPIEH12_UNSEQ;
                                *ext_hdrs |= OFPIEH12_DEST;
                        } else if (dest_options_header_count == 1) {
                                if (*ext_hdrs &
                                    ~(OFPIEH12_HOP | OFPIEH12_DEST |
                                      OFPIEH12_ROUTER | OFPIEH12_FRAG |
                                      OFPIEH12_AUTH | OFPIEH12_ESP |
                                      OFPIEH12_UNREP)) {
                                        *ext_hdrs |= OFPIEH12_UNSEQ;
                                }
                        } else {
                                *ext_hdrs |= OFPIEH12_UNREP;
                        }
                        dest_options_header_count++;
                        break;

                case IPPROTO_FRAGMENT:
                        if (*ext_hdrs & OFPIEH12_FRAG)
                                *ext_hdrs |= OFPIEH12_UNREP;
                        if ((*ext_hdrs & ~(OFPIEH12_HOP |
                                           OFPIEH12_DEST |
                                           OFPIEH12_ROUTER |
                                           OFPIEH12_UNREP)) ||
                            dest_options_header_count >= 2) {
                                *ext_hdrs |= OFPIEH12_UNSEQ;
                        }
                        *ext_hdrs |= OFPIEH12_FRAG;
                        break;

                case IPPROTO_ROUTING:
                        if (*ext_hdrs & OFPIEH12_ROUTER)
                                *ext_hdrs |= OFPIEH12_UNREP;
                        if ((*ext_hdrs & ~(OFPIEH12_HOP |
                                           OFPIEH12_DEST |
                                           OFPIEH12_UNREP)) ||
                            dest_options_header_count >= 2) {
                                *ext_hdrs |= OFPIEH12_UNSEQ;
                        }
                        *ext_hdrs |= OFPIEH12_ROUTER;
                        break;

                case IPPROTO_HOPOPTS:
                        if (*ext_hdrs & OFPIEH12_HOP)
                                *ext_hdrs |= OFPIEH12_UNREP;
                        /* OFPIEH12_HOP is set to 1 if a hop-by-hop IPv6
                         * extension header is present as the first
                         * extension header in the packet.
                         */
                        if (*ext_hdrs == 0)
                                *ext_hdrs |= OFPIEH12_HOP;
                        else
                                *ext_hdrs |= OFPIEH12_UNSEQ;
                        break;

                default:
                        return;
                }

                hp = skb_header_pointer(skb, start, sizeof(_hdr), &_hdr);
                if (!hp)
                        break;
                next_type = hp->nexthdr;
                start += ipv6_optlen(hp);
        }
}

static int parse_ipv6hdr(struct sk_buff *skb, struct sw_flow_key *key)
{
        unsigned short frag_off;
        unsigned int payload_ofs = 0;
        unsigned int nh_ofs = skb_network_offset(skb);
        unsigned int nh_len;
        struct ipv6hdr *nh;
        int err, nexthdr, flags = 0;

        err = check_header(skb, nh_ofs + sizeof(*nh));
        if (unlikely(err))
                return err;

        nh = ipv6_hdr(skb);

        get_ipv6_ext_hdrs(skb, nh, &key->ipv6.exthdrs);

        key->ip.proto = NEXTHDR_NONE;
        key->ip.tos = ipv6_get_dsfield(nh);
        key->ip.ttl = nh->hop_limit;
        key->ipv6.label = *(__be32 *)nh & htonl(IPV6_FLOWINFO_FLOWLABEL);
        key->ipv6.addr.src = nh->saddr;
        key->ipv6.addr.dst = nh->daddr;

        nexthdr = ipv6_find_hdr(skb, &payload_ofs, -1, &frag_off, &flags);
        if (flags & IP6_FH_F_FRAG) {
                if (frag_off) {
                        key->ip.frag = OVS_FRAG_TYPE_LATER;
                        key->ip.proto = NEXTHDR_FRAGMENT;
                        return 0;
                }
                key->ip.frag = OVS_FRAG_TYPE_FIRST;
        } else {
                key->ip.frag = OVS_FRAG_TYPE_NONE;
        }

        /* Delayed handling of error in ipv6_find_hdr() as it
         * always sets flags and frag_off to a valid value which may be
         * used to set key->ip.frag above.
         */
        if (unlikely(nexthdr < 0))
                return -EPROTO;

        nh_len = payload_ofs - nh_ofs;
        skb_set_transport_header(skb, nh_ofs + nh_len);
        key->ip.proto = nexthdr;
        return nh_len;
}

static bool icmp6hdr_ok(struct sk_buff *skb)
{
        return pskb_may_pull(skb, skb_transport_offset(skb) +
                                  sizeof(struct icmp6hdr));
}

/**
 * parse_vlan_tag - Parse vlan tag from vlan header.
 * @skb: skb containing frame to parse
 * @key_vh: pointer to parsed vlan tag
 * @untag_vlan: should the vlan header be removed from the frame
 *
 * Return: ERROR on memory error.
 * %0 if it encounters a non-vlan or incomplete packet.
 * %1 after successfully parsing vlan tag.
 */
static int parse_vlan_tag(struct sk_buff *skb, struct vlan_head *key_vh,
                          bool untag_vlan)
{
        struct vlan_head *vh = (struct vlan_head *)skb->data;

        if (likely(!eth_type_vlan(vh->tpid)))
                return 0;

        if (unlikely(skb->len < sizeof(struct vlan_head) + sizeof(__be16)))
                return 0;

        if (unlikely(!pskb_may_pull(skb, sizeof(struct vlan_head) +
                                 sizeof(__be16))))
                return -ENOMEM;

        vh = (struct vlan_head *)skb->data;
        key_vh->tci = vh->tci | htons(VLAN_CFI_MASK);
        key_vh->tpid = vh->tpid;

        if (unlikely(untag_vlan)) {
                int offset = skb->data - skb_mac_header(skb);
                u16 tci;
                int err;

                __skb_push(skb, offset);
                err = __skb_vlan_pop(skb, &tci);
                __skb_pull(skb, offset);
                if (err)
                        return err;
                __vlan_hwaccel_put_tag(skb, key_vh->tpid, tci);
        } else {
                __skb_pull(skb, sizeof(struct vlan_head));
        }
        return 1;
}

static void clear_vlan(struct sw_flow_key *key)
{
        key->eth.vlan.tci = 0;
        key->eth.vlan.tpid = 0;
        key->eth.cvlan.tci = 0;
        key->eth.cvlan.tpid = 0;
}

static int parse_vlan(struct sk_buff *skb, struct sw_flow_key *key)
{
        int res;

        if (skb_vlan_tag_present(skb)) {
                key->eth.vlan.tci = htons(skb->vlan_tci) | htons(VLAN_CFI_MASK);
                key->eth.vlan.tpid = skb->vlan_proto;
        } else {
                /* Parse outer vlan tag in the non-accelerated case. */
                res = parse_vlan_tag(skb, &key->eth.vlan, true);
                if (res <= 0)
                        return res;
        }

        /* Parse inner vlan tag. */
        res = parse_vlan_tag(skb, &key->eth.cvlan, false);
        if (res <= 0)
                return res;

        return 0;
}

static __be16 parse_ethertype(struct sk_buff *skb)
{
        struct llc_snap_hdr {
                u8  dsap;  /* Always 0xAA */
                u8  ssap;  /* Always 0xAA */
                u8  ctrl;
                u8  oui[3];
                __be16 ethertype;
        };
        struct llc_snap_hdr *llc;
        __be16 proto;

        proto = *(__be16 *) skb->data;
        __skb_pull(skb, sizeof(__be16));

        if (eth_proto_is_802_3(proto))
                return proto;

        if (skb->len < sizeof(struct llc_snap_hdr))
                return htons(ETH_P_802_2);

        if (unlikely(!pskb_may_pull(skb, sizeof(struct llc_snap_hdr))))
                return htons(0);

        llc = (struct llc_snap_hdr *) skb->data;
        if (llc->dsap != LLC_SAP_SNAP ||
            llc->ssap != LLC_SAP_SNAP ||
            (llc->oui[0] | llc->oui[1] | llc->oui[2]) != 0)
                return htons(ETH_P_802_2);

        __skb_pull(skb, sizeof(struct llc_snap_hdr));

        if (eth_proto_is_802_3(llc->ethertype))
                return llc->ethertype;

        return htons(ETH_P_802_2);
}

static int parse_icmpv6(struct sk_buff *skb, struct sw_flow_key *key,
                        int nh_len)
{
        struct icmp6hdr *icmp = icmp6_hdr(skb);

        /* The ICMPv6 type and code fields use the 16-bit transport port
         * fields, so we need to store them in 16-bit network byte order.
         */
        key->tp.src = htons(icmp->icmp6_type);
        key->tp.dst = htons(icmp->icmp6_code);

        if (icmp->icmp6_code == 0 &&
            (icmp->icmp6_type == NDISC_NEIGHBOUR_SOLICITATION ||
             icmp->icmp6_type == NDISC_NEIGHBOUR_ADVERTISEMENT)) {
                int icmp_len = skb->len - skb_transport_offset(skb);
                struct nd_msg *nd;
                int offset;

                memset(&key->ipv6.nd, 0, sizeof(key->ipv6.nd));

                /* In order to process neighbor discovery options, we need the
                 * entire packet.
                 */
                if (unlikely(icmp_len < sizeof(*nd)))
                        return 0;

                if (unlikely(skb_linearize(skb)))
                        return -ENOMEM;

                nd = (struct nd_msg *)skb_transport_header(skb);
                key->ipv6.nd.target = nd->target;

                icmp_len -= sizeof(*nd);
                offset = 0;
                while (icmp_len >= 8) {
                        struct nd_opt_hdr *nd_opt =
                                 (struct nd_opt_hdr *)(nd->opt + offset);
                        int opt_len = nd_opt->nd_opt_len * 8;

                        if (unlikely(!opt_len || opt_len > icmp_len))
                                return 0;

                        /* Store the link layer address if the appropriate
                         * option is provided.  It is considered an error if
                         * the same link layer option is specified twice.
                         */
                        if (nd_opt->nd_opt_type == ND_OPT_SOURCE_LL_ADDR
                            && opt_len == 8) {
                                if (unlikely(!is_zero_ether_addr(key->ipv6.nd.sll)))
                                        goto invalid;
                                ether_addr_copy(key->ipv6.nd.sll,
                                                &nd->opt[offset+sizeof(*nd_opt)]);
                        } else if (nd_opt->nd_opt_type == ND_OPT_TARGET_LL_ADDR
                                   && opt_len == 8) {
                                if (unlikely(!is_zero_ether_addr(key->ipv6.nd.tll)))
                                        goto invalid;
                                ether_addr_copy(key->ipv6.nd.tll,
                                                &nd->opt[offset+sizeof(*nd_opt)]);
                        }

                        icmp_len -= opt_len;
                        offset += opt_len;
                }
        }

        return 0;

invalid:
        memset(&key->ipv6.nd.target, 0, sizeof(key->ipv6.nd.target));
        memset(key->ipv6.nd.sll, 0, sizeof(key->ipv6.nd.sll));
        memset(key->ipv6.nd.tll, 0, sizeof(key->ipv6.nd.tll));

        return 0;
}

static int parse_nsh(struct sk_buff *skb, struct sw_flow_key *key)
{
        struct nshhdr *nh;
        unsigned int nh_ofs = skb_network_offset(skb);
        u8 version, length;
        int err;

        err = check_header(skb, nh_ofs + NSH_BASE_HDR_LEN);
        if (unlikely(err))
                return err;

        nh = nsh_hdr(skb);
        version = nsh_get_ver(nh);
        length = nsh_hdr_len(nh);

        if (version != 0)
                return -EINVAL;

        err = check_header(skb, nh_ofs + length);
        if (unlikely(err))
                return err;

        nh = nsh_hdr(skb);
        key->nsh.base.flags = nsh_get_flags(nh);
        key->nsh.base.ttl = nsh_get_ttl(nh);
        key->nsh.base.mdtype = nh->mdtype;
        key->nsh.base.np = nh->np;
        key->nsh.base.path_hdr = nh->path_hdr;
        switch (key->nsh.base.mdtype) {
        case NSH_M_TYPE1:
                if (length != NSH_M_TYPE1_LEN)
                        return -EINVAL;
                memcpy(key->nsh.context, nh->md1.context,
                       sizeof(nh->md1));
                break;
        case NSH_M_TYPE2:
                memset(key->nsh.context, 0,
                       sizeof(nh->md1));
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

/**
 * key_extract_l3l4 - extracts L3/L4 header information.
 * @skb: sk_buff that contains the frame, with skb->data pointing to the
 *       L3 header
 * @key: output flow key
 *
 * Return: %0 if successful, otherwise a negative errno value.
 */
static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key)
{
        int error;

        /* Network layer. */
        if (key->eth.type == htons(ETH_P_IP)) {
                struct iphdr *nh;
                __be16 offset;

                error = check_iphdr(skb);
                if (unlikely(error)) {
                        memset(&key->ip, 0, sizeof(key->ip));
                        memset(&key->ipv4, 0, sizeof(key->ipv4));
                        if (error == -EINVAL) {
                                skb->transport_header = skb->network_header;
                                error = 0;
                        }
                        return error;
                }

                nh = ip_hdr(skb);
                key->ipv4.addr.src = nh->saddr;
                key->ipv4.addr.dst = nh->daddr;

                key->ip.proto = nh->protocol;
                key->ip.tos = nh->tos;
                key->ip.ttl = nh->ttl;

                offset = nh->frag_off & htons(IP_OFFSET);
                if (offset) {
                        key->ip.frag = OVS_FRAG_TYPE_LATER;
                        memset(&key->tp, 0, sizeof(key->tp));
                        return 0;
                }
                if (nh->frag_off & htons(IP_MF) ||
                        skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
                        key->ip.frag = OVS_FRAG_TYPE_FIRST;
                else
                        key->ip.frag = OVS_FRAG_TYPE_NONE;

                /* Transport layer. */
                if (key->ip.proto == IPPROTO_TCP) {
                        if (tcphdr_ok(skb)) {
                                struct tcphdr *tcp = tcp_hdr(skb);
                                key->tp.src = tcp->source;
                                key->tp.dst = tcp->dest;
                                key->tp.flags = TCP_FLAGS_BE16(tcp);
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }

                } else if (key->ip.proto == IPPROTO_UDP) {
                        if (udphdr_ok(skb)) {
                                struct udphdr *udp = udp_hdr(skb);
                                key->tp.src = udp->source;
                                key->tp.dst = udp->dest;
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                } else if (key->ip.proto == IPPROTO_SCTP) {
                        if (sctphdr_ok(skb)) {
                                struct sctphdr *sctp = sctp_hdr(skb);
                                key->tp.src = sctp->source;
                                key->tp.dst = sctp->dest;
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                } else if (key->ip.proto == IPPROTO_ICMP) {
                        if (icmphdr_ok(skb)) {
                                struct icmphdr *icmp = icmp_hdr(skb);
                                /* The ICMP type and code fields use the 16-bit
                                 * transport port fields, so we need to store
                                 * them in 16-bit network byte order. */
                                key->tp.src = htons(icmp->type);
                                key->tp.dst = htons(icmp->code);
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                }

        } else if (key->eth.type == htons(ETH_P_ARP) ||
                   key->eth.type == htons(ETH_P_RARP)) {
                struct arp_eth_header *arp;
                bool arp_available = arphdr_ok(skb);

                arp = (struct arp_eth_header *)skb_network_header(skb);

                if (arp_available &&
                    arp->ar_hrd == htons(ARPHRD_ETHER) &&
                    arp->ar_pro == htons(ETH_P_IP) &&
                    arp->ar_hln == ETH_ALEN &&
                    arp->ar_pln == 4) {

                        /* We only match on the lower 8 bits of the opcode. */
                        if (ntohs(arp->ar_op) <= 0xff)
                                key->ip.proto = ntohs(arp->ar_op);
                        else
                                key->ip.proto = 0;

                        memcpy(&key->ipv4.addr.src, arp->ar_sip, sizeof(key->ipv4.addr.src));
                        memcpy(&key->ipv4.addr.dst, arp->ar_tip, sizeof(key->ipv4.addr.dst));
                        ether_addr_copy(key->ipv4.arp.sha, arp->ar_sha);
                        ether_addr_copy(key->ipv4.arp.tha, arp->ar_tha);
                } else {
                        memset(&key->ip, 0, sizeof(key->ip));
                        memset(&key->ipv4, 0, sizeof(key->ipv4));
                }
        } else if (eth_p_mpls(key->eth.type)) {
                size_t label_count = 1;

                memset(&key->mpls, 0, sizeof(key->mpls));
                skb_set_inner_network_header(skb, skb->mac_len);
                while (1) {
                        __be32 lse;

                        error = check_header(skb, skb->mac_len +
                                             label_count * MPLS_HLEN);
                        if (unlikely(error))
                                return 0;

                        memcpy(&lse, skb_inner_network_header(skb), MPLS_HLEN);

                        if (label_count <= MPLS_LABEL_DEPTH)
                                memcpy(&key->mpls.lse[label_count - 1], &lse,
                                       MPLS_HLEN);

                        skb_set_inner_network_header(skb, skb->mac_len +
                                                     label_count * MPLS_HLEN);
                        if (lse & htonl(MPLS_LS_S_MASK))
                                break;

                        label_count++;
                }
                if (label_count > MPLS_LABEL_DEPTH)
                        label_count = MPLS_LABEL_DEPTH;

                key->mpls.num_labels_mask = GENMASK(label_count - 1, 0);
        } else if (key->eth.type == htons(ETH_P_IPV6)) {
                int nh_len;             /* IPv6 Header + Extensions */

                nh_len = parse_ipv6hdr(skb, key);
                if (unlikely(nh_len < 0)) {
                        switch (nh_len) {
                        case -EINVAL:
                                memset(&key->ip, 0, sizeof(key->ip));
                                memset(&key->ipv6.addr, 0, sizeof(key->ipv6.addr));
                                fallthrough;
                        case -EPROTO:
                                skb->transport_header = skb->network_header;
                                error = 0;
                                break;
                        default:
                                error = nh_len;
                        }
                        return error;
                }

                if (key->ip.frag == OVS_FRAG_TYPE_LATER) {
                        memset(&key->tp, 0, sizeof(key->tp));
                        return 0;
                }
                if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
                        key->ip.frag = OVS_FRAG_TYPE_FIRST;

                /* Transport layer. */
                if (key->ip.proto == NEXTHDR_TCP) {
                        if (tcphdr_ok(skb)) {
                                struct tcphdr *tcp = tcp_hdr(skb);
                                key->tp.src = tcp->source;
                                key->tp.dst = tcp->dest;
                                key->tp.flags = TCP_FLAGS_BE16(tcp);
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                } else if (key->ip.proto == NEXTHDR_UDP) {
                        if (udphdr_ok(skb)) {
                                struct udphdr *udp = udp_hdr(skb);
                                key->tp.src = udp->source;
                                key->tp.dst = udp->dest;
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                } else if (key->ip.proto == NEXTHDR_SCTP) {
                        if (sctphdr_ok(skb)) {
                                struct sctphdr *sctp = sctp_hdr(skb);
                                key->tp.src = sctp->source;
                                key->tp.dst = sctp->dest;
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                } else if (key->ip.proto == NEXTHDR_ICMP) {
                        if (icmp6hdr_ok(skb)) {
                                error = parse_icmpv6(skb, key, nh_len);
                                if (error)
                                        return error;
                        } else {
                                memset(&key->tp, 0, sizeof(key->tp));
                        }
                }
        } else if (key->eth.type == htons(ETH_P_NSH)) {
                error = parse_nsh(skb, key);
                if (error)
                        return error;
        }
        return 0;
}

/**
 * key_extract - extracts a flow key from an Ethernet frame.
 * @skb: sk_buff that contains the frame, with skb->data pointing to the
 * Ethernet header
 * @key: output flow key
 *
 * The caller must ensure that skb->len >= ETH_HLEN.
 *
 * Initializes @skb header fields as follows:
 *
 *    - skb->mac_header: the L2 header.
 *
 *    - skb->network_header: just past the L2 header, or just past the
 *      VLAN header, to the first byte of the L2 payload.
 *
 *    - skb->transport_header: If key->eth.type is ETH_P_IP or ETH_P_IPV6
 *      on output, then just past the IP header, if one is present and
 *      of a correct length, otherwise the same as skb->network_header.
 *      For other key->eth.type values it is left untouched.
 *
 *    - skb->protocol: the type of the data starting at skb->network_header.
 *      Equals to key->eth.type.
 *
 * Return: %0 if successful, otherwise a negative errno value.
 */
static int key_extract(struct sk_buff *skb, struct sw_flow_key *key)
{
        struct ethhdr *eth;

        /* Flags are always used as part of stats */
        key->tp.flags = 0;

        skb_reset_mac_header(skb);

        /* Link layer. */
        clear_vlan(key);
        if (ovs_key_mac_proto(key) == MAC_PROTO_NONE) {
                if (unlikely(eth_type_vlan(skb->protocol)))
                        return -EINVAL;

                skb_reset_network_header(skb);
                key->eth.type = skb->protocol;
        } else {
                eth = eth_hdr(skb);
                ether_addr_copy(key->eth.src, eth->h_source);
                ether_addr_copy(key->eth.dst, eth->h_dest);

                __skb_pull(skb, 2 * ETH_ALEN);
                /* We are going to push all headers that we pull, so no need to
                 * update skb->csum here.
                 */

                if (unlikely(parse_vlan(skb, key)))
                        return -ENOMEM;

                key->eth.type = parse_ethertype(skb);
                if (unlikely(key->eth.type == htons(0)))
                        return -ENOMEM;

                /* Multiple tagged packets need to retain TPID to satisfy
                 * skb_vlan_pop(), which will later shift the ethertype into
                 * skb->protocol.
                 */
                if (key->eth.cvlan.tci & htons(VLAN_CFI_MASK))
                        skb->protocol = key->eth.cvlan.tpid;
                else
                        skb->protocol = key->eth.type;

                skb_reset_network_header(skb);
                __skb_push(skb, skb->data - skb_mac_header(skb));
        }

        skb_reset_mac_len(skb);

        /* Fill out L3/L4 key info, if any */
        return key_extract_l3l4(skb, key);
}

/* In the case of conntrack fragment handling it expects L3 headers,
 * add a helper.
 */
int ovs_flow_key_update_l3l4(struct sk_buff *skb, struct sw_flow_key *key)
{
        return key_extract_l3l4(skb, key);
}

int ovs_flow_key_update(struct sk_buff *skb, struct sw_flow_key *key)
{
        int res;

        res = key_extract(skb, key);
        if (!res)
                key->mac_proto &= ~SW_FLOW_KEY_INVALID;

        return res;
}

static int key_extract_mac_proto(struct sk_buff *skb)
{
        switch (skb->dev->type) {
        case ARPHRD_ETHER:
                return MAC_PROTO_ETHERNET;
        case ARPHRD_NONE:
                if (skb->protocol == htons(ETH_P_TEB))
                        return MAC_PROTO_ETHERNET;
                return MAC_PROTO_NONE;
        }
        WARN_ON_ONCE(1);
        return -EINVAL;
}

int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
                         struct sk_buff *skb, struct sw_flow_key *key)
{
#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
        struct tc_skb_ext *tc_ext;
#endif
        bool post_ct = false, post_ct_snat = false, post_ct_dnat = false;
        int res, err;
        u16 zone = 0;

        /* Extract metadata from packet. */
        if (tun_info) {
                key->tun_proto = ip_tunnel_info_af(tun_info);
                memcpy(&key->tun_key, &tun_info->key, sizeof(key->tun_key));

                if (tun_info->options_len) {
                        BUILD_BUG_ON((1 << (sizeof(tun_info->options_len) *
                                                   8)) - 1
                                        > sizeof(key->tun_opts));

                        ip_tunnel_info_opts_get(TUN_METADATA_OPTS(key, tun_info->options_len),
                                                tun_info);
                        key->tun_opts_len = tun_info->options_len;
                } else {
                        key->tun_opts_len = 0;
                }
        } else  {
                key->tun_proto = 0;
                key->tun_opts_len = 0;
                memset(&key->tun_key, 0, sizeof(key->tun_key));
        }

        key->phy.priority = skb->priority;
        key->phy.in_port = OVS_CB(skb)->input_vport->port_no;
        key->phy.skb_mark = skb->mark;
        key->ovs_flow_hash = 0;
        res = key_extract_mac_proto(skb);
        if (res < 0)
                return res;
        key->mac_proto = res;

#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
        if (tc_skb_ext_tc_enabled()) {
                tc_ext = skb_ext_find(skb, TC_SKB_EXT);
                key->recirc_id = tc_ext && !tc_ext->act_miss ?
                                 tc_ext->chain : 0;
                OVS_CB(skb)->mru = tc_ext ? tc_ext->mru : 0;
                post_ct = tc_ext ? tc_ext->post_ct : false;
                post_ct_snat = post_ct ? tc_ext->post_ct_snat : false;
                post_ct_dnat = post_ct ? tc_ext->post_ct_dnat : false;
                zone = post_ct ? tc_ext->zone : 0;
        } else {
                key->recirc_id = 0;
        }
#else
        key->recirc_id = 0;
#endif

        err = key_extract(skb, key);
        if (!err) {
                ovs_ct_fill_key(skb, key, post_ct);   /* Must be after key_extract(). */
                if (post_ct) {
                        if (!skb_get_nfct(skb)) {
                                key->ct_zone = zone;
                        } else {
                                if (!post_ct_dnat)
                                        key->ct_state &= ~OVS_CS_F_DST_NAT;
                                if (!post_ct_snat)
                                        key->ct_state &= ~OVS_CS_F_SRC_NAT;
                        }
                }
        }
        return err;
}

int ovs_flow_key_extract_userspace(struct net *net, const struct nlattr *attr,
                                   struct sk_buff *skb,
                                   struct sw_flow_key *key, bool log)
{
        const struct nlattr *a[OVS_KEY_ATTR_MAX + 1];
        u64 attrs = 0;
        int err;

        err = parse_flow_nlattrs(attr, a, &attrs, log);
        if (err)
                return -EINVAL;

        /* Extract metadata from netlink attributes. */
        err = ovs_nla_get_flow_metadata(net, a, attrs, key, log);
        if (err)
                return err;

        /* key_extract assumes that skb->protocol is set-up for
         * layer 3 packets which is the case for other callers,
         * in particular packets received from the network stack.
         * Here the correct value can be set from the metadata
         * extracted above.
         * For L2 packet key eth type would be zero. skb protocol
         * would be set to correct value later during key-extact.
         */

        skb->protocol = key->eth.type;
        err = key_extract(skb, key);
        if (err)
                return err;

        /* Check that we have conntrack original direction tuple metadata only
         * for packets for which it makes sense.  Otherwise the key may be
         * corrupted due to overlapping key fields.
         */
        if (attrs & (1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4) &&
            key->eth.type != htons(ETH_P_IP))
                return -EINVAL;
        if (attrs & (1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6) &&
            (key->eth.type != htons(ETH_P_IPV6) ||
             sw_flow_key_is_nd(key)))
                return -EINVAL;

        return 0;
}














































































    4 























    3 














    3 

    1 

    1 

    1 

    1 












    2 




    1 

    1 



















    1 
    1 































  562 
   47 

    1 


    1 

























  248 
   25 
    2 


    2 














    1 



    1 
    1 

    1 

































    1 

    1 



    7 
    1 














    3 

    1 
    3 

















    3 
    3 

    3 
























    2 
    2 
















    3 
    1 








    2 


    3 


    1 




    2 





    1 














    3 



    1 





    3 

























    3 
    2 






    3 


    1 























    1 


    9 


    4 

    3 
    1 







    1 


    2 


    2 

    2 


    3 







































   10 


   10 



    2 
    1 

    1 






    1 
























    2 











    2 


















    1 












    2 




    1 
















    1 





    2 






















































































































  252 

















    5 




























 1101 




 1100 


  248 



  562 



  252 


  224 

















    5 









    5 

































  422 


  422 

  109 






  421 










    2 







    4 











    4 
















































































    4 








    3 




    3 
    3 





























    4 













    4 






















    4 



    3 
    4 
    4 
    4 
    4 


    4 
    4 




    4 





    3 
    4 
    4 
    4 
    4 


    4 
    4 
































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
// SPDX-License-Identifier: GPL-2.0
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  Generic netlink support functions to configure an SMC-R PNET table
 *
 *  Copyright IBM Corp. 2016
 *
 *  Author(s):  Thomas Richter <tmricht@linux.vnet.ibm.com>
 */

#include <linux/module.h>
#include <linux/list.h>
#include <linux/ctype.h>
#include <linux/mutex.h>
#include <net/netlink.h>
#include <net/genetlink.h>

#include <uapi/linux/if.h>
#include <uapi/linux/smc.h>

#include <rdma/ib_verbs.h>

#include <net/netns/generic.h>
#include "smc_netns.h"

#include "smc_pnet.h"
#include "smc_ib.h"
#include "smc_ism.h"
#include "smc_core.h"

static struct net_device *__pnet_find_base_ndev(struct net_device *ndev);
static struct net_device *pnet_find_base_ndev(struct net_device *ndev);

static const struct nla_policy smc_pnet_policy[SMC_PNETID_MAX + 1] = {
        [SMC_PNETID_NAME] = {
                .type = NLA_NUL_STRING,
                .len = SMC_MAX_PNETID_LEN
        },
        [SMC_PNETID_ETHNAME] = {
                .type = NLA_NUL_STRING,
                .len = IFNAMSIZ - 1
        },
        [SMC_PNETID_IBNAME] = {
                .type = NLA_NUL_STRING,
                .len = IB_DEVICE_NAME_MAX - 1
        },
        [SMC_PNETID_IBPORT] = { .type = NLA_U8 }
};

static struct genl_family smc_pnet_nl_family;

enum smc_pnet_nametype {
        SMC_PNET_ETH        = 1,
        SMC_PNET_IB        = 2,
};

/* pnet entry stored in pnet table */
struct smc_pnetentry {
        struct list_head list;
        char pnet_name[SMC_MAX_PNETID_LEN + 1];
        enum smc_pnet_nametype type;
        union {
                struct {
                        char eth_name[IFNAMSIZ + 1];
                        struct net_device *ndev;
                        netdevice_tracker dev_tracker;
                };
                struct {
                        char ib_name[IB_DEVICE_NAME_MAX + 1];
                        u8 ib_port;
                };
        };
};

/* Check if the pnetid is set */
bool smc_pnet_is_pnetid_set(u8 *pnetid)
{
        if (pnetid[0] == 0 || pnetid[0] == _S)
                return false;
        return true;
}

/* Check if two given pnetids match */
static bool smc_pnet_match(u8 *pnetid1, u8 *pnetid2)
{
        int i;

        for (i = 0; i < SMC_MAX_PNETID_LEN; i++) {
                if ((pnetid1[i] == 0 || pnetid1[i] == _S) &&
                    (pnetid2[i] == 0 || pnetid2[i] == _S))
                        break;
                if (pnetid1[i] != pnetid2[i])
                        return false;
        }
        return true;
}

/* Remove a pnetid from the pnet table.
 */
static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct smc_ib_device *ibdev;
        struct smcd_dev *smcd;
        struct smc_net *sn;
        int rc = -ENOENT;
        int ibport;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        /* remove table entry */
        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist,
                                 list) {
                if (!pnet_name ||
                    smc_pnet_match(pnetelem->pnet_name, pnet_name)) {
                        list_del(&pnetelem->list);
                        if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev) {
                                netdev_put(pnetelem->ndev,
                                           &pnetelem->dev_tracker);
                                pr_warn_ratelimited("smc: net device %s "
                                                    "erased user defined "
                                                    "pnetid %.16s\n",
                                                    pnetelem->eth_name,
                                                    pnetelem->pnet_name);
                        }
                        kfree(pnetelem);
                        rc = 0;
                }
        }
        mutex_unlock(&pnettable->lock);

        /* if this is not the initial namespace, stop here */
        if (net != &init_net)
                return rc;

        /* remove ib devices */
        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) {
                        if (ibdev->pnetid_by_user[ibport] &&
                            (!pnet_name ||
                             smc_pnet_match(pnet_name,
                                            ibdev->pnetid[ibport]))) {
                                pr_warn_ratelimited("smc: ib device %s ibport "
                                                    "%d erased user defined "
                                                    "pnetid %.16s\n",
                                                    ibdev->ibdev->name,
                                                    ibport + 1,
                                                    ibdev->pnetid[ibport]);
                                memset(ibdev->pnetid[ibport], 0,
                                       SMC_MAX_PNETID_LEN);
                                ibdev->pnetid_by_user[ibport] = false;
                                rc = 0;
                        }
                }
        }
        mutex_unlock(&smc_ib_devices.mutex);
        /* remove smcd devices */
        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(smcd, &smcd_dev_list.list, list) {
                if (smcd->pnetid_by_user &&
                    (!pnet_name ||
                     smc_pnet_match(pnet_name, smcd->pnetid))) {
                        pr_warn_ratelimited("smc: smcd device %s "
                                            "erased user defined pnetid "
                                            "%.16s\n",
                                            dev_name(&smcd->dibs->dev),
                                            smcd->pnetid);
                        memset(smcd->pnetid, 0, SMC_MAX_PNETID_LEN);
                        smcd->pnetid_by_user = false;
                        rc = 0;
                }
        }
        mutex_unlock(&smcd_dev_list.mutex);
        return rc;
}

/* Add the reference to a given network device to the pnet table.
 */
static int smc_pnet_add_by_ndev(struct net_device *ndev)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && !pnetelem->ndev &&
                    !strncmp(pnetelem->eth_name, ndev->name, IFNAMSIZ)) {
                        netdev_hold(ndev, &pnetelem->dev_tracker, GFP_ATOMIC);
                        pnetelem->ndev = ndev;
                        rc = 0;
                        pr_warn_ratelimited("smc: adding net device %s with "
                                            "user defined pnetid %.16s\n",
                                            pnetelem->eth_name,
                                            pnetelem->pnet_name);
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

/* Remove the reference to a given network device from the pnet table.
 */
static int smc_pnet_remove_by_ndev(struct net_device *ndev)
{
        struct smc_pnetentry *pnetelem, *tmp_pe;
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev == ndev) {
                        netdev_put(pnetelem->ndev, &pnetelem->dev_tracker);
                        pnetelem->ndev = NULL;
                        rc = 0;
                        pr_warn_ratelimited("smc: removing net device %s with "
                                            "user defined pnetid %.16s\n",
                                            pnetelem->eth_name,
                                            pnetelem->pnet_name);
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

/* Apply pnetid to ib device when no pnetid is set.
 */
static bool smc_pnet_apply_ib(struct smc_ib_device *ib_dev, u8 ib_port,
                              char *pnet_name)
{
        bool applied = false;

        mutex_lock(&smc_ib_devices.mutex);
        if (!smc_pnet_is_pnetid_set(ib_dev->pnetid[ib_port - 1])) {
                memcpy(ib_dev->pnetid[ib_port - 1], pnet_name,
                       SMC_MAX_PNETID_LEN);
                ib_dev->pnetid_by_user[ib_port - 1] = true;
                applied = true;
        }
        mutex_unlock(&smc_ib_devices.mutex);
        return applied;
}

/* Apply pnetid to smcd device when no pnetid is set.
 */
static bool smc_pnet_apply_smcd(struct smcd_dev *smcd_dev, char *pnet_name)
{
        bool applied = false;

        mutex_lock(&smcd_dev_list.mutex);
        if (!smc_pnet_is_pnetid_set(smcd_dev->pnetid)) {
                memcpy(smcd_dev->pnetid, pnet_name, SMC_MAX_PNETID_LEN);
                smcd_dev->pnetid_by_user = true;
                applied = true;
        }
        mutex_unlock(&smcd_dev_list.mutex);
        return applied;
}

/* The limit for pnetid is 16 characters.
 * Valid characters should be (single-byte character set) a-z, A-Z, 0-9.
 * Lower case letters are converted to upper case.
 * Interior blanks should not be used.
 */
static bool smc_pnetid_valid(const char *pnet_name, char *pnetid)
{
        char *bf = skip_spaces(pnet_name);
        size_t len = strlen(bf);
        char *end = bf + len;

        if (!len)
                return false;
        while (--end >= bf && isspace(*end))
                ;
        if (end - bf >= SMC_MAX_PNETID_LEN)
                return false;
        while (bf <= end) {
                if (!isalnum(*bf))
                        return false;
                *pnetid++ = islower(*bf) ? toupper(*bf) : *bf;
                bf++;
        }
        *pnetid = '\0';
        return true;
}

/* Find an infiniband device by a given name. The device might not exist. */
static struct smc_ib_device *smc_pnet_find_ib(char *ib_name)
{
        struct smc_ib_device *ibdev;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                if (!strncmp(ibdev->ibdev->name, ib_name,
                             sizeof(ibdev->ibdev->name)) ||
                    (ibdev->ibdev->dev.parent &&
                     !strncmp(dev_name(ibdev->ibdev->dev.parent), ib_name,
                             IB_DEVICE_NAME_MAX - 1))) {
                        goto out;
                }
        }
        ibdev = NULL;
out:
        mutex_unlock(&smc_ib_devices.mutex);
        return ibdev;
}

/* Find an smcd device by a given name. The device might not exist. */
static struct smcd_dev *smc_pnet_find_smcd(char *smcd_name)
{
        struct smcd_dev *smcd_dev;

        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(smcd_dev, &smcd_dev_list.list, list) {
                if (!strncmp(dev_name(&smcd_dev->dibs->dev), smcd_name,
                             IB_DEVICE_NAME_MAX - 1) ||
                    (smcd_dev->dibs->dev.parent &&
                     !strncmp(dev_name(smcd_dev->dibs->dev.parent), smcd_name,
                              IB_DEVICE_NAME_MAX - 1)))
                        goto out;
        }
        smcd_dev = NULL;
out:
        mutex_unlock(&smcd_dev_list.mutex);
        return smcd_dev;
}

static int smc_pnet_add_eth(struct smc_pnettable *pnettable, struct net *net,
                            char *eth_name, char *pnet_name)
{
        struct smc_pnetentry *tmp_pe, *new_pe;
        struct net_device *ndev, *base_ndev;
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        bool new_netdev;
        int rc;

        /* check if (base) netdev already has a pnetid. If there is one, we do
         * not want to add a pnet table entry
         */
        rc = -EEXIST;
        ndev = dev_get_by_name(net, eth_name);        /* dev_hold() */
        if (ndev) {
                base_ndev = pnet_find_base_ndev(ndev);
                if (!smc_pnetid_by_dev_port(base_ndev->dev.parent,
                                            base_ndev->dev_port, ndev_pnetid))
                        goto out_put;
        }

        /* add a new netdev entry to the pnet table if there isn't one */
        rc = -ENOMEM;
        new_pe = kzalloc(sizeof(*new_pe), GFP_KERNEL);
        if (!new_pe)
                goto out_put;
        new_pe->type = SMC_PNET_ETH;
        memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN);
        strscpy(new_pe->eth_name, eth_name);
        rc = -EEXIST;
        new_netdev = true;
        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_ETH &&
                    !strncmp(tmp_pe->eth_name, eth_name, IFNAMSIZ)) {
                        new_netdev = false;
                        break;
                }
        }
        if (new_netdev) {
                if (ndev) {
                        new_pe->ndev = ndev;
                        netdev_tracker_alloc(ndev, &new_pe->dev_tracker,
                                             GFP_ATOMIC);
                }
                list_add_tail(&new_pe->list, &pnettable->pnetlist);
                mutex_unlock(&pnettable->lock);
        } else {
                mutex_unlock(&pnettable->lock);
                kfree(new_pe);
                goto out_put;
        }
        if (ndev)
                pr_warn_ratelimited("smc: net device %s "
                                    "applied user defined pnetid %.16s\n",
                                    new_pe->eth_name, new_pe->pnet_name);
        return 0;

out_put:
        dev_put(ndev);
        return rc;
}

static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
                           u8 ib_port, char *pnet_name)
{
        struct smc_pnetentry *tmp_pe, *new_pe;
        struct smc_ib_device *ib_dev;
        bool smcddev_applied = true;
        bool ibdev_applied = true;
        struct smcd_dev *smcd;
        bool new_ibdev;

        /* try to apply the pnetid to active devices */
        ib_dev = smc_pnet_find_ib(ib_name);
        if (ib_dev) {
                ibdev_applied = smc_pnet_apply_ib(ib_dev, ib_port, pnet_name);
                if (ibdev_applied)
                        pr_warn_ratelimited("smc: ib device %s ibport %d "
                                            "applied user defined pnetid "
                                            "%.16s\n", ib_dev->ibdev->name,
                                            ib_port,
                                            ib_dev->pnetid[ib_port - 1]);
        }
        smcd = smc_pnet_find_smcd(ib_name);
        if (smcd) {
                smcddev_applied = smc_pnet_apply_smcd(smcd, pnet_name);
                if (smcddev_applied) {
                        pr_warn_ratelimited("smc: smcd device %s applied user defined pnetid %.16s\n",
                                            dev_name(&smcd->dibs->dev),
                                            smcd->pnetid);
                }
        }
        /* Apply fails when a device has a hardware-defined pnetid set, do not
         * add a pnet table entry in that case.
         */
        if (!ibdev_applied || !smcddev_applied)
                return -EEXIST;

        /* add a new ib entry to the pnet table if there isn't one */
        new_pe = kzalloc(sizeof(*new_pe), GFP_KERNEL);
        if (!new_pe)
                return -ENOMEM;
        new_pe->type = SMC_PNET_IB;
        memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN);
        strscpy(new_pe->ib_name, ib_name);
        new_pe->ib_port = ib_port;

        new_ibdev = true;
        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
                        new_ibdev = false;
                        break;
                }
        }
        if (new_ibdev) {
                list_add_tail(&new_pe->list, &pnettable->pnetlist);
                mutex_unlock(&pnettable->lock);
        } else {
                mutex_unlock(&pnettable->lock);
                kfree(new_pe);
        }
        return (new_ibdev) ? 0 : -EEXIST;
}

/* Append a pnetid to the end of the pnet table if not already on this list.
 */
static int smc_pnet_enter(struct net *net, struct nlattr *tb[])
{
        char pnet_name[SMC_MAX_PNETID_LEN + 1];
        struct smc_pnettable *pnettable;
        bool new_netdev = false;
        bool new_ibdev = false;
        struct smc_net *sn;
        u8 ibport = 1;
        char *string;
        int rc;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        rc = -EINVAL;
        if (!tb[SMC_PNETID_NAME])
                goto error;
        string = (char *)nla_data(tb[SMC_PNETID_NAME]);
        if (!smc_pnetid_valid(string, pnet_name))
                goto error;

        if (tb[SMC_PNETID_ETHNAME]) {
                string = (char *)nla_data(tb[SMC_PNETID_ETHNAME]);
                rc = smc_pnet_add_eth(pnettable, net, string, pnet_name);
                if (!rc)
                        new_netdev = true;
                else if (rc != -EEXIST)
                        goto error;
        }

        /* if this is not the initial namespace, stop here */
        if (net != &init_net)
                return new_netdev ? 0 : -EEXIST;

        rc = -EINVAL;
        if (tb[SMC_PNETID_IBNAME]) {
                string = (char *)nla_data(tb[SMC_PNETID_IBNAME]);
                string = strim(string);
                if (tb[SMC_PNETID_IBPORT]) {
                        ibport = nla_get_u8(tb[SMC_PNETID_IBPORT]);
                        if (ibport < 1 || ibport > SMC_MAX_PORTS)
                                goto error;
                }
                rc = smc_pnet_add_ib(pnettable, string, ibport, pnet_name);
                if (!rc)
                        new_ibdev = true;
                else if (rc != -EEXIST)
                        goto error;
        }
        return (new_netdev || new_ibdev) ? 0 : -EEXIST;

error:
        return rc;
}

/* Convert an smc_pnetentry to a netlink attribute sequence */
static int smc_pnet_set_nla(struct sk_buff *msg,
                            struct smc_pnetentry *pnetelem)
{
        if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name))
                return -1;
        if (pnetelem->type == SMC_PNET_ETH) {
                if (nla_put_string(msg, SMC_PNETID_ETHNAME,
                                   pnetelem->eth_name))
                        return -1;
        } else {
                if (nla_put_string(msg, SMC_PNETID_ETHNAME, "n/a"))
                        return -1;
        }
        if (pnetelem->type == SMC_PNET_IB) {
                if (nla_put_string(msg, SMC_PNETID_IBNAME, pnetelem->ib_name) ||
                    nla_put_u8(msg, SMC_PNETID_IBPORT, pnetelem->ib_port))
                        return -1;
        } else {
                if (nla_put_string(msg, SMC_PNETID_IBNAME, "n/a") ||
                    nla_put_u8(msg, SMC_PNETID_IBPORT, 0xff))
                        return -1;
        }

        return 0;
}

static int smc_pnet_add(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        return smc_pnet_enter(net, info->attrs);
}

static int smc_pnet_del(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        if (!info->attrs[SMC_PNETID_NAME])
                return -EINVAL;
        return smc_pnet_remove_by_pnetid(net,
                                (char *)nla_data(info->attrs[SMC_PNETID_NAME]));
}

static int smc_pnet_dump_start(struct netlink_callback *cb)
{
        cb->args[0] = 0;
        return 0;
}

static int smc_pnet_dumpinfo(struct sk_buff *skb,
                             u32 portid, u32 seq, u32 flags,
                             struct smc_pnetentry *pnetelem)
{
        void *hdr;

        hdr = genlmsg_put(skb, portid, seq, &smc_pnet_nl_family,
                          flags, SMC_PNETID_GET);
        if (!hdr)
                return -ENOMEM;
        if (smc_pnet_set_nla(skb, pnetelem) < 0) {
                genlmsg_cancel(skb, hdr);
                return -EMSGSIZE;
        }
        genlmsg_end(skb, hdr);
        return 0;
}

static int _smc_pnet_dump(struct net *net, struct sk_buff *skb, u32 portid,
                          u32 seq, u8 *pnetid, int start_idx)
{
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *pnetelem;
        struct smc_net *sn;
        int idx = 0;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        /* dump pnettable entries */
        mutex_lock(&pnettable->lock);
        list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
                if (pnetid && !smc_pnet_match(pnetelem->pnet_name, pnetid))
                        continue;
                if (idx++ < start_idx)
                        continue;
                /* if this is not the initial namespace, dump only netdev */
                if (net != &init_net && pnetelem->type != SMC_PNET_ETH)
                        continue;
                if (smc_pnet_dumpinfo(skb, portid, seq, NLM_F_MULTI,
                                      pnetelem)) {
                        --idx;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return idx;
}

static int smc_pnet_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        int idx;

        idx = _smc_pnet_dump(net, skb, NETLINK_CB(cb->skb).portid,
                             cb->nlh->nlmsg_seq, NULL, cb->args[0]);

        cb->args[0] = idx;
        return skb->len;
}

/* Retrieve one PNETID entry */
static int smc_pnet_get(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct sk_buff *msg;
        void *hdr;

        if (!info->attrs[SMC_PNETID_NAME])
                return -EINVAL;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        _smc_pnet_dump(net, msg, info->snd_portid, info->snd_seq,
                       nla_data(info->attrs[SMC_PNETID_NAME]), 0);

        /* finish multi part message and send it */
        hdr = nlmsg_put(msg, info->snd_portid, info->snd_seq, NLMSG_DONE, 0,
                        NLM_F_MULTI);
        if (!hdr) {
                nlmsg_free(msg);
                return -EMSGSIZE;
        }
        return genlmsg_reply(msg, info);
}

/* Remove and delete all pnetids from pnet table.
 */
static int smc_pnet_flush(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);

        smc_pnet_remove_by_pnetid(net, NULL);
        return 0;
}

/* SMC_PNETID generic netlink operation definition */
static const struct genl_ops smc_pnet_ops[] = {
        {
                .cmd = SMC_PNETID_GET,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                /* can be retrieved by unprivileged users */
                .doit = smc_pnet_get,
                .dumpit = smc_pnet_dump,
                .start = smc_pnet_dump_start
        },
        {
                .cmd = SMC_PNETID_ADD,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_add
        },
        {
                .cmd = SMC_PNETID_DEL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_del
        },
        {
                .cmd = SMC_PNETID_FLUSH,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_ADMIN_PERM,
                .doit = smc_pnet_flush
        }
};

/* SMC_PNETID family definition */
static struct genl_family smc_pnet_nl_family __ro_after_init = {
        .hdrsize = 0,
        .name = SMCR_GENL_FAMILY_NAME,
        .version = SMCR_GENL_FAMILY_VERSION,
        .maxattr = SMC_PNETID_MAX,
        .policy = smc_pnet_policy,
        .netnsok = true,
        .module = THIS_MODULE,
        .ops = smc_pnet_ops,
        .n_ops =  ARRAY_SIZE(smc_pnet_ops),
        .resv_start_op = SMC_PNETID_FLUSH + 1,
};

bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe;
        bool rc = false;

        read_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry(pe, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pe->pnetid)) {
                        rc = true;
                        goto unlock;
                }
        }

unlock:
        read_unlock(&sn->pnetids_ndev.lock);
        return rc;
}

static int smc_pnet_add_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *pi;

        pe = kzalloc(sizeof(*pe), GFP_KERNEL);
        if (!pe)
                return -ENOMEM;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry(pi, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pi->pnetid)) {
                        refcount_inc(&pi->refcnt);
                        kfree(pe);
                        goto unlock;
                }
        }
        refcount_set(&pe->refcnt, 1);
        memcpy(pe->pnetid, pnetid, SMC_MAX_PNETID_LEN);
        list_add_tail(&pe->list, &sn->pnetids_ndev.list);

unlock:
        write_unlock(&sn->pnetids_ndev.lock);
        return 0;
}

static void smc_pnet_remove_pnetid(struct net *net, u8 *pnetid)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *pe2;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry_safe(pe, pe2, &sn->pnetids_ndev.list, list) {
                if (smc_pnet_match(pnetid, pe->pnetid)) {
                        if (refcount_dec_and_test(&pe->refcnt)) {
                                list_del(&pe->list);
                                kfree(pe);
                        }
                        break;
                }
        }
        write_unlock(&sn->pnetids_ndev.lock);
}

static void smc_pnet_add_base_pnetid(struct net *net, struct net_device *dev,
                                     u8 *ndev_pnetid)
{
        struct net_device *base_dev;

        base_dev = __pnet_find_base_ndev(dev);
        if (base_dev->flags & IFF_UP &&
            !smc_pnetid_by_dev_port(base_dev->dev.parent, base_dev->dev_port,
                                    ndev_pnetid)) {
                /* add to PNETIDs list */
                smc_pnet_add_pnetid(net, ndev_pnetid);
        }
}

/* create initial list of netdevice pnetids */
static void smc_pnet_create_pnetids_list(struct net *net)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct net_device *dev;

        /* Newly created netns do not have devices.
         * Do not even acquire rtnl.
         */
        if (list_empty(&net->dev_base_head))
                return;

        /* Note: This might not be needed, because smc_pnet_netdev_event()
         * is also calling smc_pnet_add_base_pnetid() when handling
         * NETDEV_UP event.
         */
        rtnl_lock();
        for_each_netdev(net, dev)
                smc_pnet_add_base_pnetid(net, dev, ndev_pnetid);
        rtnl_unlock();
}

/* clean up list of netdevice pnetids */
static void smc_pnet_destroy_pnetids_list(struct net *net)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnetids_ndev_entry *pe, *temp_pe;

        write_lock(&sn->pnetids_ndev.lock);
        list_for_each_entry_safe(pe, temp_pe, &sn->pnetids_ndev.list, list) {
                list_del(&pe->list);
                kfree(pe);
        }
        write_unlock(&sn->pnetids_ndev.lock);
}

static int smc_pnet_netdev_event(struct notifier_block *this,
                                 unsigned long event, void *ptr)
{
        struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(event_dev);
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];

        switch (event) {
        case NETDEV_REBOOT:
        case NETDEV_UNREGISTER:
                smc_pnet_remove_by_ndev(event_dev);
                smc_ib_ndev_change(event_dev, event);
                return NOTIFY_OK;
        case NETDEV_REGISTER:
                smc_pnet_add_by_ndev(event_dev);
                smc_ib_ndev_change(event_dev, event);
                return NOTIFY_OK;
        case NETDEV_UP:
                smc_pnet_add_base_pnetid(net, event_dev, ndev_pnetid);
                return NOTIFY_OK;
        case NETDEV_DOWN:
                event_dev = __pnet_find_base_ndev(event_dev);
                if (!smc_pnetid_by_dev_port(event_dev->dev.parent,
                                            event_dev->dev_port, ndev_pnetid)) {
                        /* remove from PNETIDs list */
                        smc_pnet_remove_pnetid(net, ndev_pnetid);
                }
                return NOTIFY_OK;
        default:
                return NOTIFY_DONE;
        }
}

static struct notifier_block smc_netdev_notifier = {
        .notifier_call = smc_pnet_netdev_event
};

/* init network namespace */
int smc_pnet_net_init(struct net *net)
{
        struct smc_net *sn = net_generic(net, smc_net_id);
        struct smc_pnettable *pnettable = &sn->pnettable;
        struct smc_pnetids_ndev *pnetids_ndev = &sn->pnetids_ndev;

        INIT_LIST_HEAD(&pnettable->pnetlist);
        mutex_init(&pnettable->lock);
        INIT_LIST_HEAD(&pnetids_ndev->list);
        rwlock_init(&pnetids_ndev->lock);

        smc_pnet_create_pnetids_list(net);

        return 0;
}

int __init smc_pnet_init(void)
{
        int rc;

        rc = genl_register_family(&smc_pnet_nl_family);
        if (rc)
                return rc;
        rc = register_netdevice_notifier(&smc_netdev_notifier);
        if (rc)
                genl_unregister_family(&smc_pnet_nl_family);

        return rc;
}

/* exit network namespace */
void smc_pnet_net_exit(struct net *net)
{
        /* flush pnet table */
        smc_pnet_remove_by_pnetid(net, NULL);
        smc_pnet_destroy_pnetids_list(net);
}

void smc_pnet_exit(void)
{
        unregister_netdevice_notifier(&smc_netdev_notifier);
        genl_unregister_family(&smc_pnet_nl_family);
}

static struct net_device *__pnet_find_base_ndev(struct net_device *ndev)
{
        int i, nest_lvl;

        ASSERT_RTNL();
        nest_lvl = ndev->lower_level;
        for (i = 0; i < nest_lvl; i++) {
                struct list_head *lower = &ndev->adj_list.lower;

                if (list_empty(lower))
                        break;
                lower = lower->next;
                ndev = netdev_lower_get_next(ndev, &lower);
        }
        return ndev;
}

/* Determine one base device for stacked net devices.
 * If the lower device level contains more than one devices
 * (for instance with bonding slaves), just the first device
 * is used to reach a base device.
 */
static struct net_device *pnet_find_base_ndev(struct net_device *ndev)
{
        rtnl_lock();
        ndev = __pnet_find_base_ndev(ndev);
        rtnl_unlock();
        return ndev;
}

static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *ndev,
                                              u8 *pnetid)
{
        struct smc_pnettable *pnettable;
        struct net *net = dev_net(ndev);
        struct smc_pnetentry *pnetelem;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for namespace */
        sn = net_generic(net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(pnetelem, &pnettable->pnetlist, list) {
                if (pnetelem->type == SMC_PNET_ETH && ndev == pnetelem->ndev) {
                        /* get pnetid of netdev device */
                        memcpy(pnetid, pnetelem->pnet_name, SMC_MAX_PNETID_LEN);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);
        return rc;
}

static int smc_pnet_determine_gid(struct smc_ib_device *ibdev, int i,
                                  struct smc_init_info *ini)
{
        if (!ini->check_smcrv2 &&
            !smc_ib_determine_gid(ibdev, i, ini->vlan_id, ini->ib_gid, NULL,
                                  NULL)) {
                ini->ib_dev = ibdev;
                ini->ib_port = i;
                return 0;
        }
        if (ini->check_smcrv2 &&
            !smc_ib_determine_gid(ibdev, i, ini->vlan_id, ini->smcrv2.ib_gid_v2,
                                  NULL, &ini->smcrv2)) {
                ini->smcrv2.ib_dev_v2 = ibdev;
                ini->smcrv2.ib_port_v2 = i;
                return 0;
        }
        return -ENODEV;
}

/* find a roce device for the given pnetid */
static void _smc_pnet_find_roce_by_pnetid(u8 *pnet_id,
                                          struct smc_init_info *ini,
                                          struct smc_ib_device *known_dev,
                                          struct net *net)
{
        struct smc_ib_device *ibdev;
        int i;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                if (ibdev == known_dev ||
                    !rdma_dev_access_netns(ibdev->ibdev, net))
                        continue;
                for (i = 1; i <= SMC_MAX_PORTS; i++) {
                        if (!rdma_is_port_valid(ibdev->ibdev, i))
                                continue;
                        if (smc_pnet_match(ibdev->pnetid[i - 1], pnet_id) &&
                            smc_ib_port_active(ibdev, i) &&
                            !test_bit(i - 1, ibdev->ports_going_away)) {
                                if (!smc_pnet_determine_gid(ibdev, i, ini))
                                        goto out;
                        }
                }
        }
out:
        mutex_unlock(&smc_ib_devices.mutex);
}

/* find alternate roce device with same pnet_id, vlan_id and net namespace */
void smc_pnet_find_alt_roce(struct smc_link_group *lgr,
                            struct smc_init_info *ini,
                            struct smc_ib_device *known_dev)
{
        struct net *net = lgr->net;

        _smc_pnet_find_roce_by_pnetid(lgr->pnet_id, ini, known_dev, net);
}

/* if handshake network device belongs to a roce device, return its
 * IB device and port
 */
static void smc_pnet_find_rdma_dev(struct net_device *netdev,
                                   struct smc_init_info *ini)
{
        struct net *net = dev_net(netdev);
        struct smc_ib_device *ibdev;

        mutex_lock(&smc_ib_devices.mutex);
        list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
                struct net_device *ndev;
                int i;

                /* check rdma net namespace */
                if (!rdma_dev_access_netns(ibdev->ibdev, net))
                        continue;

                for (i = 1; i <= SMC_MAX_PORTS; i++) {
                        if (!rdma_is_port_valid(ibdev->ibdev, i))
                                continue;
                        ndev = ib_device_get_netdev(ibdev->ibdev, i);
                        if (!ndev)
                                continue;
                        dev_put(ndev);
                        if (netdev == ndev &&
                            smc_ib_port_active(ibdev, i) &&
                            !test_bit(i - 1, ibdev->ports_going_away)) {
                                if (!smc_pnet_determine_gid(ibdev, i, ini))
                                        break;
                        }
                }
        }
        mutex_unlock(&smc_ib_devices.mutex);
}

/* Determine the corresponding IB device port based on the hardware PNETID.
 * Searching stops at the first matching active IB device port with vlan_id
 * configured.
 * If nothing found, check pnetid table.
 * If nothing found, try to use handshake device
 */
static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,
                                         struct smc_init_info *ini)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct net_device *base_ndev;
        struct net *net;

        base_ndev = pnet_find_base_ndev(ndev);
        net = dev_net(ndev);
        if (smc_pnetid_by_dev_port(base_ndev->dev.parent, base_ndev->dev_port,
                                   ndev_pnetid) &&
            smc_pnet_find_ndev_pnetid_by_table(base_ndev, ndev_pnetid) &&
            smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid)) {
                smc_pnet_find_rdma_dev(base_ndev, ini);
                return; /* pnetid could not be determined */
        }
        _smc_pnet_find_roce_by_pnetid(ndev_pnetid, ini, NULL, net);
}

static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
                                        struct smc_init_info *ini)
{
        u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
        struct smcd_dev *ismdev;

        ndev = pnet_find_base_ndev(ndev);
        if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port,
                                   ndev_pnetid) &&
            smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid))
                return; /* pnetid could not be determined */

        mutex_lock(&smcd_dev_list.mutex);
        list_for_each_entry(ismdev, &smcd_dev_list.list, list) {
                if (smc_pnet_match(ismdev->pnetid, ndev_pnetid) &&
                    !ismdev->going_away &&
                    (!ini->ism_peer_gid[0].gid ||
                     !smc_ism_cantalk(&ini->ism_peer_gid[0], ini->vlan_id,
                                      ismdev))) {
                        ini->ism_dev[0] = ismdev;
                        break;
                }
        }
        mutex_unlock(&smcd_dev_list.mutex);
}

/* PNET table analysis for a given sock:
 * determine ib_device and port belonging to used internal TCP socket
 * ethernet interface.
 */
void smc_pnet_find_roce_resource(struct sock *sk, struct smc_init_info *ini)
{
        struct net_device *dev;
        struct dst_entry *dst;

        rcu_read_lock();
        dst = __sk_dst_get(sk);
        dev = dst ? dst_dev_rcu(dst) : NULL;
        dev_hold(dev);
        rcu_read_unlock();

        if (dev) {
                smc_pnet_find_roce_by_pnetid(dev, ini);
                dev_put(dev);
        }
}

void smc_pnet_find_ism_resource(struct sock *sk, struct smc_init_info *ini)
{
        struct net_device *dev;
        struct dst_entry *dst;

        ini->ism_dev[0] = NULL;

        rcu_read_lock();
        dst = __sk_dst_get(sk);
        dev = dst ? dst_dev_rcu(dst) : NULL;
        dev_hold(dev);
        rcu_read_unlock();

        if (dev) {
                smc_pnet_find_ism_by_pnetid(dev, ini);
                dev_put(dev);
        }
}

/* Lookup and apply a pnet table entry to the given ib device.
 */
int smc_pnetid_by_table_ib(struct smc_ib_device *smcibdev, u8 ib_port)
{
        char *ib_name = smcibdev->ibdev->name;
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *tmp_pe;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for init namespace */
        sn = net_generic(&init_net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX) &&
                    tmp_pe->ib_port == ib_port) {
                        smc_pnet_apply_ib(smcibdev, ib_port, tmp_pe->pnet_name);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);

        return rc;
}

/* Lookup and apply a pnet table entry to the given smcd device.
 */
int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
{
        struct smc_pnettable *pnettable;
        struct smc_pnetentry *tmp_pe;
        struct smc_net *sn;
        int rc = -ENOENT;

        /* get pnettable for init namespace */
        sn = net_generic(&init_net, smc_net_id);
        pnettable = &sn->pnettable;

        mutex_lock(&pnettable->lock);
        list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
                if (tmp_pe->type == SMC_PNET_IB &&
                    (!strncmp(tmp_pe->ib_name,
                               dev_name(&smcddev->dibs->dev),
                               sizeof(tmp_pe->ib_name)) ||
                     (smcddev->dibs->dev.parent &&
                      !strncmp(tmp_pe->ib_name,
                               dev_name(smcddev->dibs->dev.parent),
                               sizeof(tmp_pe->ib_name))))) {
                        smc_pnet_apply_smcd(smcddev, tmp_pe->pnet_name);
                        rc = 0;
                        break;
                }
        }
        mutex_unlock(&pnettable->lock);

        return rc;
}























    1 





    2 





    1 


    1 






















    1 





    2 











    1 











    1 



    1 






































    2 
    1 


    1 
























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
// SPDX-License-Identifier: GPL-2.0
/* net/atm/pvc.c - ATM PVC sockets */

/* Written 1995-2000 by Werner Almesberger, EPFL LRC/ICA */


#include <linux/net.h>                /* struct socket, struct proto_ops */
#include <linux/atm.h>                /* ATM stuff */
#include <linux/atmdev.h>        /* ATM devices */
#include <linux/errno.h>        /* error codes */
#include <linux/kernel.h>        /* printk */
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/bitops.h>
#include <linux/export.h>
#include <net/sock.h>                /* for sock_no_* */

#include "resources.h"                /* devs and vccs */
#include "common.h"                /* common for PVCs and SVCs */


static int pvc_shutdown(struct socket *sock, int how)
{
        return 0;
}

static int pvc_bind(struct socket *sock, struct sockaddr *sockaddr,
                    int sockaddr_len)
{
        struct sock *sk = sock->sk;
        struct sockaddr_atmpvc *addr;
        struct atm_vcc *vcc;
        int error;

        if (sockaddr_len != sizeof(struct sockaddr_atmpvc))
                return -EINVAL;
        addr = (struct sockaddr_atmpvc *)sockaddr;
        if (addr->sap_family != AF_ATMPVC)
                return -EAFNOSUPPORT;
        lock_sock(sk);
        vcc = ATM_SD(sock);
        if (!test_bit(ATM_VF_HASQOS, &vcc->flags)) {
                error = -EBADFD;
                goto out;
        }
        if (test_bit(ATM_VF_PARTIAL, &vcc->flags)) {
                if (vcc->vpi != ATM_VPI_UNSPEC)
                        addr->sap_addr.vpi = vcc->vpi;
                if (vcc->vci != ATM_VCI_UNSPEC)
                        addr->sap_addr.vci = vcc->vci;
        }
        error = vcc_connect(sock, addr->sap_addr.itf, addr->sap_addr.vpi,
                            addr->sap_addr.vci);
out:
        release_sock(sk);
        return error;
}

static int pvc_connect(struct socket *sock, struct sockaddr *sockaddr,
                       int sockaddr_len, int flags)
{
        return pvc_bind(sock, sockaddr, sockaddr_len);
}

static int pvc_setsockopt(struct socket *sock, int level, int optname,
                          sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        int error;

        lock_sock(sk);
        error = vcc_setsockopt(sock, level, optname, optval, optlen);
        release_sock(sk);
        return error;
}

static int pvc_getsockopt(struct socket *sock, int level, int optname,
                          char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        int error;

        lock_sock(sk);
        error = vcc_getsockopt(sock, level, optname, optval, optlen);
        release_sock(sk);
        return error;
}

static int pvc_getname(struct socket *sock, struct sockaddr *sockaddr,
                       int peer)
{
        struct sockaddr_atmpvc *addr;
        struct atm_vcc *vcc = ATM_SD(sock);

        if (!vcc->dev || !test_bit(ATM_VF_ADDR, &vcc->flags))
                return -ENOTCONN;
        addr = (struct sockaddr_atmpvc *)sockaddr;
        memset(addr, 0, sizeof(*addr));
        addr->sap_family = AF_ATMPVC;
        addr->sap_addr.itf = vcc->dev->number;
        addr->sap_addr.vpi = vcc->vpi;
        addr->sap_addr.vci = vcc->vci;
        return sizeof(struct sockaddr_atmpvc);
}

static const struct proto_ops pvc_proto_ops = {
        .family =        PF_ATMPVC,
        .owner =        THIS_MODULE,

        .release =        vcc_release,
        .bind =                pvc_bind,
        .connect =        pvc_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        sock_no_accept,
        .getname =        pvc_getname,
        .poll =                vcc_poll,
        .ioctl =        vcc_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = vcc_compat_ioctl,
#endif
        .gettstamp =        sock_gettstamp,
        .listen =        sock_no_listen,
        .shutdown =        pvc_shutdown,
        .setsockopt =        pvc_setsockopt,
        .getsockopt =        pvc_getsockopt,
        .sendmsg =        vcc_sendmsg,
        .recvmsg =        vcc_recvmsg,
        .mmap =                sock_no_mmap,
};


static int pvc_create(struct net *net, struct socket *sock, int protocol,
                      int kern)
{
        if (net != &init_net)
                return -EAFNOSUPPORT;

        sock->ops = &pvc_proto_ops;
        return vcc_create(net, sock, protocol, PF_ATMPVC, kern);
}

static const struct net_proto_family pvc_family_ops = {
        .family = PF_ATMPVC,
        .create = pvc_create,
        .owner = THIS_MODULE,
};


/*
 *        Initialize the ATM PVC protocol family
 */


int __init atmpvc_init(void)
{
        return sock_register(&pvc_family_ops);
}

void atmpvc_exit(void)
{
        sock_unregister(PF_ATMPVC);
}























































    2 


















    1 








    1 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * IPVS:        Never Queue scheduling module
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *
 * Changes:
 */

/*
 * The NQ algorithm adopts a two-speed model. When there is an idle server
 * available, the job will be sent to the idle server, instead of waiting
 * for a fast one. When there is no idle server available, the job will be
 * sent to the server that minimize its expected delay (The Shortest
 * Expected Delay scheduling algorithm).
 *
 * See the following paper for more information:
 * A. Weinrib and S. Shenker, Greed is not enough: Adaptive load sharing
 * in large heterogeneous systems. In Proceedings IEEE INFOCOM'88,
 * pages 986-994, 1988.
 *
 * Thanks must go to Marko Buuri <marko@buuri.name> for talking NQ to me.
 *
 * The difference between NQ and SED is that NQ can improve overall
 * system utilization.
 *
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>

#include <net/ip_vs.h>


static inline int
ip_vs_nq_dest_overhead(struct ip_vs_dest *dest)
{
        /*
         * We only use the active connection number in the cost
         * calculation here.
         */
        return atomic_read(&dest->activeconns) + 1;
}


/*
 *        Weighted Least Connection scheduling
 */
static struct ip_vs_dest *
ip_vs_nq_schedule(struct ip_vs_service *svc, const struct sk_buff *skb,
                  struct ip_vs_iphdr *iph)
{
        struct ip_vs_dest *dest, *least = NULL;
        int loh = 0, doh;

        IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);

        /*
         * We calculate the load of each dest server as follows:
         *        (server expected overhead) / dest->weight
         *
         * Remember -- no floats in kernel mode!!!
         * The comparison of h1*w2 > h2*w1 is equivalent to that of
         *                  h1/w1 > h2/w2
         * if every weight is larger than zero.
         *
         * The server with weight=0 is quiesced and will not receive any
         * new connections.
         */

        list_for_each_entry_rcu(dest, &svc->destinations, n_list) {

                if (dest->flags & IP_VS_DEST_F_OVERLOAD ||
                    !atomic_read(&dest->weight))
                        continue;

                doh = ip_vs_nq_dest_overhead(dest);

                /* return the server directly if it is idle */
                if (atomic_read(&dest->activeconns) == 0) {
                        least = dest;
                        loh = doh;
                        goto out;
                }

                if (!least ||
                    ((__s64)loh * atomic_read(&dest->weight) >
                     (__s64)doh * atomic_read(&least->weight))) {
                        least = dest;
                        loh = doh;
                }
        }

        if (!least) {
                ip_vs_scheduler_err(svc, "no destination available");
                return NULL;
        }

  out:
        IP_VS_DBG_BUF(6, "NQ: server %s:%u "
                      "activeconns %d refcnt %d weight %d overhead %d\n",
                      IP_VS_DBG_ADDR(least->af, &least->addr),
                      ntohs(least->port),
                      atomic_read(&least->activeconns),
                      refcount_read(&least->refcnt),
                      atomic_read(&least->weight), loh);

        return least;
}


static struct ip_vs_scheduler ip_vs_nq_scheduler =
{
        .name =                        "nq",
        .refcnt =                ATOMIC_INIT(0),
        .module =                THIS_MODULE,
        .n_list =                LIST_HEAD_INIT(ip_vs_nq_scheduler.n_list),
        .schedule =                ip_vs_nq_schedule,
};


static int __init ip_vs_nq_init(void)
{
        return register_ip_vs_scheduler(&ip_vs_nq_scheduler);
}

static void __exit ip_vs_nq_cleanup(void)
{
        unregister_ip_vs_scheduler(&ip_vs_nq_scheduler);
        synchronize_rcu();
}

module_init(ip_vs_nq_init);
module_exit(ip_vs_nq_cleanup);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("ipvs never queue scheduler");






















    3 





    3 
    1 

    3 






    2 


    1 

    1 






    3 






    2 
    1 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
// SPDX-License-Identifier: GPL-2.0-only
#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <asm/string.h>
#include <linux/kmod.h>
#include <linux/sysctl.h>

#include <net/ip_vs.h>

/* IPVS pe list */
static LIST_HEAD(ip_vs_pe);

/* semaphore for IPVS PEs. */
static DEFINE_MUTEX(ip_vs_pe_mutex);

/* Get pe in the pe list by name */
struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name)
{
        struct ip_vs_pe *pe;

        IP_VS_DBG(10, "%s(): pe_name \"%s\"\n", __func__,
                  pe_name);

        rcu_read_lock();
        list_for_each_entry_rcu(pe, &ip_vs_pe, n_list) {
                /* Test and get the modules atomically */
                if (pe->module &&
                    !try_module_get(pe->module)) {
                        /* This pe is just deleted */
                        continue;
                }
                if (strcmp(pe_name, pe->name)==0) {
                        /* HIT */
                        rcu_read_unlock();
                        return pe;
                }
                module_put(pe->module);
        }
        rcu_read_unlock();

        return NULL;
}

/* Lookup pe and try to load it if it doesn't exist */
struct ip_vs_pe *ip_vs_pe_getbyname(const char *name)
{
        struct ip_vs_pe *pe;

        /* Search for the pe by name */
        pe = __ip_vs_pe_getbyname(name);

        /* If pe not found, load the module and search again */
        if (!pe) {
                request_module("ip_vs_pe_%s", name);
                pe = __ip_vs_pe_getbyname(name);
        }

        return pe;
}

/* Register a pe in the pe list */
int register_ip_vs_pe(struct ip_vs_pe *pe)
{
        struct ip_vs_pe *tmp;

        /* increase the module use count */
        if (!ip_vs_use_count_inc())
                return -ENOENT;

        mutex_lock(&ip_vs_pe_mutex);
        /* Make sure that the pe with this name doesn't exist
         * in the pe list.
         */
        list_for_each_entry(tmp, &ip_vs_pe, n_list) {
                if (strcmp(tmp->name, pe->name) == 0) {
                        mutex_unlock(&ip_vs_pe_mutex);
                        ip_vs_use_count_dec();
                        pr_err("%s(): [%s] pe already existed "
                               "in the system\n", __func__, pe->name);
                        return -EINVAL;
                }
        }
        /* Add it into the d-linked pe list */
        list_add_rcu(&pe->n_list, &ip_vs_pe);
        mutex_unlock(&ip_vs_pe_mutex);

        pr_info("[%s] pe registered.\n", pe->name);

        return 0;
}
EXPORT_SYMBOL_GPL(register_ip_vs_pe);

/* Unregister a pe from the pe list */
int unregister_ip_vs_pe(struct ip_vs_pe *pe)
{
        mutex_lock(&ip_vs_pe_mutex);
        /* Remove it from the d-linked pe list */
        list_del_rcu(&pe->n_list);
        mutex_unlock(&ip_vs_pe_mutex);

        /* decrease the module use count */
        ip_vs_use_count_dec();

        pr_info("[%s] pe unregistered.\n", pe->name);

        return 0;
}
EXPORT_SYMBOL_GPL(unregister_ip_vs_pe);






















    1 







    1 



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        MPLS GSO Support
 *
 *        Authors: Simon Horman (horms@verge.net.au)
 *
 *        Based on: GSO portions of net/ipv4/gre.c
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/err.h>
#include <linux/module.h>
#include <linux/netdev_features.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <net/gso.h>
#include <net/mpls.h>

static struct sk_buff *mpls_gso_segment(struct sk_buff *skb,
                                       netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        u16 mac_offset = skb->mac_header;
        netdev_features_t mpls_features;
        u16 mac_len = skb->mac_len;
        __be16 mpls_protocol;
        unsigned int mpls_hlen;

        if (!skb_inner_network_header_was_set(skb))
                goto out;

        skb_reset_network_header(skb);
        mpls_hlen = skb_inner_network_header(skb) - skb_network_header(skb);
        if (unlikely(!mpls_hlen || mpls_hlen % MPLS_HLEN))
                goto out;
        if (unlikely(!pskb_may_pull(skb, mpls_hlen)))
                goto out;

        /* Setup inner SKB. */
        mpls_protocol = skb->protocol;
        skb->protocol = skb->inner_protocol;

        __skb_pull(skb, mpls_hlen);

        skb->mac_len = 0;
        skb_reset_mac_header(skb);

        /* Segment inner packet. */
        mpls_features = skb->dev->mpls_features & features;
        segs = skb_mac_gso_segment(skb, mpls_features);
        if (IS_ERR_OR_NULL(segs)) {
                skb_gso_error_unwind(skb, mpls_protocol, mpls_hlen, mac_offset,
                                     mac_len);
                goto out;
        }
        skb = segs;

        mpls_hlen += mac_len;
        do {
                skb->mac_len = mac_len;
                skb->protocol = mpls_protocol;

                skb_reset_inner_network_header(skb);

                __skb_push(skb, mpls_hlen);

                skb_reset_mac_header(skb);
                skb_set_network_header(skb, mac_len);
        } while ((skb = skb->next));

out:
        return segs;
}

static struct packet_offload mpls_mc_offload __read_mostly = {
        .type = cpu_to_be16(ETH_P_MPLS_MC),
        .priority = 15,
        .callbacks = {
                .gso_segment    =        mpls_gso_segment,
        },
};

static struct packet_offload mpls_uc_offload __read_mostly = {
        .type = cpu_to_be16(ETH_P_MPLS_UC),
        .priority = 15,
        .callbacks = {
                .gso_segment    =        mpls_gso_segment,
        },
};

static int __init mpls_gso_init(void)
{
        pr_info("MPLS GSO support\n");

        dev_add_offload(&mpls_uc_offload);
        dev_add_offload(&mpls_mc_offload);

        return 0;
}

static void __exit mpls_gso_exit(void)
{
        dev_remove_offload(&mpls_uc_offload);
        dev_remove_offload(&mpls_mc_offload);
}

module_init(mpls_gso_init);
module_exit(mpls_gso_exit);

MODULE_DESCRIPTION("MPLS GSO support");
MODULE_AUTHOR("Simon Horman <horms@verge.net.au>");
MODULE_LICENSE("GPL");





































    2 






    2 










    2 












    2 

    2 

    2 

















































































































































































    2 














    2 













    2 













    2 











    2 
    2 

























    2 
















    2 
    2 



























    2 




















    2 




    2 


    2 
    2 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
/* mpicoder.c  -  Coder for the external representation of MPIs
 * Copyright (C) 1998, 1999 Free Software Foundation, Inc.
 *
 * This file is part of GnuPG.
 *
 * GnuPG is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * GnuPG is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
 */

#include <linux/bitops.h>
#include <linux/byteorder/generic.h>
#include <linux/count_zeros.h>
#include <linux/export.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#include "mpi-internal.h"

#define MAX_EXTERN_MPI_BITS 16384

/**
 * mpi_read_raw_data - Read a raw byte stream as a positive integer
 * @xbuffer: The data to read
 * @nbytes: The amount of data to read
 */
MPI mpi_read_raw_data(const void *xbuffer, size_t nbytes)
{
        const uint8_t *buffer = xbuffer;
        int i, j;
        unsigned nbits, nlimbs;
        mpi_limb_t a;
        MPI val = NULL;

        while (nbytes > 0 && buffer[0] == 0) {
                buffer++;
                nbytes--;
        }

        nbits = nbytes * 8;
        if (nbits > MAX_EXTERN_MPI_BITS) {
                pr_info("MPI: mpi too large (%u bits)\n", nbits);
                return NULL;
        }
        if (nbytes > 0)
                nbits -= count_leading_zeros(buffer[0]) - (BITS_PER_LONG - 8);

        nlimbs = DIV_ROUND_UP(nbytes, BYTES_PER_MPI_LIMB);
        val = mpi_alloc(nlimbs);
        if (!val)
                return NULL;
        val->nbits = nbits;
        val->sign = 0;
        val->nlimbs = nlimbs;

        if (nbytes > 0) {
                i = BYTES_PER_MPI_LIMB - nbytes % BYTES_PER_MPI_LIMB;
                i %= BYTES_PER_MPI_LIMB;
                for (j = nlimbs; j > 0; j--) {
                        a = 0;
                        for (; i < BYTES_PER_MPI_LIMB; i++) {
                                a <<= 8;
                                a |= *buffer++;
                        }
                        i = 0;
                        val->d[j - 1] = a;
                }
        }
        return val;
}
EXPORT_SYMBOL_GPL(mpi_read_raw_data);

MPI mpi_read_from_buffer(const void *xbuffer, unsigned *ret_nread)
{
        const uint8_t *buffer = xbuffer;
        unsigned int nbits, nbytes;
        MPI val;

        if (*ret_nread < 2)
                return ERR_PTR(-EINVAL);
        nbits = buffer[0] << 8 | buffer[1];

        if (nbits > MAX_EXTERN_MPI_BITS) {
                pr_info("MPI: mpi too large (%u bits)\n", nbits);
                return ERR_PTR(-EINVAL);
        }

        nbytes = DIV_ROUND_UP(nbits, 8);
        if (nbytes + 2 > *ret_nread) {
                pr_info("MPI: mpi larger than buffer nbytes=%u ret_nread=%u\n",
                                nbytes, *ret_nread);
                return ERR_PTR(-EINVAL);
        }

        val = mpi_read_raw_data(buffer + 2, nbytes);
        if (!val)
                return ERR_PTR(-ENOMEM);

        *ret_nread = nbytes + 2;
        return val;
}
EXPORT_SYMBOL_GPL(mpi_read_from_buffer);

static int count_lzeros(MPI a)
{
        mpi_limb_t alimb;
        int i, lzeros = 0;

        for (i = a->nlimbs - 1; i >= 0; i--) {
                alimb = a->d[i];
                if (alimb == 0) {
                        lzeros += sizeof(mpi_limb_t);
                } else {
                        lzeros += count_leading_zeros(alimb) / 8;
                        break;
                }
        }
        return lzeros;
}

/**
 * mpi_read_buffer() - read MPI to a buffer provided by user (msb first)
 *
 * @a:                a multi precision integer
 * @buf:        buffer to which the output will be written to. Needs to be at
 *                least mpi_get_size(a) long.
 * @buf_len:        size of the buf.
 * @nbytes:        receives the actual length of the data written on success and
 *                the data to-be-written on -EOVERFLOW in case buf_len was too
 *                small.
 * @sign:        if not NULL, it will be set to the sign of a.
 *
 * Return:        0 on success or error code in case of error
 */
int mpi_read_buffer(MPI a, uint8_t *buf, unsigned buf_len, unsigned *nbytes,
                    int *sign)
{
        uint8_t *p;
#if BYTES_PER_MPI_LIMB == 4
        __be32 alimb;
#elif BYTES_PER_MPI_LIMB == 8
        __be64 alimb;
#else
#error please implement for this limb size.
#endif
        unsigned int n = mpi_get_size(a);
        int i, lzeros;

        if (!buf || !nbytes)
                return -EINVAL;

        if (sign)
                *sign = a->sign;

        lzeros = count_lzeros(a);

        if (buf_len < n - lzeros) {
                *nbytes = n - lzeros;
                return -EOVERFLOW;
        }

        p = buf;
        *nbytes = n - lzeros;

        for (i = a->nlimbs - 1 - lzeros / BYTES_PER_MPI_LIMB,
                        lzeros %= BYTES_PER_MPI_LIMB;
                i >= 0; i--) {
#if BYTES_PER_MPI_LIMB == 4
                alimb = cpu_to_be32(a->d[i]);
#elif BYTES_PER_MPI_LIMB == 8
                alimb = cpu_to_be64(a->d[i]);
#else
#error please implement for this limb size.
#endif
                memcpy(p, (u8 *)&alimb + lzeros, BYTES_PER_MPI_LIMB - lzeros);
                p += BYTES_PER_MPI_LIMB - lzeros;
                lzeros = 0;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(mpi_read_buffer);

/*
 * mpi_get_buffer() - Returns an allocated buffer with the MPI (msb first).
 * Caller must free the return string.
 * This function does return a 0 byte buffer with nbytes set to zero if the
 * value of A is zero.
 *
 * @a:                a multi precision integer.
 * @nbytes:        receives the length of this buffer.
 * @sign:        if not NULL, it will be set to the sign of the a.
 *
 * Return:        Pointer to MPI buffer or NULL on error
 */
void *mpi_get_buffer(MPI a, unsigned *nbytes, int *sign)
{
        uint8_t *buf;
        unsigned int n;
        int ret;

        if (!nbytes)
                return NULL;

        n = mpi_get_size(a);

        if (!n)
                n++;

        buf = kmalloc(n, GFP_KERNEL);

        if (!buf)
                return NULL;

        ret = mpi_read_buffer(a, buf, n, nbytes, sign);

        if (ret) {
                kfree(buf);
                return NULL;
        }
        return buf;
}
EXPORT_SYMBOL_GPL(mpi_get_buffer);

/**
 * mpi_write_to_sgl() - Funnction exports MPI to an sgl (msb first)
 *
 * This function works in the same way as the mpi_read_buffer, but it
 * takes an sgl instead of u8 * buf.
 *
 * @a:                a multi precision integer
 * @sgl:        scatterlist to write to. Needs to be at least
 *                mpi_get_size(a) long.
 * @nbytes:        the number of bytes to write.  Leading bytes will be
 *                filled with zero.
 * @sign:        if not NULL, it will be set to the sign of a.
 *
 * Return:        0 on success or error code in case of error
 */
int mpi_write_to_sgl(MPI a, struct scatterlist *sgl, unsigned nbytes,
                     int *sign)
{
        u8 *p, *p2;
#if BYTES_PER_MPI_LIMB == 4
        __be32 alimb;
#elif BYTES_PER_MPI_LIMB == 8
        __be64 alimb;
#else
#error please implement for this limb size.
#endif
        unsigned int n = mpi_get_size(a);
        struct sg_mapping_iter miter;
        int i, x, buf_len;
        int nents;

        if (sign)
                *sign = a->sign;

        if (nbytes < n)
                return -EOVERFLOW;

        nents = sg_nents_for_len(sgl, nbytes);
        if (nents < 0)
                return -EINVAL;

        sg_miter_start(&miter, sgl, nents, SG_MITER_ATOMIC | SG_MITER_TO_SG);
        sg_miter_next(&miter);
        buf_len = miter.length;
        p2 = miter.addr;

        while (nbytes > n) {
                i = min_t(unsigned, nbytes - n, buf_len);
                memset(p2, 0, i);
                p2 += i;
                nbytes -= i;

                buf_len -= i;
                if (!buf_len) {
                        sg_miter_next(&miter);
                        buf_len = miter.length;
                        p2 = miter.addr;
                }
        }

        for (i = a->nlimbs - 1; i >= 0; i--) {
#if BYTES_PER_MPI_LIMB == 4
                alimb = a->d[i] ? cpu_to_be32(a->d[i]) : 0;
#elif BYTES_PER_MPI_LIMB == 8
                alimb = a->d[i] ? cpu_to_be64(a->d[i]) : 0;
#else
#error please implement for this limb size.
#endif
                p = (u8 *)&alimb;

                for (x = 0; x < sizeof(alimb); x++) {
                        *p2++ = *p++;
                        if (!--buf_len) {
                                sg_miter_next(&miter);
                                buf_len = miter.length;
                                p2 = miter.addr;
                        }
                }
        }

        sg_miter_stop(&miter);
        return 0;
}
EXPORT_SYMBOL_GPL(mpi_write_to_sgl);

/*
 * mpi_read_raw_from_sgl() - Function allocates an MPI and populates it with
 *                             data from the sgl
 *
 * This function works in the same way as the mpi_read_raw_data, but it
 * takes an sgl instead of void * buffer. i.e. it allocates
 * a new MPI and reads the content of the sgl to the MPI.
 *
 * @sgl:        scatterlist to read from
 * @nbytes:        number of bytes to read
 *
 * Return:        Pointer to a new MPI or NULL on error
 */
MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes)
{
        struct sg_mapping_iter miter;
        unsigned int nbits, nlimbs;
        int x, j, z, lzeros, ents;
        unsigned int len;
        const u8 *buff;
        mpi_limb_t a;
        MPI val = NULL;

        ents = sg_nents_for_len(sgl, nbytes);
        if (ents < 0)
                return NULL;

        sg_miter_start(&miter, sgl, ents, SG_MITER_ATOMIC | SG_MITER_FROM_SG);

        lzeros = 0;
        len = 0;
        while (nbytes > 0) {
                while (len && !*buff) {
                        lzeros++;
                        len--;
                        buff++;
                }

                if (len && *buff)
                        break;

                sg_miter_next(&miter);
                buff = miter.addr;
                len = miter.length;

                nbytes -= lzeros;
                lzeros = 0;
        }

        miter.consumed = lzeros;

        nbytes -= lzeros;
        nbits = nbytes * 8;
        if (nbits > MAX_EXTERN_MPI_BITS) {
                sg_miter_stop(&miter);
                pr_info("MPI: mpi too large (%u bits)\n", nbits);
                return NULL;
        }

        if (nbytes > 0)
                nbits -= count_leading_zeros(*buff) - (BITS_PER_LONG - 8);

        sg_miter_stop(&miter);

        nlimbs = DIV_ROUND_UP(nbytes, BYTES_PER_MPI_LIMB);
        val = mpi_alloc(nlimbs);
        if (!val)
                return NULL;

        val->nbits = nbits;
        val->sign = 0;
        val->nlimbs = nlimbs;

        if (nbytes == 0)
                return val;

        j = nlimbs - 1;
        a = 0;
        z = BYTES_PER_MPI_LIMB - nbytes % BYTES_PER_MPI_LIMB;
        z %= BYTES_PER_MPI_LIMB;

        while (sg_miter_next(&miter)) {
                buff = miter.addr;
                len = min_t(unsigned, miter.length, nbytes);
                nbytes -= len;

                for (x = 0; x < len; x++) {
                        a <<= 8;
                        a |= *buff++;
                        if (((z + x + 1) % BYTES_PER_MPI_LIMB) == 0) {
                                val->d[j--] = a;
                                a = 0;
                        }
                }
                z += x;
        }

        return val;
}
EXPORT_SYMBOL_GPL(mpi_read_raw_from_sgl);





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_TC_MIR_H
#define __NET_TC_MIR_H

#include <net/act_api.h>
#include <linux/tc_act/tc_mirred.h>

struct tcf_mirred {
        struct tc_action        common;
        int                        tcfm_eaction;
        u32                     tcfm_blockid;
        bool                        tcfm_mac_header_xmit;
        struct net_device __rcu        *tcfm_dev;
        netdevice_tracker        tcfm_dev_tracker;
        struct list_head        tcfm_list;
};
#define to_mirred(a) ((struct tcf_mirred *)a)

static inline bool is_tcf_mirred_egress_redirect(const struct tc_action *a)
{
#ifdef CONFIG_NET_CLS_ACT
        if (a->ops && a->ops->id == TCA_ID_MIRRED)
                return to_mirred(a)->tcfm_eaction == TCA_EGRESS_REDIR;
#endif
        return false;
}

static inline bool is_tcf_mirred_egress_mirror(const struct tc_action *a)
{
#ifdef CONFIG_NET_CLS_ACT
        if (a->ops && a->ops->id == TCA_ID_MIRRED)
                return to_mirred(a)->tcfm_eaction == TCA_EGRESS_MIRROR;
#endif
        return false;
}

static inline bool is_tcf_mirred_ingress_redirect(const struct tc_action *a)
{
#ifdef CONFIG_NET_CLS_ACT
        if (a->ops && a->ops->id == TCA_ID_MIRRED)
                return to_mirred(a)->tcfm_eaction == TCA_INGRESS_REDIR;
#endif
        return false;
}

static inline bool is_tcf_mirred_ingress_mirror(const struct tc_action *a)
{
#ifdef CONFIG_NET_CLS_ACT
        if (a->ops && a->ops->id == TCA_ID_MIRRED)
                return to_mirred(a)->tcfm_eaction == TCA_INGRESS_MIRROR;
#endif
        return false;
}

static inline struct net_device *tcf_mirred_dev(const struct tc_action *a)
{
        return rtnl_dereference(to_mirred(a)->tcfm_dev);
}

#endif /* __NET_TC_MIR_H */

























    2 


    2 



    2 




















   17 








   20 










   20 




   19 



   20 


   19 




   21 





















    2 




    1 

    1 


    2 























    1 





    1 

    1 
    1 








































    3 











    1 

    2 









    3 











   12 

























   16 









    4 

   12 



    4 
   16 









   20 








    4 







   20 






















   22 








    1 





   20 

   21 



   21 






   21 




   20 
    1 









   21 










   21 







    1 








    4 
   20 








   20 










   16 



    5 







    4 









   15 
    5 
    4 
   12 




   16 


   20 



   19 
    4 







    3 




    4 









    1 





    3 






















    3 


   21 
    3 






   17 


    1 






    1 


   24 





    1 


    1 


    2 




    1 


    3 





    2 





    2 




    1 


    2 




    1 



    1 

    4 

    3 








    4 



























    1 





































   48 

















   48 



   22 

    1 










    1 






   20 


    4 


    2 


















    2 









   22 




















    1 












   22 







    3 
   22 
































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
// SPDX-License-Identifier: GPL-2.0-or-later
/* AF_RXRPC sendmsg() implementation.
 *
 * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/net.h>
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Propose an abort to be made in the I/O thread.
 */
bool rxrpc_propose_abort(struct rxrpc_call *call, s32 abort_code, int error,
                         enum rxrpc_abort_reason why)
{
        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        if (!call->send_abort && !rxrpc_call_is_complete(call)) {
                call->send_abort_why = why;
                call->send_abort_err = error;
                call->send_abort_seq = 0;
                trace_rxrpc_abort_call(call, abort_code);
                /* Request abort locklessly vs rxrpc_input_call_event(). */
                smp_store_release(&call->send_abort, abort_code);
                rxrpc_poke_call(call, rxrpc_call_poke_abort);
                return true;
        }

        return false;
}

/*
 * Wait for a call to become connected.  Interruption here doesn't cause the
 * call to be aborted.
 */
static int rxrpc_wait_to_be_connected(struct rxrpc_call *call, long *timeo)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret = 0;

        _enter("%d", call->debug_id);

        if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                goto no_wait;

        add_wait_queue_exclusive(&call->waitq, &myself);

        for (;;) {
                switch (call->interruptibility) {
                case RXRPC_INTERRUPTIBLE:
                case RXRPC_PREINTERRUPTIBLE:
                        set_current_state(TASK_INTERRUPTIBLE);
                        break;
                case RXRPC_UNINTERRUPTIBLE:
                default:
                        set_current_state(TASK_UNINTERRUPTIBLE);
                        break;
                }

                if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                        break;
                if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
                     call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
                    signal_pending(current)) {
                        ret = sock_intr_errno(*timeo);
                        break;
                }
                *timeo = schedule_timeout(*timeo);
        }

        remove_wait_queue(&call->waitq, &myself);
        __set_current_state(TASK_RUNNING);

no_wait:
        if (ret == 0 && rxrpc_call_is_complete(call))
                ret = call->error;

        _leave(" = %d", ret);
        return ret;
}

/*
 * Return true if there's sufficient Tx queue space.
 */
static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)
{
        rxrpc_seq_t tx_bottom = READ_ONCE(call->tx_bottom);

        if (_tx_win)
                *_tx_win = tx_bottom;
        return call->send_top - tx_bottom < 256;
}

/*
 * Wait for space to appear in the Tx queue or a signal to occur.
 */
static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx,
                                         struct rxrpc_call *call,
                                         long *timeo)
{
        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (signal_pending(current))
                        return sock_intr_errno(*timeo);

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly, but with
 * a timeout of 2*RTT if no progress was made and a signal occurred.
 */
static int rxrpc_wait_for_tx_window_waitall(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call)
{
        rxrpc_seq_t tx_start, tx_win;
        signed long rtt, timeout;

        rtt = READ_ONCE(call->srtt_us) >> 3;
        rtt = usecs_to_jiffies(rtt) * 2;
        if (rtt < 2)
                rtt = 2;

        timeout = rtt;
        tx_start = READ_ONCE(call->tx_bottom);

        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);

                if (rxrpc_check_tx_space(call, &tx_win))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (timeout == 0 &&
                    tx_win == tx_start && signal_pending(current))
                        return -EINTR;

                if (tx_win != tx_start) {
                        timeout = rtt;
                        tx_start = tx_win;
                }

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                timeout = schedule_timeout(timeout);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly.
 */
static int rxrpc_wait_for_tx_window_nonintr(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call,
                                            long *timeo)
{
        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * wait for space to appear in the transmit/ACK window
 * - caller holds the socket locked
 */
static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
                                    struct rxrpc_call *call,
                                    long *timeo,
                                    bool waitall)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret;

        _enter(",{%u,%u,%u}",
               call->tx_bottom, call->tx_top, call->tx_winsize);

        add_wait_queue(&call->waitq, &myself);

        switch (call->interruptibility) {
        case RXRPC_INTERRUPTIBLE:
                if (waitall)
                        ret = rxrpc_wait_for_tx_window_waitall(rx, call);
                else
                        ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);
                break;
        case RXRPC_PREINTERRUPTIBLE:
        case RXRPC_UNINTERRUPTIBLE:
        default:
                ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);
                break;
        }

        remove_wait_queue(&call->waitq, &myself);
        set_current_state(TASK_RUNNING);
        _leave(" = %d", ret);
        return ret;
}

/*
 * Notify the owner of the call that the transmit phase is ended and the last
 * packet has been queued.
 */
static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,
                                rxrpc_notify_end_tx_t notify_end_tx)
{
        if (notify_end_tx)
                notify_end_tx(&rx->sk, call, call->user_call_ID);
}

/*
 * Queue a DATA packet for transmission, set the resend timeout and send
 * the packet immediately.  Returns the error from rxrpc_send_data_packet()
 * in case the caller wants to do something with it.
 */
static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,
                               struct rxrpc_txbuf *txb,
                               rxrpc_notify_end_tx_t notify_end_tx)
{
        struct rxrpc_txqueue *sq = call->send_queue;
        rxrpc_seq_t seq = txb->seq;
        bool poke, last = txb->flags & RXRPC_LAST_PACKET;
        int ix = seq & RXRPC_TXQ_MASK;
        rxrpc_inc_stat(call->rxnet, stat_tx_data);

        ASSERTCMP(txb->seq, ==, call->send_top + 1);

        if (last)
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue_last);
        else
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue);

        if (WARN_ON_ONCE(sq->bufs[ix]))
                trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue_dup);
        else
                trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue);

        /* Add the packet to the call's output buffer */
        poke = (READ_ONCE(call->tx_bottom) == call->send_top);
        sq->bufs[ix] = txb;
        /* Order send_top after the queue->next pointer and txb content. */
        smp_store_release(&call->send_top, seq);
        if (last) {
                set_bit(RXRPC_CALL_TX_NO_MORE, &call->flags);
                rxrpc_notify_end_tx(rx, call, notify_end_tx);
                call->send_queue = NULL;
        }

        if (poke)
                rxrpc_poke_call(call, rxrpc_call_poke_start);
}

/*
 * Allocate a new txqueue unit and add it to the transmission queue.
 */
static int rxrpc_alloc_txqueue(struct sock *sk, struct rxrpc_call *call)
{
        struct rxrpc_txqueue *tq;

        tq = kzalloc(sizeof(*tq), sk->sk_allocation);
        if (!tq)
                return -ENOMEM;

        tq->xmit_ts_base = KTIME_MIN;
        for (int i = 0; i < RXRPC_NR_TXQUEUE; i++)
                tq->segment_xmit_ts[i] = UINT_MAX;

        if (call->send_queue) {
                tq->qbase = call->send_top + 1;
                call->send_queue->next = tq;
                call->send_queue = tq;
        } else if (WARN_ON(call->tx_queue)) {
                kfree(tq);
                return -ENOMEM;
        } else {
                /* We start at seq 1, so pretend seq 0 is hard-acked. */
                tq->nr_reported_acks = 1;
                tq->segment_acked = 1UL;
                tq->qbase = 0;
                call->tx_qbase = 0;
                call->send_queue = tq;
                call->tx_qtail = tq;
                call->tx_queue = tq;
        }

        trace_rxrpc_tq(call, tq, call->send_top, rxrpc_tq_alloc);
        return 0;
}

/*
 * send data through a socket
 * - must be called in process context
 * - The caller holds the call user access mutex, but not the socket lock.
 */
static int rxrpc_send_data(struct rxrpc_sock *rx,
                           struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx,
                           bool *_dropped_lock)
{
        struct rxrpc_txbuf *txb;
        struct sock *sk = &rx->sk;
        enum rxrpc_call_state state;
        long timeo;
        bool more = msg->msg_flags & MSG_MORE;
        int ret, copied = 0;

        if (test_bit(RXRPC_CALL_TX_NO_MORE, &call->flags)) {
                trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,
                                  call->cid, call->call_id, call->rx_consumed,
                                  0, -EPROTO);
                return -EPROTO;
        }

        timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

        ret = rxrpc_wait_to_be_connected(call, &timeo);
        if (ret < 0)
                return ret;

        if (call->conn->state == RXRPC_CONN_CLIENT_UNSECURED) {
                ret = rxrpc_init_client_conn_security(call->conn);
                if (ret < 0)
                        return ret;
        }

        /* this should be in poll */
        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);

reload:
        txb = call->tx_pending;
        call->tx_pending = NULL;
        if (txb)
                rxrpc_see_txbuf(txb, rxrpc_txbuf_see_send_more);

        ret = -EPIPE;
        if (sk->sk_shutdown & SEND_SHUTDOWN)
                goto maybe_error;
        state = rxrpc_call_state(call);
        ret = -ESHUTDOWN;
        if (state >= RXRPC_CALL_COMPLETE)
                goto maybe_error;
        ret = -EPROTO;
        if (state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
            state != RXRPC_CALL_SERVER_ACK_REQUEST &&
            state != RXRPC_CALL_SERVER_SEND_REPLY) {
                /* Request phase complete for this client call */
                trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,
                                  call->cid, call->call_id, call->rx_consumed,
                                  0, -EPROTO);
                goto maybe_error;
        }

        ret = -EMSGSIZE;
        if (call->tx_total_len != -1) {
                if (len - copied > call->tx_total_len)
                        goto maybe_error;
                if (!more && len - copied != call->tx_total_len)
                        goto maybe_error;
        }

        do {
                if (!txb) {
                        size_t remain;

                        _debug("alloc");

                        if (!rxrpc_check_tx_space(call, NULL))
                                goto wait_for_space;

                        /* See if we need to begin/extend the Tx queue. */
                        if (!call->send_queue || !((call->send_top + 1) & RXRPC_TXQ_MASK)) {
                                ret = rxrpc_alloc_txqueue(sk, call);
                                if (ret < 0)
                                        goto maybe_error;
                        }

                        /* Work out the maximum size of a packet.  Assume that
                         * the security header is going to be in the padded
                         * region (enc blocksize), but the trailer is not.
                         */
                        remain = more ? INT_MAX : msg_data_left(msg);
                        txb = call->conn->security->alloc_txbuf(call, remain, sk->sk_allocation);
                        if (!txb) {
                                ret = -ENOMEM;
                                goto maybe_error;
                        }
                }

                _debug("append");

                /* append next segment of data to the current buffer */
                if (msg_data_left(msg) > 0) {
                        size_t copy = umin(txb->space, msg_data_left(msg));

                        _debug("add %zu", copy);
                        if (!copy_from_iter_full(txb->data + txb->offset,
                                                 copy, &msg->msg_iter))
                                goto efault;
                        _debug("added");
                        txb->space -= copy;
                        txb->len += copy;
                        txb->offset += copy;
                        copied += copy;
                        if (call->tx_total_len != -1)
                                call->tx_total_len -= copy;
                }

                /* check for the far side aborting the call or a network error
                 * occurring */
                if (rxrpc_call_is_complete(call))
                        goto call_terminated;

                /* add the packet to the send queue if it's now full */
                if (!txb->space ||
                    (msg_data_left(msg) == 0 && !more)) {
                        if (msg_data_left(msg) == 0 && !more)
                                txb->flags |= RXRPC_LAST_PACKET;

                        ret = call->security->secure_packet(call, txb);
                        if (ret < 0)
                                goto out;
                        rxrpc_queue_packet(rx, call, txb, notify_end_tx);
                        txb = NULL;
                }
        } while (msg_data_left(msg) > 0);

success:
        ret = copied;
        if (rxrpc_call_is_complete(call) &&
            call->error < 0)
                ret = call->error;
out:
        call->tx_pending = txb;
        _leave(" = %d", ret);
        return ret;

call_terminated:
        rxrpc_put_txbuf(txb, rxrpc_txbuf_put_send_aborted);
        _leave(" = %d", call->error);
        return call->error;

maybe_error:
        if (copied)
                goto success;
        goto out;

efault:
        ret = -EFAULT;
        goto out;

wait_for_space:
        ret = -EAGAIN;
        if (msg->msg_flags & MSG_DONTWAIT)
                goto maybe_error;
        mutex_unlock(&call->user_mutex);
        *_dropped_lock = true;
        ret = rxrpc_wait_for_tx_window(rx, call, &timeo,
                                       msg->msg_flags & MSG_WAITALL);
        if (ret < 0)
                goto maybe_error;
        if (call->interruptibility == RXRPC_INTERRUPTIBLE) {
                if (mutex_lock_interruptible(&call->user_mutex) < 0) {
                        ret = sock_intr_errno(timeo);
                        goto maybe_error;
                }
        } else {
                mutex_lock(&call->user_mutex);
        }
        *_dropped_lock = false;
        goto reload;
}

/*
 * extract control messages from the sendmsg() control buffer
 */
static int rxrpc_sendmsg_cmsg(struct msghdr *msg, struct rxrpc_send_params *p)
{
        struct cmsghdr *cmsg;
        bool got_user_ID = false;
        int len;

        if (msg->msg_controllen == 0)
                return -EINVAL;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                len = cmsg->cmsg_len - sizeof(struct cmsghdr);
                _debug("CMSG %d, %d, %d",
                       cmsg->cmsg_level, cmsg->cmsg_type, len);

                if (cmsg->cmsg_level != SOL_RXRPC)
                        continue;

                switch (cmsg->cmsg_type) {
                case RXRPC_USER_CALL_ID:
                        if (msg->msg_flags & MSG_CMSG_COMPAT) {
                                if (len != sizeof(u32))
                                        return -EINVAL;
                                p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);
                        } else {
                                if (len != sizeof(unsigned long))
                                        return -EINVAL;
                                p->call.user_call_ID = *(unsigned long *)
                                        CMSG_DATA(cmsg);
                        }
                        got_user_ID = true;
                        break;

                case RXRPC_ABORT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_SEND_ABORT;
                        if (len != sizeof(p->abort_code))
                                return -EINVAL;
                        p->abort_code = *(unsigned int *)CMSG_DATA(cmsg);
                        if (p->abort_code == 0)
                                return -EINVAL;
                        break;

                case RXRPC_CHARGE_ACCEPT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_CHARGE_ACCEPT;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_EXCLUSIVE_CALL:
                        p->exclusive = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_UPGRADE_SERVICE:
                        p->upgrade = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_TX_LENGTH:
                        if (p->call.tx_total_len != -1 || len != sizeof(__s64))
                                return -EINVAL;
                        p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);
                        if (p->call.tx_total_len < 0)
                                return -EINVAL;
                        break;

                case RXRPC_SET_CALL_TIMEOUT:
                        if (len & 3 || len < 4 || len > 12)
                                return -EINVAL;
                        memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);
                        p->call.nr_timeouts = len / 4;
                        if (p->call.timeouts.hard > INT_MAX / HZ)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)
                                return -ERANGE;
                        break;

                default:
                        return -EINVAL;
                }
        }

        if (!got_user_ID)
                return -EINVAL;
        if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)
                return -EINVAL;
        _leave(" = 0");
        return 0;
}

/*
 * Create a new client call for sendmsg().
 * - Called with the socket lock held, which it must release.
 * - If it returns a call, the call's lock will need releasing by the caller.
 */
static struct rxrpc_call *
rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
                                  struct rxrpc_send_params *p)
        __releases(&rx->sk.sk_lock)
        __acquires(&call->user_mutex)
{
        struct rxrpc_conn_parameters cp;
        struct rxrpc_peer *peer;
        struct rxrpc_call *call;
        struct key *key;

        DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name);

        _enter("");

        if (!msg->msg_name) {
                release_sock(&rx->sk);
                return ERR_PTR(-EDESTADDRREQ);
        }

        peer = rxrpc_lookup_peer(rx->local, srx, GFP_KERNEL);
        if (!peer) {
                release_sock(&rx->sk);
                return ERR_PTR(-ENOMEM);
        }

        key = rx->key;
        if (key && !rx->key->payload.data[0])
                key = NULL;

        memset(&cp, 0, sizeof(cp));
        cp.local                = rx->local;
        cp.peer                        = peer;
        cp.key                        = rx->key;
        cp.security_level        = rx->min_sec_level;
        cp.exclusive                = rx->exclusive | p->exclusive;
        cp.upgrade                = p->upgrade;
        cp.service_id                = srx->srx_service;
        call = rxrpc_new_client_call(rx, &cp, &p->call, GFP_KERNEL,
                                     atomic_inc_return(&rxrpc_debug_id));
        /* The socket is now unlocked */

        rxrpc_put_peer(peer, rxrpc_peer_put_application);
        _leave(" = %p\n", call);
        return call;
}

/*
 * send a message forming part of a client call through an RxRPC socket
 * - caller holds the socket locked
 * - the socket may be either a client socket or a server socket
 */
int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
{
        struct rxrpc_call *call;
        bool dropped_lock = false;
        int ret;

        struct rxrpc_send_params p = {
                .call.tx_total_len        = -1,
                .call.user_call_ID        = 0,
                .call.nr_timeouts        = 0,
                .call.interruptibility        = RXRPC_INTERRUPTIBLE,
                .abort_code                = 0,
                .command                = RXRPC_CMD_SEND_DATA,
                .exclusive                = false,
                .upgrade                = false,
        };

        _enter("");

        ret = rxrpc_sendmsg_cmsg(msg, &p);
        if (ret < 0)
                goto error_release_sock;

        if (p.command == RXRPC_CMD_CHARGE_ACCEPT) {
                ret = -EINVAL;
                if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
                        goto error_release_sock;
                ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);
                goto error_release_sock;
        }

        call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);
        if (!call) {
                ret = -EBADSLT;
                if (p.command != RXRPC_CMD_SEND_DATA)
                        goto error_release_sock;
                call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);
                /* The socket is now unlocked... */
                if (IS_ERR(call))
                        return PTR_ERR(call);
                /* ... and we have the call lock. */
                p.call.nr_timeouts = 0;
                ret = 0;
                if (rxrpc_call_is_complete(call))
                        goto out_put_unlock;
        } else {
                switch (rxrpc_call_state(call)) {
                case RXRPC_CALL_CLIENT_AWAIT_CONN:
                case RXRPC_CALL_SERVER_RECV_REQUEST:
                        if (p.command == RXRPC_CMD_SEND_ABORT)
                                break;
                        fallthrough;
                case RXRPC_CALL_UNINITIALISED:
                case RXRPC_CALL_SERVER_PREALLOC:
                        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
                        ret = -EBUSY;
                        goto error_release_sock;
                default:
                        break;
                }

                ret = mutex_lock_interruptible(&call->user_mutex);
                release_sock(&rx->sk);
                if (ret < 0) {
                        ret = -ERESTARTSYS;
                        goto error_put;
                }

                if (p.call.tx_total_len != -1) {
                        ret = -EINVAL;
                        if (call->tx_total_len != -1 ||
                            call->tx_pending ||
                            call->tx_top != 0)
                                goto out_put_unlock;
                        call->tx_total_len = p.call.tx_total_len;
                }
        }

        switch (p.call.nr_timeouts) {
        case 3:
                WRITE_ONCE(call->next_rx_timo, p.call.timeouts.normal);
                fallthrough;
        case 2:
                WRITE_ONCE(call->next_req_timo, p.call.timeouts.idle);
                fallthrough;
        case 1:
                if (p.call.timeouts.hard > 0) {
                        ktime_t delay = ms_to_ktime(p.call.timeouts.hard * MSEC_PER_SEC);

                        WRITE_ONCE(call->expect_term_by,
                                   ktime_add(p.call.timeouts.hard,
                                             ktime_get_real()));
                        trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_hard);
                        rxrpc_poke_call(call, rxrpc_call_poke_set_timeout);

                }
                break;
        }

        if (rxrpc_call_is_complete(call)) {
                /* it's too late for this call */
                ret = -ESHUTDOWN;
                goto out_put_unlock;
        }

        switch (p.command) {
        case RXRPC_CMD_SEND_ABORT:
                rxrpc_propose_abort(call, p.abort_code, -ECONNABORTED,
                                    rxrpc_abort_call_sendmsg);
                ret = 0;
                break;
        case RXRPC_CMD_SEND_DATA:
                ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock);
                break;
        default:
                ret = -EINVAL;
                break;
        }

out_put_unlock:
        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
error_put:
        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
        _leave(" = %d", ret);
        return ret;

error_release_sock:
        release_sock(&rx->sk);
        return ret;
}

/**
 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 * @sock: The socket the call is on
 * @call: The call to send data through
 * @msg: The data to send
 * @len: The amount of data to send
 * @notify_end_tx: Notification that the last packet is queued.
 *
 * Allow a kernel service to send data on a call.  The call must be in an state
 * appropriate to sending data.  No control data should be supplied in @msg,
 * nor should an address be supplied.  MSG_MORE should be flagged if there's
 * more data to come, otherwise this data will end the transmission phase.
 *
 * Return: %0 if successful and a negative error code otherwise.
 */
int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx)
{
        bool dropped_lock = false;
        int ret;

        _enter("{%d},", call->debug_id);

        ASSERTCMP(msg->msg_name, ==, NULL);
        ASSERTCMP(msg->msg_control, ==, NULL);

        mutex_lock(&call->user_mutex);

        ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,
                              notify_end_tx, &dropped_lock);
        if (ret == -ESHUTDOWN)
                ret = call->error;

        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
        _leave(" = %d", ret);
        return ret;
}
EXPORT_SYMBOL(rxrpc_kernel_send_data);

/**
 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 * @sock: The socket the call is on
 * @call: The call to be aborted
 * @abort_code: The abort code to stick into the ABORT packet
 * @error: Local error value
 * @why: Indication as to why.
 *
 * Allow a kernel service to abort a call if it's still in an abortable state.
 *
 * Return: %true if the call was aborted, %false if it was already complete.
 */
bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,
                             u32 abort_code, int error, enum rxrpc_abort_reason why)
{
        bool aborted;

        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        mutex_lock(&call->user_mutex);
        aborted = rxrpc_propose_abort(call, abort_code, error, why);
        mutex_unlock(&call->user_mutex);
        return aborted;
}
EXPORT_SYMBOL(rxrpc_kernel_abort_call);

/**
 * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
 * @sock: The socket the call is on
 * @call: The call to be informed
 * @tx_total_len: The amount of data to be transmitted for this call
 *
 * Allow a kernel service to set the total transmit length on a call.  This
 * allows buffer-to-packet encrypt-and-copy to be performed.
 *
 * This function is primarily for use for setting the reply length since the
 * request length can be set when beginning the call.
 */
void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,
                                s64 tx_total_len)
{
        WARN_ON(call->tx_total_len != -1);
        call->tx_total_len = tx_total_len;
}
EXPORT_SYMBOL(rxrpc_kernel_set_tx_length);














   65 



























   74 

























  116 



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM page_pool

#if !defined(_TRACE_PAGE_POOL_H) || defined(TRACE_HEADER_MULTI_READ)
#define      _TRACE_PAGE_POOL_H

#include <linux/types.h>
#include <linux/tracepoint.h>

#include <trace/events/mmflags.h>
#include <net/page_pool/types.h>

TRACE_EVENT(page_pool_release,

        TP_PROTO(const struct page_pool *pool,
                 s32 inflight, u32 hold, u32 release),

        TP_ARGS(pool, inflight, hold, release),

        TP_STRUCT__entry(
                __field(const struct page_pool *, pool)
                __field(s32,        inflight)
                __field(u32,        hold)
                __field(u32,        release)
                __field(u64,        cnt)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->inflight        = inflight;
                __entry->hold                = hold;
                __entry->release        = release;
                __entry->cnt                = pool->destroy_cnt;
        ),

        TP_printk("page_pool=%p inflight=%d hold=%u release=%u cnt=%llu",
                __entry->pool, __entry->inflight, __entry->hold,
                __entry->release, __entry->cnt)
);

TRACE_EVENT(page_pool_state_release,

        TP_PROTO(const struct page_pool *pool,
                 netmem_ref netmem, u32 release),

        TP_ARGS(pool, netmem, release),

        TP_STRUCT__entry(
                __field(const struct page_pool *,        pool)
                __field(unsigned long,                        netmem)
                __field(u32,                                release)
                __field(unsigned long,                        pfn)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->netmem                = (__force unsigned long)netmem;
                __entry->release        = release;
                __entry->pfn                = netmem_pfn_trace(netmem);
        ),

        TP_printk("page_pool=%p netmem=%p is_net_iov=%lu pfn=0x%lx release=%u",
                  __entry->pool, (void *)__entry->netmem,
                  __entry->netmem & NET_IOV, __entry->pfn, __entry->release)
);

TRACE_EVENT(page_pool_state_hold,

        TP_PROTO(const struct page_pool *pool,
                 netmem_ref netmem, u32 hold),

        TP_ARGS(pool, netmem, hold),

        TP_STRUCT__entry(
                __field(const struct page_pool *,        pool)
                __field(unsigned long,                        netmem)
                __field(u32,                                hold)
                __field(unsigned long,                        pfn)
        ),

        TP_fast_assign(
                __entry->pool        = pool;
                __entry->netmem        = (__force unsigned long)netmem;
                __entry->hold        = hold;
                __entry->pfn        = netmem_pfn_trace(netmem);
        ),

        TP_printk("page_pool=%p netmem=%p is_net_iov=%lu, pfn=0x%lx hold=%u",
                  __entry->pool, (void *)__entry->netmem,
                  __entry->netmem & NET_IOV, __entry->pfn, __entry->hold)
);

TRACE_EVENT(page_pool_update_nid,

        TP_PROTO(const struct page_pool *pool, int new_nid),

        TP_ARGS(pool, new_nid),

        TP_STRUCT__entry(
                __field(const struct page_pool *, pool)
                __field(int,                          pool_nid)
                __field(int,                          new_nid)
        ),

        TP_fast_assign(
                __entry->pool                = pool;
                __entry->pool_nid        = pool->p.nid;
                __entry->new_nid        = new_nid;
        ),

        TP_printk("page_pool=%p pool_nid=%d new_nid=%d",
                  __entry->pool, __entry->pool_nid, __entry->new_nid)
);

#endif /* _TRACE_PAGE_POOL_H */

/* This part must be outside protection */
#include <trace/define_trace.h>










































































































































    5 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
/*
 * Copyright (c) 2005 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
 * Copyright (c) 2005-2017 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef RDMA_CORE_H
#define RDMA_CORE_H

#include <linux/idr.h>
#include <rdma/uverbs_types.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/ib_verbs.h>
#include <linux/mutex.h>

struct ib_uverbs_device;

void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
                             enum rdma_remove_reason reason);

int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs);

/*
 * Get an ib_uobject that corresponds to the given id from ufile, assuming
 * the object is from the given type. Lock it to the required access when
 * applicable.
 * This function could create (access == NEW), destroy (access == DESTROY)
 * or unlock (access == READ || access == WRITE) objects if required.
 * The action will be finalized only when uverbs_finalize_object or
 * uverbs_finalize_objects are called.
 */
struct ib_uobject *
uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
                             s64 id, struct uverbs_attr_bundle *attrs);

void uverbs_finalize_object(struct ib_uobject *uobj,
                            enum uverbs_obj_access access, bool hw_obj_valid,
                            bool commit, struct uverbs_attr_bundle *attrs);

int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);

void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);

struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs);

/*
 * This is the runtime description of the uverbs API, used by the syscall
 * machinery to validate and dispatch calls.
 */

/*
 * Depending on ID the slot pointer in the radix tree points at one of these
 * structs.
 */

struct uverbs_api_ioctl_method {
        int(__rcu *handler)(struct uverbs_attr_bundle *attrs);
        DECLARE_BITMAP(attr_mandatory, UVERBS_API_ATTR_BKEY_LEN);
        u16 bundle_size;
        u8 use_stack:1;
        u8 driver_method:1;
        u8 disabled:1;
        u8 has_udata:1;
        u8 key_bitmap_len;
        u8 destroy_bkey;
};

struct uverbs_api_write_method {
        int (*handler)(struct uverbs_attr_bundle *attrs);
        u8 disabled:1;
        u8 is_ex:1;
        u8 has_udata:1;
        u8 has_resp:1;
        u8 req_size;
        u8 resp_size;
};

struct uverbs_api_attr {
        struct uverbs_attr_spec spec;
};

struct uverbs_api {
        /* radix tree contains struct uverbs_api_* pointers */
        struct radix_tree_root radix;
        enum rdma_driver_id driver_id;

        unsigned int num_write;
        unsigned int num_write_ex;
        struct uverbs_api_write_method notsupp_method;
        const struct uverbs_api_write_method **write_methods;
        const struct uverbs_api_write_method **write_ex_methods;
};

/*
 * Get an uverbs_api_object that corresponds to the given object_id.
 * Note:
 * -ENOMSG means that any object is allowed to match during lookup.
 */
static inline const struct uverbs_api_object *
uapi_get_object(struct uverbs_api *uapi, u16 object_id)
{
        const struct uverbs_api_object *res;

        if (object_id == UVERBS_IDR_ANY_OBJECT)
                return ERR_PTR(-ENOMSG);

        res = radix_tree_lookup(&uapi->radix, uapi_key_obj(object_id));
        if (!res)
                return ERR_PTR(-ENOENT);

        return res;
}

char *uapi_key_format(char *S, unsigned int key);
struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev);
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev);
void uverbs_disassociate_api(struct uverbs_api *uapi);
void uverbs_destroy_api(struct uverbs_api *uapi);
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
                              unsigned int num_attrs);
void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile);

extern const struct uapi_definition uverbs_def_obj_async_fd[];
extern const struct uapi_definition uverbs_def_obj_counters[];
extern const struct uapi_definition uverbs_def_obj_cq[];
extern const struct uapi_definition uverbs_def_obj_device[];
extern const struct uapi_definition uverbs_def_obj_dm[];
extern const struct uapi_definition uverbs_def_obj_dmah[];
extern const struct uapi_definition uverbs_def_obj_flow_action[];
extern const struct uapi_definition uverbs_def_obj_intf[];
extern const struct uapi_definition uverbs_def_obj_mr[];
extern const struct uapi_definition uverbs_def_obj_qp[];
extern const struct uapi_definition uverbs_def_obj_srq[];
extern const struct uapi_definition uverbs_def_obj_wq[];
extern const struct uapi_definition uverbs_def_write_intf[];

static inline const struct uverbs_api_write_method *
uapi_get_method(const struct uverbs_api *uapi, u32 command)
{
        u32 cmd_idx = command & IB_USER_VERBS_CMD_COMMAND_MASK;

        if (command & ~(u32)(IB_USER_VERBS_CMD_FLAG_EXTENDED |
                             IB_USER_VERBS_CMD_COMMAND_MASK))
                return ERR_PTR(-EINVAL);

        if (command & IB_USER_VERBS_CMD_FLAG_EXTENDED) {
                if (cmd_idx >= uapi->num_write_ex)
                        return ERR_PTR(-EOPNOTSUPP);
                return uapi->write_ex_methods[cmd_idx];
        }

        if (cmd_idx >= uapi->num_write)
                return ERR_PTR(-EOPNOTSUPP);
        return uapi->write_methods[cmd_idx];
}

void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
                       struct ib_udata *udata, unsigned int attr_in,
                       unsigned int attr_out);

#endif /* RDMA_CORE_H */


























































    1 




    1 




    1 













    1 



















    1 

































    1 





    1 





















    1 





    1 



















































































    1 





    1 










































   12 























    1 








    1 








   10 







    1 






    1 



    1 





    7 




    3 



    4 

    2 
    2 























    2 















    4 











    4 



    4 



    4 
    4 





   12 




   12 


   12 



    3 










    3 


    1 
    2 

    1 
    2 




    3 

































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
// SPDX-License-Identifier: GPL-2.0

/* net/sched/sch_etf.c  Earliest TxTime First queueing discipline.
 *
 * Authors:        Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
 *                Vinicius Costa Gomes <vinicius.gomes@intel.com>
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/errqueue.h>
#include <linux/rbtree.h>
#include <linux/skbuff.h>
#include <linux/posix-timers.h>
#include <net/netlink.h>
#include <net/sch_generic.h>
#include <net/pkt_sched.h>
#include <net/sock.h>

#define DEADLINE_MODE_IS_ON(x) ((x)->flags & TC_ETF_DEADLINE_MODE_ON)
#define OFFLOAD_IS_ON(x) ((x)->flags & TC_ETF_OFFLOAD_ON)
#define SKIP_SOCK_CHECK_IS_SET(x) ((x)->flags & TC_ETF_SKIP_SOCK_CHECK)

struct etf_sched_data {
        bool offload;
        bool deadline_mode;
        bool skip_sock_check;
        int clockid;
        int queue;
        s32 delta; /* in ns */
        ktime_t last; /* The txtime of the last skb sent to the netdevice. */
        struct rb_root_cached head;
        struct qdisc_watchdog watchdog;
        ktime_t (*get_time)(void);
};

static const struct nla_policy etf_policy[TCA_ETF_MAX + 1] = {
        [TCA_ETF_PARMS]        = { .len = sizeof(struct tc_etf_qopt) },
};

static inline int validate_input_params(struct tc_etf_qopt *qopt,
                                        struct netlink_ext_ack *extack)
{
        /* Check if params comply to the following rules:
         *        * Clockid and delta must be valid.
         *
         *        * Dynamic clockids are not supported.
         *
         *        * Delta must be a positive integer.
         *
         * Also note that for the HW offload case, we must
         * expect that system clocks have been synchronized to PHC.
         */
        if (qopt->clockid < 0) {
                NL_SET_ERR_MSG(extack, "Dynamic clockids are not supported");
                return -ENOTSUPP;
        }

        if (qopt->clockid != CLOCK_TAI) {
                NL_SET_ERR_MSG(extack, "Invalid clockid. CLOCK_TAI must be used");
                return -EINVAL;
        }

        if (qopt->delta < 0) {
                NL_SET_ERR_MSG(extack, "Delta must be positive");
                return -EINVAL;
        }

        return 0;
}

static bool is_packet_valid(struct Qdisc *sch, struct sk_buff *nskb)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        ktime_t txtime = nskb->tstamp;
        struct sock *sk = nskb->sk;
        ktime_t now;

        if (q->skip_sock_check)
                goto skip;

        if (!sk || !sk_fullsock(sk))
                return false;

        if (!sock_flag(sk, SOCK_TXTIME))
                return false;

        /* We don't perform crosstimestamping.
         * Drop if packet's clockid differs from qdisc's.
         */
        if (sk->sk_clockid != q->clockid)
                return false;

        if (sk->sk_txtime_deadline_mode != q->deadline_mode)
                return false;

skip:
        now = q->get_time();
        if (ktime_before(txtime, now) || ktime_before(txtime, q->last))
                return false;

        return true;
}

static struct sk_buff *etf_peek_timesortedlist(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct rb_node *p;

        p = rb_first_cached(&q->head);
        if (!p)
                return NULL;

        return rb_to_skb(p);
}

static void reset_watchdog(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct sk_buff *skb = etf_peek_timesortedlist(sch);
        ktime_t next;

        if (!skb) {
                qdisc_watchdog_cancel(&q->watchdog);
                return;
        }

        next = ktime_sub_ns(skb->tstamp, q->delta);
        qdisc_watchdog_schedule_ns(&q->watchdog, ktime_to_ns(next));
}

static void report_sock_error(struct sk_buff *skb, u32 err, u8 code)
{
        struct sock_exterr_skb *serr;
        struct sk_buff *clone;
        ktime_t txtime = skb->tstamp;
        struct sock *sk = skb->sk;

        if (!sk || !sk_fullsock(sk) || !(sk->sk_txtime_report_errors))
                return;

        clone = skb_clone(skb, GFP_ATOMIC);
        if (!clone)
                return;

        serr = SKB_EXT_ERR(clone);
        serr->ee.ee_errno = err;
        serr->ee.ee_origin = SO_EE_ORIGIN_TXTIME;
        serr->ee.ee_type = 0;
        serr->ee.ee_code = code;
        serr->ee.ee_pad = 0;
        serr->ee.ee_data = (txtime >> 32); /* high part of tstamp */
        serr->ee.ee_info = txtime; /* low part of tstamp */

        if (sock_queue_err_skb(sk, clone))
                kfree_skb(clone);
}

static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch,
                                      struct sk_buff **to_free)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct rb_node **p = &q->head.rb_root.rb_node, *parent = NULL;
        ktime_t txtime = nskb->tstamp;
        bool leftmost = true;

        if (!is_packet_valid(sch, nskb)) {
                report_sock_error(nskb, EINVAL,
                                  SO_EE_CODE_TXTIME_INVALID_PARAM);
                return qdisc_drop(nskb, sch, to_free);
        }

        while (*p) {
                struct sk_buff *skb;

                parent = *p;
                skb = rb_to_skb(parent);
                if (ktime_compare(txtime, skb->tstamp) >= 0) {
                        p = &parent->rb_right;
                        leftmost = false;
                } else {
                        p = &parent->rb_left;
                }
        }
        rb_link_node(&nskb->rbnode, parent, p);
        rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost);

        qdisc_qstats_backlog_inc(sch, nskb);
        sch->q.qlen++;

        /* Now we may need to re-arm the qdisc watchdog for the next packet. */
        reset_watchdog(sch);

        return NET_XMIT_SUCCESS;
}

static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb,
                                ktime_t now)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct sk_buff *to_free = NULL;
        struct sk_buff *tmp = NULL;

        skb_rbtree_walk_from_safe(skb, tmp) {
                if (ktime_after(skb->tstamp, now))
                        break;

                rb_erase_cached(&skb->rbnode, &q->head);

                /* The rbnode field in the skb re-uses these fields, now that
                 * we are done with the rbnode, reset them.
                 */
                skb->next = NULL;
                skb->prev = NULL;
                skb->dev = qdisc_dev(sch);

                report_sock_error(skb, ECANCELED, SO_EE_CODE_TXTIME_MISSED);

                qdisc_qstats_backlog_dec(sch, skb);
                qdisc_drop(skb, sch, &to_free);
                qdisc_qstats_overlimit(sch);
                sch->q.qlen--;
        }

        kfree_skb_list(to_free);
}

static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb)
{
        struct etf_sched_data *q = qdisc_priv(sch);

        rb_erase_cached(&skb->rbnode, &q->head);

        /* The rbnode field in the skb re-uses these fields, now that
         * we are done with the rbnode, reset them.
         */
        skb->next = NULL;
        skb->prev = NULL;
        skb->dev = qdisc_dev(sch);

        qdisc_qstats_backlog_dec(sch, skb);

        qdisc_bstats_update(sch, skb);

        q->last = skb->tstamp;

        sch->q.qlen--;
}

static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct sk_buff *skb;
        ktime_t now, next;

        skb = etf_peek_timesortedlist(sch);
        if (!skb)
                return NULL;

        now = q->get_time();

        /* Drop if packet has expired while in queue. */
        if (ktime_before(skb->tstamp, now)) {
                timesortedlist_drop(sch, skb, now);
                skb = NULL;
                goto out;
        }

        /* When in deadline mode, dequeue as soon as possible and change the
         * txtime from deadline to (now + delta).
         */
        if (q->deadline_mode) {
                timesortedlist_remove(sch, skb);
                skb->tstamp = now;
                goto out;
        }

        next = ktime_sub_ns(skb->tstamp, q->delta);

        /* Dequeue only if now is within the [txtime - delta, txtime] range. */
        if (ktime_after(now, next))
                timesortedlist_remove(sch, skb);
        else
                skb = NULL;

out:
        /* Now we may need to re-arm the qdisc watchdog for the next packet. */
        reset_watchdog(sch);

        return skb;
}

static void etf_disable_offload(struct net_device *dev,
                                struct etf_sched_data *q)
{
        struct tc_etf_qopt_offload etf = { };
        const struct net_device_ops *ops;
        int err;

        if (!q->offload)
                return;

        ops = dev->netdev_ops;
        if (!ops->ndo_setup_tc)
                return;

        etf.queue = q->queue;
        etf.enable = 0;

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
        if (err < 0)
                pr_warn("Couldn't disable ETF offload for queue %d\n",
                        etf.queue);
}

static int etf_enable_offload(struct net_device *dev, struct etf_sched_data *q,
                              struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct tc_etf_qopt_offload etf = { };
        int err;

        if (!ops->ndo_setup_tc) {
                NL_SET_ERR_MSG(extack, "Specified device does not support ETF offload");
                return -EOPNOTSUPP;
        }

        etf.queue = q->queue;
        etf.enable = 1;

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Specified device failed to setup ETF hardware offload");
                return err;
        }

        return 0;
}

static int etf_init(struct Qdisc *sch, struct nlattr *opt,
                    struct netlink_ext_ack *extack)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct nlattr *tb[TCA_ETF_MAX + 1];
        struct tc_etf_qopt *qopt;
        int err;

        if (!opt) {
                NL_SET_ERR_MSG(extack,
                               "Missing ETF qdisc options which are mandatory");
                return -EINVAL;
        }

        err = nla_parse_nested_deprecated(tb, TCA_ETF_MAX, opt, etf_policy,
                                          extack);
        if (err < 0)
                return err;

        if (!tb[TCA_ETF_PARMS]) {
                NL_SET_ERR_MSG(extack, "Missing mandatory ETF parameters");
                return -EINVAL;
        }

        qopt = nla_data(tb[TCA_ETF_PARMS]);

        pr_debug("delta %d clockid %d offload %s deadline %s\n",
                 qopt->delta, qopt->clockid,
                 OFFLOAD_IS_ON(qopt) ? "on" : "off",
                 DEADLINE_MODE_IS_ON(qopt) ? "on" : "off");

        err = validate_input_params(qopt, extack);
        if (err < 0)
                return err;

        q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);

        if (OFFLOAD_IS_ON(qopt)) {
                err = etf_enable_offload(dev, q, extack);
                if (err < 0)
                        return err;
        }

        /* Everything went OK, save the parameters used. */
        q->delta = qopt->delta;
        q->clockid = qopt->clockid;
        q->offload = OFFLOAD_IS_ON(qopt);
        q->deadline_mode = DEADLINE_MODE_IS_ON(qopt);
        q->skip_sock_check = SKIP_SOCK_CHECK_IS_SET(qopt);

        switch (q->clockid) {
        case CLOCK_REALTIME:
                q->get_time = ktime_get_real;
                break;
        case CLOCK_MONOTONIC:
                q->get_time = ktime_get;
                break;
        case CLOCK_BOOTTIME:
                q->get_time = ktime_get_boottime;
                break;
        case CLOCK_TAI:
                q->get_time = ktime_get_clocktai;
                break;
        default:
                NL_SET_ERR_MSG(extack, "Clockid is not supported");
                return -ENOTSUPP;
        }

        qdisc_watchdog_init_clockid(&q->watchdog, sch, q->clockid);

        return 0;
}

static void timesortedlist_clear(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct rb_node *p = rb_first_cached(&q->head);

        while (p) {
                struct sk_buff *skb = rb_to_skb(p);

                p = rb_next(p);

                rb_erase_cached(&skb->rbnode, &q->head);
                rtnl_kfree_skbs(skb, skb);
                sch->q.qlen--;
        }
}

static void etf_reset(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);

        /* Only cancel watchdog if it's been initialized. */
        if (q->watchdog.qdisc == sch)
                qdisc_watchdog_cancel(&q->watchdog);

        /* No matter which mode we are on, it's safe to clear both lists. */
        timesortedlist_clear(sch);
        __qdisc_reset_queue(&sch->q);

        q->last = 0;
}

static void etf_destroy(struct Qdisc *sch)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);

        /* Only cancel watchdog if it's been initialized. */
        if (q->watchdog.qdisc == sch)
                qdisc_watchdog_cancel(&q->watchdog);

        etf_disable_offload(dev, q);
}

static int etf_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct etf_sched_data *q = qdisc_priv(sch);
        struct tc_etf_qopt opt = { };
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (!nest)
                goto nla_put_failure;

        opt.delta = READ_ONCE(q->delta);
        opt.clockid = READ_ONCE(q->clockid);
        if (READ_ONCE(q->offload))
                opt.flags |= TC_ETF_OFFLOAD_ON;

        if (READ_ONCE(q->deadline_mode))
                opt.flags |= TC_ETF_DEADLINE_MODE_ON;

        if (READ_ONCE(q->skip_sock_check))
                opt.flags |= TC_ETF_SKIP_SOCK_CHECK;

        if (nla_put(skb, TCA_ETF_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        return nla_nest_end(skb, nest);

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static struct Qdisc_ops etf_qdisc_ops __read_mostly = {
        .id                =        "etf",
        .priv_size        =        sizeof(struct etf_sched_data),
        .enqueue        =        etf_enqueue_timesortedlist,
        .dequeue        =        etf_dequeue_timesortedlist,
        .peek                =        etf_peek_timesortedlist,
        .init                =        etf_init,
        .reset                =        etf_reset,
        .destroy        =        etf_destroy,
        .dump                =        etf_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("etf");

static int __init etf_module_init(void)
{
        return register_qdisc(&etf_qdisc_ops);
}

static void __exit etf_module_exit(void)
{
        unregister_qdisc(&etf_qdisc_ops);
}
module_init(etf_module_init)
module_exit(etf_module_exit)
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Earliest TxTime First (ETF) qdisc");






































































































































































































































































































































































































































































































































































































































   55 










   56 

   55 





   56 

































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
// SPDX-License-Identifier: GPL-2.0
/*
 * Workingset detection
 *
 * Copyright (C) 2013 Red Hat, Inc., Johannes Weiner
 */

#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/writeback.h>
#include <linux/shmem_fs.h>
#include <linux/pagemap.h>
#include <linux/atomic.h>
#include <linux/module.h>
#include <linux/swap.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include "internal.h"

/*
 *                Double CLOCK lists
 *
 * Per node, two clock lists are maintained for file pages: the
 * inactive and the active list.  Freshly faulted pages start out at
 * the head of the inactive list and page reclaim scans pages from the
 * tail.  Pages that are accessed multiple times on the inactive list
 * are promoted to the active list, to protect them from reclaim,
 * whereas active pages are demoted to the inactive list when the
 * active list grows too big.
 *
 *   fault ------------------------+
 *                                 |
 *              +--------------+   |            +-------------+
 *   reclaim <- |   inactive   | <-+-- demotion |    active   | <--+
 *              +--------------+                +-------------+    |
 *                     |                                           |
 *                     +-------------- promotion ------------------+
 *
 *
 *                Access frequency and refault distance
 *
 * A workload is thrashing when its pages are frequently used but they
 * are evicted from the inactive list every time before another access
 * would have promoted them to the active list.
 *
 * In cases where the average access distance between thrashing pages
 * is bigger than the size of memory there is nothing that can be
 * done - the thrashing set could never fit into memory under any
 * circumstance.
 *
 * However, the average access distance could be bigger than the
 * inactive list, yet smaller than the size of memory.  In this case,
 * the set could fit into memory if it weren't for the currently
 * active pages - which may be used more, hopefully less frequently:
 *
 *      +-memory available to cache-+
 *      |                           |
 *      +-inactive------+-active----+
 *  a b | c d e f g h i | J K L M N |
 *      +---------------+-----------+
 *
 * It is prohibitively expensive to accurately track access frequency
 * of pages.  But a reasonable approximation can be made to measure
 * thrashing on the inactive list, after which refaulting pages can be
 * activated optimistically to compete with the existing active pages.
 *
 * Approximating inactive page access frequency - Observations:
 *
 * 1. When a page is accessed for the first time, it is added to the
 *    head of the inactive list, slides every existing inactive page
 *    towards the tail by one slot, and pushes the current tail page
 *    out of memory.
 *
 * 2. When a page is accessed for the second time, it is promoted to
 *    the active list, shrinking the inactive list by one slot.  This
 *    also slides all inactive pages that were faulted into the cache
 *    more recently than the activated page towards the tail of the
 *    inactive list.
 *
 * Thus:
 *
 * 1. The sum of evictions and activations between any two points in
 *    time indicate the minimum number of inactive pages accessed in
 *    between.
 *
 * 2. Moving one inactive page N page slots towards the tail of the
 *    list requires at least N inactive page accesses.
 *
 * Combining these:
 *
 * 1. When a page is finally evicted from memory, the number of
 *    inactive pages accessed while the page was in cache is at least
 *    the number of page slots on the inactive list.
 *
 * 2. In addition, measuring the sum of evictions and activations (E)
 *    at the time of a page's eviction, and comparing it to another
 *    reading (R) at the time the page faults back into memory tells
 *    the minimum number of accesses while the page was not cached.
 *    This is called the refault distance.
 *
 * Because the first access of the page was the fault and the second
 * access the refault, we combine the in-cache distance with the
 * out-of-cache distance to get the complete minimum access distance
 * of this page:
 *
 *      NR_inactive + (R - E)
 *
 * And knowing the minimum access distance of a page, we can easily
 * tell if the page would be able to stay in cache assuming all page
 * slots in the cache were available:
 *
 *   NR_inactive + (R - E) <= NR_inactive + NR_active
 *
 * If we have swap we should consider about NR_inactive_anon and
 * NR_active_anon, so for page cache and anonymous respectively:
 *
 *   NR_inactive_file + (R - E) <= NR_inactive_file + NR_active_file
 *   + NR_inactive_anon + NR_active_anon
 *
 *   NR_inactive_anon + (R - E) <= NR_inactive_anon + NR_active_anon
 *   + NR_inactive_file + NR_active_file
 *
 * Which can be further simplified to:
 *
 *   (R - E) <= NR_active_file + NR_inactive_anon + NR_active_anon
 *
 *   (R - E) <= NR_active_anon + NR_inactive_file + NR_active_file
 *
 * Put into words, the refault distance (out-of-cache) can be seen as
 * a deficit in inactive list space (in-cache).  If the inactive list
 * had (R - E) more page slots, the page would not have been evicted
 * in between accesses, but activated instead.  And on a full system,
 * the only thing eating into inactive list space is active pages.
 *
 *
 *                Refaulting inactive pages
 *
 * All that is known about the active list is that the pages have been
 * accessed more than once in the past.  This means that at any given
 * time there is actually a good chance that pages on the active list
 * are no longer in active use.
 *
 * So when a refault distance of (R - E) is observed and there are at
 * least (R - E) pages in the userspace workingset, the refaulting page
 * is activated optimistically in the hope that (R - E) pages are actually
 * used less frequently than the refaulting page - or even not used at
 * all anymore.
 *
 * That means if inactive cache is refaulting with a suitable refault
 * distance, we assume the cache workingset is transitioning and put
 * pressure on the current workingset.
 *
 * If this is wrong and demotion kicks in, the pages which are truly
 * used more frequently will be reactivated while the less frequently
 * used once will be evicted from memory.
 *
 * But if this is right, the stale pages will be pushed out of memory
 * and the used pages get to stay in cache.
 *
 *                Refaulting active pages
 *
 * If on the other hand the refaulting pages have recently been
 * deactivated, it means that the active list is no longer protecting
 * actively used cache from reclaim. The cache is NOT transitioning to
 * a different workingset; the existing workingset is thrashing in the
 * space allocated to the page cache.
 *
 *
 *                Implementation
 *
 * For each node's LRU lists, a counter for inactive evictions and
 * activations is maintained (node->nonresident_age).
 *
 * On eviction, a snapshot of this counter (along with some bits to
 * identify the node) is stored in the now empty page cache
 * slot of the evicted page.  This is called a shadow entry.
 *
 * On cache misses for which there are shadow entries, an eligible
 * refault distance will immediately activate the refaulting page.
 */

#define WORKINGSET_SHIFT 1
#define EVICTION_SHIFT        ((BITS_PER_LONG - BITS_PER_XA_VALUE) +        \
                         WORKINGSET_SHIFT + NODES_SHIFT + \
                         MEM_CGROUP_ID_SHIFT)
#define EVICTION_MASK        (~0UL >> EVICTION_SHIFT)

/*
 * Eviction timestamps need to be able to cover the full range of
 * actionable refaults. However, bits are tight in the xarray
 * entry, and after storing the identifier for the lruvec there might
 * not be enough left to represent every single actionable refault. In
 * that case, we have to sacrifice granularity for distance, and group
 * evictions into coarser buckets by shaving off lower timestamp bits.
 */
static unsigned int bucket_order __read_mostly;

static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
                         bool workingset)
{
        eviction &= EVICTION_MASK;
        eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
        eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
        eviction = (eviction << WORKINGSET_SHIFT) | workingset;

        return xa_mk_value(eviction);
}

static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
                          unsigned long *evictionp, bool *workingsetp)
{
        unsigned long entry = xa_to_value(shadow);
        int memcgid, nid;
        bool workingset;

        workingset = entry & ((1UL << WORKINGSET_SHIFT) - 1);
        entry >>= WORKINGSET_SHIFT;
        nid = entry & ((1UL << NODES_SHIFT) - 1);
        entry >>= NODES_SHIFT;
        memcgid = entry & ((1UL << MEM_CGROUP_ID_SHIFT) - 1);
        entry >>= MEM_CGROUP_ID_SHIFT;

        *memcgidp = memcgid;
        *pgdat = NODE_DATA(nid);
        *evictionp = entry;
        *workingsetp = workingset;
}

#ifdef CONFIG_LRU_GEN

static void *lru_gen_eviction(struct folio *folio)
{
        int hist;
        unsigned long token;
        unsigned long min_seq;
        struct lruvec *lruvec;
        struct lru_gen_folio *lrugen;
        int type = folio_is_file_lru(folio);
        int delta = folio_nr_pages(folio);
        int refs = folio_lru_refs(folio);
        bool workingset = folio_test_workingset(folio);
        int tier = lru_tier_from_refs(refs, workingset);
        struct mem_cgroup *memcg = folio_memcg(folio);
        struct pglist_data *pgdat = folio_pgdat(folio);

        BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);

        lruvec = mem_cgroup_lruvec(memcg, pgdat);
        lrugen = &lruvec->lrugen;
        min_seq = READ_ONCE(lrugen->min_seq[type]);
        token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);

        hist = lru_hist_from_seq(min_seq);
        atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);

        return pack_shadow(mem_cgroup_id(memcg), pgdat, token, workingset);
}

/*
 * Tests if the shadow entry is for a folio that was recently evicted.
 * Fills in @lruvec, @token, @workingset with the values unpacked from shadow.
 */
static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
                                unsigned long *token, bool *workingset)
{
        int memcg_id;
        unsigned long max_seq;
        struct mem_cgroup *memcg;
        struct pglist_data *pgdat;

        unpack_shadow(shadow, &memcg_id, &pgdat, token, workingset);

        memcg = mem_cgroup_from_id(memcg_id);
        *lruvec = mem_cgroup_lruvec(memcg, pgdat);

        max_seq = READ_ONCE((*lruvec)->lrugen.max_seq);
        max_seq &= EVICTION_MASK >> LRU_REFS_WIDTH;

        return abs_diff(max_seq, *token >> LRU_REFS_WIDTH) < MAX_NR_GENS;
}

static void lru_gen_refault(struct folio *folio, void *shadow)
{
        bool recent;
        int hist, tier, refs;
        bool workingset;
        unsigned long token;
        struct lruvec *lruvec;
        struct lru_gen_folio *lrugen;
        int type = folio_is_file_lru(folio);
        int delta = folio_nr_pages(folio);

        rcu_read_lock();

        recent = lru_gen_test_recent(shadow, &lruvec, &token, &workingset);
        if (lruvec != folio_lruvec(folio))
                goto unlock;

        mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);

        if (!recent)
                goto unlock;

        lrugen = &lruvec->lrugen;

        hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
        refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + 1;
        tier = lru_tier_from_refs(refs, workingset);

        atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);

        /* see folio_add_lru() where folio_set_active() will be called */
        if (lru_gen_in_fault())
                mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);

        if (workingset) {
                folio_set_workingset(folio);
                mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
        } else
                set_mask_bits(&folio->flags.f, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF);
unlock:
        rcu_read_unlock();
}

#else /* !CONFIG_LRU_GEN */

static void *lru_gen_eviction(struct folio *folio)
{
        return NULL;
}

static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
                                unsigned long *token, bool *workingset)
{
        return false;
}

static void lru_gen_refault(struct folio *folio, void *shadow)
{
}

#endif /* CONFIG_LRU_GEN */

/**
 * workingset_age_nonresident - age non-resident entries as LRU ages
 * @lruvec: the lruvec that was aged
 * @nr_pages: the number of pages to count
 *
 * As in-memory pages are aged, non-resident pages need to be aged as
 * well, in order for the refault distances later on to be comparable
 * to the in-memory dimensions. This function allows reclaim and LRU
 * operations to drive the non-resident aging along in parallel.
 */
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages)
{
        /*
         * Reclaiming a cgroup means reclaiming all its children in a
         * round-robin fashion. That means that each cgroup has an LRU
         * order that is composed of the LRU orders of its child
         * cgroups; and every page has an LRU position not just in the
         * cgroup that owns it, but in all of that group's ancestors.
         *
         * So when the physical inactive list of a leaf cgroup ages,
         * the virtual inactive lists of all its parents, including
         * the root cgroup's, age as well.
         */
        do {
                atomic_long_add(nr_pages, &lruvec->nonresident_age);
        } while ((lruvec = parent_lruvec(lruvec)));
}

/**
 * workingset_eviction - note the eviction of a folio from memory
 * @target_memcg: the cgroup that is causing the reclaim
 * @folio: the folio being evicted
 *
 * Return: a shadow entry to be stored in @folio->mapping->i_pages in place
 * of the evicted @folio so that a later refault can be detected.
 */
void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg)
{
        struct pglist_data *pgdat = folio_pgdat(folio);
        unsigned long eviction;
        struct lruvec *lruvec;
        int memcgid;

        /* Folio is fully exclusive and pins folio's memory cgroup pointer */
        VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
        VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

        if (lru_gen_enabled())
                return lru_gen_eviction(folio);

        lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
        /* XXX: target_memcg can be NULL, go through lruvec */
        memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
        eviction = atomic_long_read(&lruvec->nonresident_age);
        eviction >>= bucket_order;
        workingset_age_nonresident(lruvec, folio_nr_pages(folio));
        return pack_shadow(memcgid, pgdat, eviction,
                                folio_test_workingset(folio));
}

/**
 * workingset_test_recent - tests if the shadow entry is for a folio that was
 * recently evicted. Also fills in @workingset with the value unpacked from
 * shadow.
 * @shadow: the shadow entry to be tested.
 * @file: whether the corresponding folio is from the file lru.
 * @workingset: where the workingset value unpacked from shadow should
 * be stored.
 * @flush: whether to flush cgroup rstat.
 *
 * Return: true if the shadow is for a recently evicted folio; false otherwise.
 */
bool workingset_test_recent(void *shadow, bool file, bool *workingset,
                                bool flush)
{
        struct mem_cgroup *eviction_memcg;
        struct lruvec *eviction_lruvec;
        unsigned long refault_distance;
        unsigned long workingset_size;
        unsigned long refault;
        int memcgid;
        struct pglist_data *pgdat;
        unsigned long eviction;

        if (lru_gen_enabled()) {
                bool recent;

                rcu_read_lock();
                recent = lru_gen_test_recent(shadow, &eviction_lruvec, &eviction, workingset);
                rcu_read_unlock();
                return recent;
        }

        rcu_read_lock();
        unpack_shadow(shadow, &memcgid, &pgdat, &eviction, workingset);
        eviction <<= bucket_order;

        /*
         * Look up the memcg associated with the stored ID. It might
         * have been deleted since the folio's eviction.
         *
         * Note that in rare events the ID could have been recycled
         * for a new cgroup that refaults a shared folio. This is
         * impossible to tell from the available data. However, this
         * should be a rare and limited disturbance, and activations
         * are always speculative anyway. Ultimately, it's the aging
         * algorithm's job to shake out the minimum access frequency
         * for the active cache.
         *
         * XXX: On !CONFIG_MEMCG, this will always return NULL; it
         * would be better if the root_mem_cgroup existed in all
         * configurations instead.
         */
        eviction_memcg = mem_cgroup_from_id(memcgid);
        if (!mem_cgroup_tryget(eviction_memcg))
                eviction_memcg = NULL;
        rcu_read_unlock();

        if (!mem_cgroup_disabled() && !eviction_memcg)
                return false;
        /*
         * Flush stats (and potentially sleep) outside the RCU read section.
         *
         * Note that workingset_test_recent() itself might be called in RCU read
         * section (for e.g, in cachestat) - these callers need to skip flushing
         * stats (via the flush argument).
         *
         * XXX: With per-memcg flushing and thresholding, is ratelimiting
         * still needed here?
         */
        if (flush)
                mem_cgroup_flush_stats_ratelimited(eviction_memcg);

        eviction_lruvec = mem_cgroup_lruvec(eviction_memcg, pgdat);
        refault = atomic_long_read(&eviction_lruvec->nonresident_age);

        /*
         * Calculate the refault distance
         *
         * The unsigned subtraction here gives an accurate distance
         * across nonresident_age overflows in most cases. There is a
         * special case: usually, shadow entries have a short lifetime
         * and are either refaulted or reclaimed along with the inode
         * before they get too old.  But it is not impossible for the
         * nonresident_age to lap a shadow entry in the field, which
         * can then result in a false small refault distance, leading
         * to a false activation should this old entry actually
         * refault again.  However, earlier kernels used to deactivate
         * unconditionally with *every* reclaim invocation for the
         * longest time, so the occasional inappropriate activation
         * leading to pressure on the active list is not a problem.
         */
        refault_distance = (refault - eviction) & EVICTION_MASK;

        /*
         * Compare the distance to the existing workingset size. We
         * don't activate pages that couldn't stay resident even if
         * all the memory was available to the workingset. Whether
         * workingset competition needs to consider anon or not depends
         * on having free swap space.
         */
        workingset_size = lruvec_page_state(eviction_lruvec, NR_ACTIVE_FILE);
        if (!file) {
                workingset_size += lruvec_page_state(eviction_lruvec,
                                                     NR_INACTIVE_FILE);
        }
        if (mem_cgroup_get_nr_swap_pages(eviction_memcg) > 0) {
                workingset_size += lruvec_page_state(eviction_lruvec,
                                                     NR_ACTIVE_ANON);
                if (file) {
                        workingset_size += lruvec_page_state(eviction_lruvec,
                                                     NR_INACTIVE_ANON);
                }
        }

        mem_cgroup_put(eviction_memcg);
        return refault_distance <= workingset_size;
}

/**
 * workingset_refault - Evaluate the refault of a previously evicted folio.
 * @folio: The freshly allocated replacement folio.
 * @shadow: Shadow entry of the evicted folio.
 *
 * Calculates and evaluates the refault distance of the previously
 * evicted folio in the context of the node and the memcg whose memory
 * pressure caused the eviction.
 */
void workingset_refault(struct folio *folio, void *shadow)
{
        bool file = folio_is_file_lru(folio);
        struct pglist_data *pgdat;
        struct mem_cgroup *memcg;
        struct lruvec *lruvec;
        bool workingset;
        long nr;

        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

        if (lru_gen_enabled()) {
                lru_gen_refault(folio, shadow);
                return;
        }

        /*
         * The activation decision for this folio is made at the level
         * where the eviction occurred, as that is where the LRU order
         * during folio reclaim is being determined.
         *
         * However, the cgroup that will own the folio is the one that
         * is actually experiencing the refault event. Make sure the folio is
         * locked to guarantee folio_memcg() stability throughout.
         */
        nr = folio_nr_pages(folio);
        memcg = folio_memcg(folio);
        pgdat = folio_pgdat(folio);
        lruvec = mem_cgroup_lruvec(memcg, pgdat);

        mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);

        if (!workingset_test_recent(shadow, file, &workingset, true))
                return;

        folio_set_active(folio);
        workingset_age_nonresident(lruvec, nr);
        mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + file, nr);

        /* Folio was active prior to eviction */
        if (workingset) {
                folio_set_workingset(folio);
                /*
                 * XXX: Move to folio_add_lru() when it supports new vs
                 * putback
                 */
                lru_note_cost_refault(folio);
                mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
        }
}

/**
 * workingset_activation - note a page activation
 * @folio: Folio that is being activated.
 */
void workingset_activation(struct folio *folio)
{
        /*
         * Filter non-memcg pages here, e.g. unmap can call
         * mark_page_accessed() on VDSO pages.
         */
        if (mem_cgroup_disabled() || folio_memcg_charged(folio))
                workingset_age_nonresident(folio_lruvec(folio), folio_nr_pages(folio));
}

/*
 * Shadow entries reflect the share of the working set that does not
 * fit into memory, so their number depends on the access pattern of
 * the workload.  In most cases, they will refault or get reclaimed
 * along with the inode, but a (malicious) workload that streams
 * through files with a total size several times that of available
 * memory, while preventing the inodes from being reclaimed, can
 * create excessive amounts of shadow nodes.  To keep a lid on this,
 * track shadow nodes and reclaim them when they grow way past the
 * point where they would still be useful.
 */

struct list_lru shadow_nodes;

void workingset_update_node(struct xa_node *node)
{
        struct page *page = virt_to_page(node);

        /*
         * Track non-empty nodes that contain only shadow entries;
         * unlink those that contain pages or are being freed.
         *
         * Avoid acquiring the list_lru lock when the nodes are
         * already where they should be. The list_empty() test is safe
         * as node->private_list is protected by the i_pages lock.
         */
        lockdep_assert_held(&node->array->xa_lock);

        if (node->count && node->count == node->nr_values) {
                if (list_empty(&node->private_list)) {
                        list_lru_add_obj(&shadow_nodes, &node->private_list);
                        __inc_node_page_state(page, WORKINGSET_NODES);
                }
        } else {
                if (!list_empty(&node->private_list)) {
                        list_lru_del_obj(&shadow_nodes, &node->private_list);
                        __dec_node_page_state(page, WORKINGSET_NODES);
                }
        }
}

static unsigned long count_shadow_nodes(struct shrinker *shrinker,
                                        struct shrink_control *sc)
{
        unsigned long max_nodes;
        unsigned long nodes;
        unsigned long pages;

        nodes = list_lru_shrink_count(&shadow_nodes, sc);
        if (!nodes)
                return SHRINK_EMPTY;

        /*
         * Approximate a reasonable limit for the nodes
         * containing shadow entries. We don't need to keep more
         * shadow entries than possible pages on the active list,
         * since refault distances bigger than that are dismissed.
         *
         * The size of the active list converges toward 100% of
         * overall page cache as memory grows, with only a tiny
         * inactive list. Assume the total cache size for that.
         *
         * Nodes might be sparsely populated, with only one shadow
         * entry in the extreme case. Obviously, we cannot keep one
         * node for every eligible shadow entry, so compromise on a
         * worst-case density of 1/8th. Below that, not all eligible
         * refaults can be detected anymore.
         *
         * On 64-bit with 7 xa_nodes per page and 64 slots
         * each, this will reclaim shadow entries when they consume
         * ~1.8% of available memory:
         *
         * PAGE_SIZE / xa_nodes / node_entries * 8 / PAGE_SIZE
         */
#ifdef CONFIG_MEMCG
        if (sc->memcg) {
                struct lruvec *lruvec;
                int i;

                mem_cgroup_flush_stats_ratelimited(sc->memcg);
                lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
                for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
                        pages += lruvec_page_state_local(lruvec,
                                                         NR_LRU_BASE + i);
                pages += lruvec_page_state_local(
                        lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT;
                pages += lruvec_page_state_local(
                        lruvec, NR_SLAB_UNRECLAIMABLE_B) >> PAGE_SHIFT;
        } else
#endif
                pages = node_present_pages(sc->nid);

        max_nodes = pages >> (XA_CHUNK_SHIFT - 3);

        if (nodes <= max_nodes)
                return 0;
        return nodes - max_nodes;
}

static enum lru_status shadow_lru_isolate(struct list_head *item,
                                          struct list_lru_one *lru,
                                          void *arg) __must_hold(lru->lock)
{
        struct xa_node *node = container_of(item, struct xa_node, private_list);
        struct address_space *mapping;
        int ret;

        /*
         * Page cache insertions and deletions synchronously maintain
         * the shadow node LRU under the i_pages lock and the
         * &lru->lock. Because the page cache tree is emptied before
         * the inode can be destroyed, holding the &lru->lock pins any
         * address_space that has nodes on the LRU.
         *
         * We can then safely transition to the i_pages lock to
         * pin only the address_space of the particular node we want
         * to reclaim, take the node off-LRU, and drop the &lru->lock.
         */

        mapping = container_of(node->array, struct address_space, i_pages);

        /* Coming from the list, invert the lock order */
        if (!xa_trylock(&mapping->i_pages)) {
                spin_unlock_irq(&lru->lock);
                ret = LRU_RETRY;
                goto out;
        }

        /* For page cache we need to hold i_lock */
        if (mapping->host != NULL) {
                if (!spin_trylock(&mapping->host->i_lock)) {
                        xa_unlock(&mapping->i_pages);
                        spin_unlock_irq(&lru->lock);
                        ret = LRU_RETRY;
                        goto out;
                }
        }

        list_lru_isolate(lru, item);
        __dec_node_page_state(virt_to_page(node), WORKINGSET_NODES);

        spin_unlock(&lru->lock);

        /*
         * The nodes should only contain one or more shadow entries,
         * no pages, so we expect to be able to remove them all and
         * delete and free the empty node afterwards.
         */
        if (WARN_ON_ONCE(!node->nr_values))
                goto out_invalid;
        if (WARN_ON_ONCE(node->count != node->nr_values))
                goto out_invalid;
        xa_delete_node(node, workingset_update_node);
        __inc_lruvec_kmem_state(node, WORKINGSET_NODERECLAIM);

out_invalid:
        xa_unlock_irq(&mapping->i_pages);
        if (mapping->host != NULL) {
                if (mapping_shrinkable(mapping))
                        inode_add_lru(mapping->host);
                spin_unlock(&mapping->host->i_lock);
        }
        ret = LRU_REMOVED_RETRY;
out:
        cond_resched();
        return ret;
}

static unsigned long scan_shadow_nodes(struct shrinker *shrinker,
                                       struct shrink_control *sc)
{
        /* list_lru lock nests inside the IRQ-safe i_pages lock */
        return list_lru_shrink_walk_irq(&shadow_nodes, sc, shadow_lru_isolate,
                                        NULL);
}

/*
 * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe
 * i_pages lock.
 */
static struct lock_class_key shadow_nodes_key;

static int __init workingset_init(void)
{
        struct shrinker *workingset_shadow_shrinker;
        unsigned int timestamp_bits;
        unsigned int max_order;
        int ret = -ENOMEM;

        BUILD_BUG_ON(BITS_PER_LONG < EVICTION_SHIFT);
        /*
         * Calculate the eviction bucket size to cover the longest
         * actionable refault distance, which is currently half of
         * memory (totalram_pages/2). However, memory hotplug may add
         * some more pages at runtime, so keep working with up to
         * double the initial memory by using totalram_pages as-is.
         */
        timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT;
        max_order = fls_long(totalram_pages() - 1);
        if (max_order > timestamp_bits)
                bucket_order = max_order - timestamp_bits;
        pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
               timestamp_bits, max_order, bucket_order);

        workingset_shadow_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
                                                    SHRINKER_MEMCG_AWARE,
                                                    "mm-shadow");
        if (!workingset_shadow_shrinker)
                goto err;

        ret = list_lru_init_memcg_key(&shadow_nodes, workingset_shadow_shrinker,
                                      &shadow_nodes_key);
        if (ret)
                goto err_list_lru;

        workingset_shadow_shrinker->count_objects = count_shadow_nodes;
        workingset_shadow_shrinker->scan_objects = scan_shadow_nodes;
        /* ->count reports only fully expendable nodes */
        workingset_shadow_shrinker->seeks = 0;

        shrinker_register(workingset_shadow_shrinker);
        return 0;
err_list_lru:
        shrinker_free(workingset_shadow_shrinker);
err:
        return ret;
}
module_init(workingset_init);



































































































































































    3 
















































































































    2 



    2 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TTY_PORT_H
#define _LINUX_TTY_PORT_H

#include <linux/kfifo.h>
#include <linux/kref.h>
#include <linux/mutex.h>
#include <linux/tty_buffer.h>
#include <linux/wait.h>

struct attribute_group;
struct tty_driver;
struct tty_port;
struct tty_struct;

/**
 * struct tty_port_operations -- operations on tty_port
 * @carrier_raised: return true if the carrier is raised on @port
 * @dtr_rts: raise the DTR line if @active is true, otherwise lower DTR
 * @shutdown: called when the last close completes or a hangup finishes IFF the
 *        port was initialized. Do not use to free resources. Turn off the device
 *        only. Called under the port mutex to serialize against @activate and
 *        @shutdown.
 * @activate: called under the port mutex from tty_port_open(), serialized using
 *        the port mutex. Supposed to turn on the device.
 *
 *        FIXME: long term getting the tty argument *out* of this would be good
 *        for consoles.
 *
 * @destruct: called on the final put of a port. Free resources, possibly incl.
 *        the port itself.
 */
struct tty_port_operations {
        bool (*carrier_raised)(struct tty_port *port);
        void (*dtr_rts)(struct tty_port *port, bool active);
        void (*shutdown)(struct tty_port *port);
        int (*activate)(struct tty_port *port, struct tty_struct *tty);
        void (*destruct)(struct tty_port *port);
};

struct tty_port_client_operations {
        size_t (*receive_buf)(struct tty_port *port, const u8 *cp, const u8 *fp,
                              size_t count);
        void (*lookahead_buf)(struct tty_port *port, const u8 *cp,
                              const u8 *fp, size_t count);
        void (*write_wakeup)(struct tty_port *port);
};

extern const struct tty_port_client_operations tty_port_default_client_ops;

/**
 * struct tty_port -- port level information
 *
 * @buf: buffer for this port, locked internally
 * @tty: back pointer to &struct tty_struct, valid only if the tty is open. Use
 *         tty_port_tty_get() to obtain it (and tty_kref_put() to release).
 * @itty: internal back pointer to &struct tty_struct. Avoid this. It should be
 *          eliminated in the long term.
 * @ops: tty port operations (like activate, shutdown), see &struct
 *         tty_port_operations
 * @client_ops: tty port client operations (like receive_buf, write_wakeup).
 *                By default, tty_port_default_client_ops is used.
 * @lock: lock protecting @tty
 * @blocked_open: # of procs waiting for open in tty_port_block_til_ready()
 * @count: usage count
 * @open_wait: open waiters queue (waiting e.g. for a carrier)
 * @delta_msr_wait: modem status change queue (waiting for MSR changes)
 * @flags: user TTY flags (%ASYNC_)
 * @iflags: internal flags (%TTY_PORT_)
 * @console: when set, the port is a console
 * @mutex: locking, for open, shutdown and other port operations
 * @buf_mutex: @xmit_buf alloc lock
 * @xmit_buf: optional xmit buffer used by some drivers
 * @xmit_fifo: optional xmit buffer used by some drivers
 * @close_delay: delay in jiffies to wait when closing the port
 * @closing_wait: delay in jiffies for output to be sent before closing
 * @drain_delay: set to zero if no pure time based drain is needed else set to
 *                 size of fifo
 * @kref: references counter. Reaching zero calls @ops->destruct() if non-%NULL
 *          or frees the port otherwise.
 * @client_data: pointer to private data, for @client_ops
 *
 * Each device keeps its own port level information. &struct tty_port was
 * introduced as a common structure for such information. As every TTY device
 * shall have a backing tty_port structure, every driver can use these members.
 *
 * The tty port has a different lifetime to the tty so must be kept apart.
 * In addition be careful as tty -> port mappings are valid for the life
 * of the tty object but in many cases port -> tty mappings are valid only
 * until a hangup so don't use the wrong path.
 *
 * Tty port shall be initialized by tty_port_init() and shut down either by
 * tty_port_destroy() (refcounting not used), or tty_port_put() (refcounting).
 *
 * There is a lot of helpers around &struct tty_port too. To name the most
 * significant ones: tty_port_open(), tty_port_close() (or
 * tty_port_close_start() and tty_port_close_end() separately if need be), and
 * tty_port_hangup(). These call @ops->activate() and @ops->shutdown() as
 * needed.
 */
struct tty_port {
        struct tty_bufhead        buf;
        struct tty_struct        *tty;
        struct tty_struct        *itty;
        const struct tty_port_operations *ops;
        const struct tty_port_client_operations *client_ops;
        spinlock_t                lock;
        int                        blocked_open;
        int                        count;
        wait_queue_head_t        open_wait;
        wait_queue_head_t        delta_msr_wait;
        unsigned long                flags;
        unsigned long                iflags;
        unsigned char                console:1;
        struct mutex                mutex;
        struct mutex                buf_mutex;
        u8                        *xmit_buf;
        DECLARE_KFIFO_PTR(xmit_fifo, u8);
        unsigned int                close_delay;
        unsigned int                closing_wait;
        int                        drain_delay;
        struct kref                kref;
        void                        *client_data;
};

/* tty_port::iflags bits -- use atomic bit ops */
#define TTY_PORT_INITIALIZED        0        /* device is initialized */
#define TTY_PORT_SUSPENDED        1        /* device is suspended */
#define TTY_PORT_ACTIVE                2        /* device is open */

/*
 * uart drivers: use the uart_port::status field and the UPSTAT_* defines
 * for s/w-based flow control steering and carrier detection status
 */
#define TTY_PORT_CTS_FLOW        3        /* h/w flow control enabled */
#define TTY_PORT_CHECK_CD        4        /* carrier detect enabled */
#define TTY_PORT_KOPENED        5        /* device exclusively opened by
                                           kernel */

void tty_port_init(struct tty_port *port);
void tty_port_link_device(struct tty_port *port, struct tty_driver *driver,
                unsigned index);
struct device *tty_port_register_device(struct tty_port *port,
                struct tty_driver *driver, unsigned index,
                struct device *device);
struct device *tty_port_register_device_attr(struct tty_port *port,
                struct tty_driver *driver, unsigned index,
                struct device *device, void *drvdata,
                const struct attribute_group **attr_grp);
struct device *tty_port_register_device_attr_serdev(struct tty_port *port,
                struct tty_driver *driver, unsigned index,
                struct device *host, struct device *parent, void *drvdata,
                const struct attribute_group **attr_grp);
void tty_port_unregister_device(struct tty_port *port,
                struct tty_driver *driver, unsigned index);
int tty_port_alloc_xmit_buf(struct tty_port *port);
void tty_port_free_xmit_buf(struct tty_port *port);
void tty_port_destroy(struct tty_port *port);
void tty_port_put(struct tty_port *port);

static inline struct tty_port *tty_port_get(struct tty_port *port)
{
        if (port && kref_get_unless_zero(&port->kref))
                return port;
        return NULL;
}

/* If the cts flow control is enabled, return true. */
static inline bool tty_port_cts_enabled(const struct tty_port *port)
{
        return test_bit(TTY_PORT_CTS_FLOW, &port->iflags);
}

static inline void tty_port_set_cts_flow(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_CTS_FLOW, &port->iflags, val);
}

static inline bool tty_port_active(const struct tty_port *port)
{
        return test_bit(TTY_PORT_ACTIVE, &port->iflags);
}

static inline void tty_port_set_active(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_ACTIVE, &port->iflags, val);
}

static inline bool tty_port_check_carrier(const struct tty_port *port)
{
        return test_bit(TTY_PORT_CHECK_CD, &port->iflags);
}

static inline void tty_port_set_check_carrier(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_CHECK_CD, &port->iflags, val);
}

static inline bool tty_port_suspended(const struct tty_port *port)
{
        return test_bit(TTY_PORT_SUSPENDED, &port->iflags);
}

static inline void tty_port_set_suspended(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_SUSPENDED, &port->iflags, val);
}

static inline bool tty_port_initialized(const struct tty_port *port)
{
        return test_bit(TTY_PORT_INITIALIZED, &port->iflags);
}

static inline void tty_port_set_initialized(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_INITIALIZED, &port->iflags, val);
}

static inline bool tty_port_kopened(const struct tty_port *port)
{
        return test_bit(TTY_PORT_KOPENED, &port->iflags);
}

static inline void tty_port_set_kopened(struct tty_port *port, bool val)
{
        assign_bit(TTY_PORT_KOPENED, &port->iflags, val);
}

struct tty_struct *tty_port_tty_get(struct tty_port *port);
void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty);
bool tty_port_carrier_raised(struct tty_port *port);
void tty_port_raise_dtr_rts(struct tty_port *port);
void tty_port_lower_dtr_rts(struct tty_port *port);
void tty_port_hangup(struct tty_port *port);
void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async);
void tty_port_tty_wakeup(struct tty_port *port);
int tty_port_block_til_ready(struct tty_port *port, struct tty_struct *tty,
                struct file *filp);
int tty_port_close_start(struct tty_port *port, struct tty_struct *tty,
                struct file *filp);
void tty_port_close_end(struct tty_port *port, struct tty_struct *tty);
void tty_port_close(struct tty_port *port, struct tty_struct *tty,
                struct file *filp);
int tty_port_install(struct tty_port *port, struct tty_driver *driver,
                struct tty_struct *tty);
int tty_port_open(struct tty_port *port, struct tty_struct *tty,
                struct file *filp);

static inline int tty_port_users(struct tty_port *port)
{
        return port->count + port->blocked_open;
}

/**
 * tty_port_tty_hangup - helper to hang up a tty asynchronously
 * @port: tty port
 * @check_clocal: hang only ttys with %CLOCAL unset?
 */
static inline void tty_port_tty_hangup(struct tty_port *port, bool check_clocal)
{
        __tty_port_tty_hangup(port, check_clocal, true);
}

/**
 * tty_port_tty_vhangup - helper to hang up a tty synchronously
 * @port: tty port
 */
static inline void tty_port_tty_vhangup(struct tty_port *port)
{
        __tty_port_tty_hangup(port, false, false);
}

#ifdef CONFIG_TTY
void tty_kref_put(struct tty_struct *tty);
__DEFINE_CLASS_IS_CONDITIONAL(tty_port_tty, true);
__DEFINE_UNLOCK_GUARD(tty_port_tty, struct tty_struct, tty_kref_put(_T->lock));
static inline class_tty_port_tty_t class_tty_port_tty_constructor(struct tty_port *tport)
{
        class_tty_port_tty_t _t = {
                .lock = tty_port_tty_get(tport),
        };
        return _t;
}
#define scoped_tty()        ((struct tty_struct *)(__guard_ptr(tty_port_tty)(&scope)))
#endif

#endif

























































































































































































































































































































































































































































   10 






   10 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2007-2012 Siemens AG
 *
 * Written by:
 * Pavel Smolenskiy <pavel.smolenskiy@gmail.com>
 * Maxim Gorbachyov <maxim.gorbachev@siemens.com>
 * Dmitry Eremin-Solenikov <dbaryshkov@gmail.com>
 * Alexander Smirnov <alex.bluesman.smirnov@gmail.com>
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/crc-ccitt.h>
#include <linux/unaligned.h>

#include <net/mac802154.h>
#include <net/ieee802154_netdev.h>
#include <net/nl802154.h>

#include "ieee802154_i.h"

static int ieee802154_deliver_skb(struct sk_buff *skb)
{
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        skb->protocol = htons(ETH_P_IEEE802154);

        return netif_receive_skb(skb);
}

void mac802154_rx_beacon_worker(struct work_struct *work)
{
        struct ieee802154_local *local =
                container_of(work, struct ieee802154_local, rx_beacon_work);
        struct cfg802154_mac_pkt *mac_pkt;

        mac_pkt = list_first_entry_or_null(&local->rx_beacon_list,
                                           struct cfg802154_mac_pkt, node);
        if (!mac_pkt)
                return;

        mac802154_process_beacon(local, mac_pkt->skb, mac_pkt->page, mac_pkt->channel);

        list_del(&mac_pkt->node);
        kfree_skb(mac_pkt->skb);
        kfree(mac_pkt);
}

static bool mac802154_should_answer_beacon_req(struct ieee802154_local *local)
{
        struct cfg802154_beacon_request *beacon_req;
        unsigned int interval;

        rcu_read_lock();
        beacon_req = rcu_dereference(local->beacon_req);
        if (!beacon_req) {
                rcu_read_unlock();
                return false;
        }

        interval = beacon_req->interval;
        rcu_read_unlock();

        if (!mac802154_is_beaconing(local))
                return false;

        return interval == IEEE802154_ACTIVE_SCAN_DURATION;
}

void mac802154_rx_mac_cmd_worker(struct work_struct *work)
{
        struct ieee802154_local *local =
                container_of(work, struct ieee802154_local, rx_mac_cmd_work);
        struct cfg802154_mac_pkt *mac_pkt;
        u8 mac_cmd;
        int rc;

        mac_pkt = list_first_entry_or_null(&local->rx_mac_cmd_list,
                                           struct cfg802154_mac_pkt, node);
        if (!mac_pkt)
                return;

        rc = ieee802154_get_mac_cmd(mac_pkt->skb, &mac_cmd);
        if (rc)
                goto out;

        switch (mac_cmd) {
        case IEEE802154_CMD_BEACON_REQ:
                dev_dbg(&mac_pkt->sdata->dev->dev, "processing BEACON REQ\n");
                if (!mac802154_should_answer_beacon_req(local))
                        break;

                queue_delayed_work(local->mac_wq, &local->beacon_work, 0);
                break;

        case IEEE802154_CMD_ASSOCIATION_RESP:
                dev_dbg(&mac_pkt->sdata->dev->dev, "processing ASSOC RESP\n");
                if (!mac802154_is_associating(local))
                        break;

                mac802154_process_association_resp(mac_pkt->sdata, mac_pkt->skb);
                break;

        case IEEE802154_CMD_ASSOCIATION_REQ:
                dev_dbg(&mac_pkt->sdata->dev->dev, "processing ASSOC REQ\n");
                if (mac_pkt->sdata->wpan_dev.iftype != NL802154_IFTYPE_COORD)
                        break;

                mac802154_process_association_req(mac_pkt->sdata, mac_pkt->skb);
                break;

        case IEEE802154_CMD_DISASSOCIATION_NOTIFY:
                dev_dbg(&mac_pkt->sdata->dev->dev, "processing DISASSOC NOTIF\n");
                if (mac_pkt->sdata->wpan_dev.iftype != NL802154_IFTYPE_COORD)
                        break;

                mac802154_process_disassociation_notif(mac_pkt->sdata, mac_pkt->skb);
                break;

        default:
                break;
        }

out:
        list_del(&mac_pkt->node);
        kfree_skb(mac_pkt->skb);
        kfree(mac_pkt);
}

static int
ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata,
                       struct sk_buff *skb, const struct ieee802154_hdr *hdr)
{
        struct wpan_phy *wpan_phy = sdata->local->hw.phy;
        struct wpan_dev *wpan_dev = &sdata->wpan_dev;
        struct cfg802154_mac_pkt *mac_pkt;
        __le16 span, sshort;
        int rc;

        pr_debug("getting packet via slave interface %s\n", sdata->dev->name);

        span = wpan_dev->pan_id;
        sshort = wpan_dev->short_addr;

        /* Level 3 filtering: Only beacons are accepted during scans */
        if (sdata->required_filtering == IEEE802154_FILTERING_3_SCAN &&
            sdata->required_filtering > wpan_phy->filtering) {
                if (mac_cb(skb)->type != IEEE802154_FC_TYPE_BEACON) {
                        dev_dbg(&sdata->dev->dev,
                                "drop non-beacon frame (0x%x) during scan\n",
                                mac_cb(skb)->type);
                        goto fail;
                }
        }

        switch (mac_cb(skb)->dest.mode) {
        case IEEE802154_ADDR_NONE:
                if (hdr->source.mode == IEEE802154_ADDR_NONE)
                        /* ACK comes with both addresses empty */
                        skb->pkt_type = PACKET_HOST;
                else if (!wpan_dev->parent)
                        /* No dest means PAN coordinator is the recipient */
                        skb->pkt_type = PACKET_HOST;
                else
                        /* We are not the PAN coordinator, just relaying */
                        skb->pkt_type = PACKET_OTHERHOST;
                break;
        case IEEE802154_ADDR_LONG:
                if (mac_cb(skb)->dest.pan_id != span &&
                    mac_cb(skb)->dest.pan_id != cpu_to_le16(IEEE802154_PANID_BROADCAST))
                        skb->pkt_type = PACKET_OTHERHOST;
                else if (mac_cb(skb)->dest.extended_addr == wpan_dev->extended_addr)
                        skb->pkt_type = PACKET_HOST;
                else
                        skb->pkt_type = PACKET_OTHERHOST;
                break;
        case IEEE802154_ADDR_SHORT:
                if (mac_cb(skb)->dest.pan_id != span &&
                    mac_cb(skb)->dest.pan_id != cpu_to_le16(IEEE802154_PANID_BROADCAST))
                        skb->pkt_type = PACKET_OTHERHOST;
                else if (mac_cb(skb)->dest.short_addr == sshort)
                        skb->pkt_type = PACKET_HOST;
                else if (mac_cb(skb)->dest.short_addr ==
                          cpu_to_le16(IEEE802154_ADDR_BROADCAST))
                        skb->pkt_type = PACKET_BROADCAST;
                else
                        skb->pkt_type = PACKET_OTHERHOST;
                break;
        default:
                pr_debug("invalid dest mode\n");
                goto fail;
        }

        skb->dev = sdata->dev;

        /* TODO this should be moved after netif_receive_skb call, otherwise
         * wireshark will show a mac header with security fields and the
         * payload is already decrypted.
         */
        rc = mac802154_llsec_decrypt(&sdata->sec, skb);
        if (rc) {
                pr_debug("decryption failed: %i\n", rc);
                goto fail;
        }

        sdata->dev->stats.rx_packets++;
        sdata->dev->stats.rx_bytes += skb->len;

        switch (mac_cb(skb)->type) {
        case IEEE802154_FC_TYPE_BEACON:
                dev_dbg(&sdata->dev->dev, "BEACON received\n");
                if (!mac802154_is_scanning(sdata->local))
                        goto fail;

                mac_pkt = kzalloc(sizeof(*mac_pkt), GFP_ATOMIC);
                if (!mac_pkt)
                        goto fail;

                mac_pkt->skb = skb_get(skb);
                mac_pkt->sdata = sdata;
                mac_pkt->page = sdata->local->scan_page;
                mac_pkt->channel = sdata->local->scan_channel;
                list_add_tail(&mac_pkt->node, &sdata->local->rx_beacon_list);
                queue_work(sdata->local->mac_wq, &sdata->local->rx_beacon_work);
                return NET_RX_SUCCESS;

        case IEEE802154_FC_TYPE_MAC_CMD:
                dev_dbg(&sdata->dev->dev, "MAC COMMAND received\n");
                mac_pkt = kzalloc(sizeof(*mac_pkt), GFP_ATOMIC);
                if (!mac_pkt)
                        goto fail;

                mac_pkt->skb = skb_get(skb);
                mac_pkt->sdata = sdata;
                list_add_tail(&mac_pkt->node, &sdata->local->rx_mac_cmd_list);
                queue_work(sdata->local->mac_wq, &sdata->local->rx_mac_cmd_work);
                return NET_RX_SUCCESS;

        case IEEE802154_FC_TYPE_ACK:
                goto fail;

        case IEEE802154_FC_TYPE_DATA:
                return ieee802154_deliver_skb(skb);
        default:
                pr_warn_ratelimited("ieee802154: bad frame received "
                                    "(type = %d)\n", mac_cb(skb)->type);
                goto fail;
        }

fail:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static void
ieee802154_print_addr(const char *name, const struct ieee802154_addr *addr)
{
        if (addr->mode == IEEE802154_ADDR_NONE) {
                pr_debug("%s not present\n", name);
                return;
        }

        pr_debug("%s PAN ID: %04x\n", name, le16_to_cpu(addr->pan_id));
        if (addr->mode == IEEE802154_ADDR_SHORT) {
                pr_debug("%s is short: %04x\n", name,
                         le16_to_cpu(addr->short_addr));
        } else {
                u64 hw = swab64((__force u64)addr->extended_addr);

                pr_debug("%s is hardware: %8phC\n", name, &hw);
        }
}

static int
ieee802154_parse_frame_start(struct sk_buff *skb, struct ieee802154_hdr *hdr)
{
        int hlen;
        struct ieee802154_mac_cb *cb = mac_cb(skb);

        skb_reset_mac_header(skb);

        hlen = ieee802154_hdr_pull(skb, hdr);
        if (hlen < 0)
                return -EINVAL;

        skb->mac_len = hlen;

        pr_debug("fc: %04x dsn: %02x\n", le16_to_cpup((__le16 *)&hdr->fc),
                 hdr->seq);

        cb->type = hdr->fc.type;
        cb->ackreq = hdr->fc.ack_request;
        cb->secen = hdr->fc.security_enabled;

        ieee802154_print_addr("destination", &hdr->dest);
        ieee802154_print_addr("source", &hdr->source);

        cb->source = hdr->source;
        cb->dest = hdr->dest;

        if (hdr->fc.security_enabled) {
                u64 key;

                pr_debug("seclevel %i\n", hdr->sec.level);

                switch (hdr->sec.key_id_mode) {
                case IEEE802154_SCF_KEY_IMPLICIT:
                        pr_debug("implicit key\n");
                        break;

                case IEEE802154_SCF_KEY_INDEX:
                        pr_debug("key %02x\n", hdr->sec.key_id);
                        break;

                case IEEE802154_SCF_KEY_SHORT_INDEX:
                        pr_debug("key %04x:%04x %02x\n",
                                 le32_to_cpu(hdr->sec.short_src) >> 16,
                                 le32_to_cpu(hdr->sec.short_src) & 0xffff,
                                 hdr->sec.key_id);
                        break;

                case IEEE802154_SCF_KEY_HW_INDEX:
                        key = swab64((__force u64)hdr->sec.extended_src);
                        pr_debug("key source %8phC %02x\n", &key,
                                 hdr->sec.key_id);
                        break;
                }
        }

        return 0;
}

static void
__ieee802154_rx_handle_packet(struct ieee802154_local *local,
                              struct sk_buff *skb)
{
        int ret;
        struct ieee802154_sub_if_data *sdata;
        struct ieee802154_hdr hdr;
        struct sk_buff *skb2;

        ret = ieee802154_parse_frame_start(skb, &hdr);
        if (ret) {
                pr_debug("got invalid frame\n");
                return;
        }

        list_for_each_entry_rcu(sdata, &local->interfaces, list) {
                if (sdata->wpan_dev.iftype == NL802154_IFTYPE_MONITOR)
                        continue;

                if (!ieee802154_sdata_running(sdata))
                        continue;

                /* Do not deliver packets received on interfaces expecting
                 * AACK=1 if the address filters where disabled.
                 */
                if (local->hw.phy->filtering < IEEE802154_FILTERING_4_FRAME_FIELDS &&
                    sdata->required_filtering == IEEE802154_FILTERING_4_FRAME_FIELDS)
                        continue;

                skb2 = skb_clone(skb, GFP_ATOMIC);
                if (skb2) {
                        skb2->dev = sdata->dev;
                        ieee802154_subif_frame(sdata, skb2, &hdr);
                }
        }
}

static void
ieee802154_monitors_rx(struct ieee802154_local *local, struct sk_buff *skb)
{
        struct sk_buff *skb2;
        struct ieee802154_sub_if_data *sdata;

        skb_reset_mac_header(skb);
        skb->ip_summed = CHECKSUM_UNNECESSARY;
        skb->pkt_type = PACKET_OTHERHOST;
        skb->protocol = htons(ETH_P_IEEE802154);

        list_for_each_entry_rcu(sdata, &local->interfaces, list) {
                if (sdata->wpan_dev.iftype != NL802154_IFTYPE_MONITOR)
                        continue;

                if (!ieee802154_sdata_running(sdata))
                        continue;

                skb2 = skb_clone(skb, GFP_ATOMIC);
                if (skb2) {
                        skb2->dev = sdata->dev;
                        ieee802154_deliver_skb(skb2);

                        sdata->dev->stats.rx_packets++;
                        sdata->dev->stats.rx_bytes += skb->len;
                }
        }
}

void ieee802154_rx(struct ieee802154_local *local, struct sk_buff *skb)
{
        u16 crc;

        WARN_ON_ONCE(softirq_count() == 0);

        if (local->suspended)
                goto free_skb;

        /* TODO: When a transceiver omits the checksum here, we
         * add an own calculated one. This is currently an ugly
         * solution because the monitor needs a crc here.
         */
        if (local->hw.flags & IEEE802154_HW_RX_OMIT_CKSUM) {
                crc = crc_ccitt(0, skb->data, skb->len);
                put_unaligned_le16(crc, skb_put(skb, 2));
        }

        rcu_read_lock();

        ieee802154_monitors_rx(local, skb);

        /* Level 1 filtering: Check the FCS by software when relevant */
        if (local->hw.phy->filtering == IEEE802154_FILTERING_NONE) {
                crc = crc_ccitt(0, skb->data, skb->len);
                if (crc)
                        goto drop;
        }
        /* remove crc */
        skb_trim(skb, skb->len - 2);

        __ieee802154_rx_handle_packet(local, skb);

drop:
        rcu_read_unlock();
free_skb:
        kfree_skb(skb);
}

void
ieee802154_rx_irqsafe(struct ieee802154_hw *hw, struct sk_buff *skb, u8 lqi)
{
        struct ieee802154_local *local = hw_to_local(hw);
        struct ieee802154_mac_cb *cb = mac_cb_init(skb);

        cb->lqi = lqi;
        skb->pkt_type = IEEE802154_RX_MSG;
        skb_queue_tail(&local->skb_queue, skb);
        tasklet_schedule(&local->tasklet);
}
EXPORT_SYMBOL(ieee802154_rx_irqsafe);















    6 
    1 
    3 
    4 

    6 
    1 
















    4 

    7 
    4 

    6 
    1 











   35 





   10 




   14 

   11 





    6 
   14 
    6 

































    1 




    1 


    1 










   10 
   10 






    7 



    3 

   10 









    4 
    4 








    4 

    4 


















   10 
   10 

    2 











    8 
    7 
    5 
    2 


    4 




    5 




    5 
    3 
    2 
    1 


    4 







   25 









   24 

    8 




    7 


    3 





   12 



    9 
    5 
    4 







    2 





    4 
    1 



    6 







    1 

   10 

   16 





    7 
    7 









    7 








    7 

    4 
    4 









   18 






    1 

   17 

   14 




    9 


    7 






    8 
    4 
    3 







    3 





    5 
    1 



    1 







    3 

    4 

   14 





    9 










    2 

    3 

    4 

    1 









    2 

    4 











    4 


    3 
    3 


    3 


    3 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2023 Isovalent */

#include <linux/bpf.h>
#include <linux/bpf_mprog.h>

static int bpf_mprog_link(struct bpf_tuple *tuple,
                          u32 id_or_fd, u32 flags,
                          enum bpf_prog_type type)
{
        struct bpf_link *link = ERR_PTR(-EINVAL);
        bool id = flags & BPF_F_ID;

        if (id)
                link = bpf_link_by_id(id_or_fd);
        else if (id_or_fd)
                link = bpf_link_get_from_fd(id_or_fd);
        if (IS_ERR(link))
                return PTR_ERR(link);
        if (type && link->prog->type != type) {
                bpf_link_put(link);
                return -EINVAL;
        }

        tuple->link = link;
        tuple->prog = link->prog;
        return 0;
}

static int bpf_mprog_prog(struct bpf_tuple *tuple,
                          u32 id_or_fd, u32 flags,
                          enum bpf_prog_type type)
{
        struct bpf_prog *prog = ERR_PTR(-EINVAL);
        bool id = flags & BPF_F_ID;

        if (id)
                prog = bpf_prog_by_id(id_or_fd);
        else if (id_or_fd)
                prog = bpf_prog_get(id_or_fd);
        if (IS_ERR(prog))
                return PTR_ERR(prog);
        if (type && prog->type != type) {
                bpf_prog_put(prog);
                return -EINVAL;
        }

        tuple->link = NULL;
        tuple->prog = prog;
        return 0;
}

static int bpf_mprog_tuple_relative(struct bpf_tuple *tuple,
                                    u32 id_or_fd, u32 flags,
                                    enum bpf_prog_type type)
{
        bool link = flags & BPF_F_LINK;
        bool id = flags & BPF_F_ID;

        memset(tuple, 0, sizeof(*tuple));
        if (link)
                return bpf_mprog_link(tuple, id_or_fd, flags, type);
        /* If no relevant flag is set and no id_or_fd was passed, then
         * tuple link/prog is just NULLed. This is the case when before/
         * after selects first/last position without passing fd.
         */
        if (!id && !id_or_fd)
                return 0;
        return bpf_mprog_prog(tuple, id_or_fd, flags, type);
}

static void bpf_mprog_tuple_put(struct bpf_tuple *tuple)
{
        if (tuple->link)
                bpf_link_put(tuple->link);
        else if (tuple->prog)
                bpf_prog_put(tuple->prog);
}

/* The bpf_mprog_{replace,delete}() operate on exact idx position with the
 * one exception that for deletion we support delete from front/back. In
 * case of front idx is -1, in case of back idx is bpf_mprog_total(entry).
 * Adjustment to first and last entry is trivial. The bpf_mprog_insert()
 * we have to deal with the following cases:
 *
 * idx + before:
 *
 * Insert P4 before P3: idx for old array is 1, idx for new array is 2,
 * hence we adjust target idx for the new array, so that memmove copies
 * P1 and P2 to the new entry, and we insert P4 into idx 2. Inserting
 * before P1 would have old idx -1 and new idx 0.
 *
 * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 * |P1|P2|P3| ==> |P1|P2|  |P3| ==> |P1|P2|P4|P3|
 * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 *
 * idx + after:
 *
 * Insert P4 after P2: idx for old array is 2, idx for new array is 2.
 * Again, memmove copies P1 and P2 to the new entry, and we insert P4
 * into idx 2. Inserting after P3 would have both old/new idx at 4 aka
 * bpf_mprog_total(entry).
 *
 * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 * |P1|P2|P3| ==> |P1|P2|  |P3| ==> |P1|P2|P4|P3|
 * +--+--+--+     +--+--+--+--+     +--+--+--+--+
 */
static int bpf_mprog_replace(struct bpf_mprog_entry *entry,
                             struct bpf_mprog_entry **entry_new,
                             struct bpf_tuple *ntuple, int idx)
{
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;
        struct bpf_prog *oprog;

        bpf_mprog_read(entry, idx, &fp, &cp);
        oprog = READ_ONCE(fp->prog);
        bpf_mprog_write(fp, cp, ntuple);
        if (!ntuple->link) {
                WARN_ON_ONCE(cp->link);
                bpf_prog_put(oprog);
        }
        *entry_new = entry;
        return 0;
}

static int bpf_mprog_insert(struct bpf_mprog_entry *entry,
                            struct bpf_mprog_entry **entry_new,
                            struct bpf_tuple *ntuple, int idx, u32 flags)
{
        int total = bpf_mprog_total(entry);
        struct bpf_mprog_entry *peer;
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;

        peer = bpf_mprog_peer(entry);
        bpf_mprog_entry_copy(peer, entry);
        if (idx == total)
                goto insert;
        else if (flags & BPF_F_BEFORE)
                idx += 1;
        bpf_mprog_entry_grow(peer, idx);
insert:
        bpf_mprog_read(peer, idx, &fp, &cp);
        bpf_mprog_write(fp, cp, ntuple);
        bpf_mprog_inc(peer);
        *entry_new = peer;
        return 0;
}

static int bpf_mprog_delete(struct bpf_mprog_entry *entry,
                            struct bpf_mprog_entry **entry_new,
                            struct bpf_tuple *dtuple, int idx)
{
        int total = bpf_mprog_total(entry);
        struct bpf_mprog_entry *peer;

        peer = bpf_mprog_peer(entry);
        bpf_mprog_entry_copy(peer, entry);
        if (idx == -1)
                idx = 0;
        else if (idx == total)
                idx = total - 1;
        bpf_mprog_entry_shrink(peer, idx);
        bpf_mprog_dec(peer);
        bpf_mprog_mark_for_release(peer, dtuple);
        *entry_new = peer;
        return 0;
}

/* In bpf_mprog_pos_*() we evaluate the target position for the BPF
 * program/link that needs to be replaced, inserted or deleted for
 * each "rule" independently. If all rules agree on that position
 * or existing element, then enact replacement, addition or deletion.
 * If this is not the case, then the request cannot be satisfied and
 * we bail out with an error.
 */
static int bpf_mprog_pos_exact(struct bpf_mprog_entry *entry,
                               struct bpf_tuple *tuple)
{
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;
        int i;

        for (i = 0; i < bpf_mprog_total(entry); i++) {
                bpf_mprog_read(entry, i, &fp, &cp);
                if (tuple->prog == READ_ONCE(fp->prog))
                        return tuple->link == cp->link ? i : -EBUSY;
        }
        return -ENOENT;
}

static int bpf_mprog_pos_before(struct bpf_mprog_entry *entry,
                                struct bpf_tuple *tuple)
{
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;
        int i;

        for (i = 0; i < bpf_mprog_total(entry); i++) {
                bpf_mprog_read(entry, i, &fp, &cp);
                if (tuple->prog == READ_ONCE(fp->prog) &&
                    (!tuple->link || tuple->link == cp->link))
                        return i - 1;
        }
        return tuple->prog ? -ENOENT : -1;
}

static int bpf_mprog_pos_after(struct bpf_mprog_entry *entry,
                               struct bpf_tuple *tuple)
{
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;
        int i;

        for (i = 0; i < bpf_mprog_total(entry); i++) {
                bpf_mprog_read(entry, i, &fp, &cp);
                if (tuple->prog == READ_ONCE(fp->prog) &&
                    (!tuple->link || tuple->link == cp->link))
                        return i + 1;
        }
        return tuple->prog ? -ENOENT : bpf_mprog_total(entry);
}

int bpf_mprog_attach(struct bpf_mprog_entry *entry,
                     struct bpf_mprog_entry **entry_new,
                     struct bpf_prog *prog_new, struct bpf_link *link,
                     struct bpf_prog *prog_old,
                     u32 flags, u32 id_or_fd, u64 revision)
{
        struct bpf_tuple rtuple, ntuple = {
                .prog = prog_new,
                .link = link,
        }, otuple = {
                .prog = prog_old,
                .link = link,
        };
        int ret, idx = -ERANGE, tidx;

        if (revision && revision != bpf_mprog_revision(entry))
                return -ESTALE;
        if (bpf_mprog_exists(entry, prog_new))
                return -EEXIST;
        ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd,
                                       flags & ~BPF_F_REPLACE,
                                       prog_new->type);
        if (ret)
                return ret;
        if (flags & BPF_F_REPLACE) {
                tidx = bpf_mprog_pos_exact(entry, &otuple);
                if (tidx < 0) {
                        ret = tidx;
                        goto out;
                }
                idx = tidx;
        } else if (bpf_mprog_total(entry) == bpf_mprog_max()) {
                ret = -ERANGE;
                goto out;
        }
        if (flags & BPF_F_BEFORE) {
                tidx = bpf_mprog_pos_before(entry, &rtuple);
                if (tidx < -1 || (idx >= -1 && tidx != idx)) {
                        ret = tidx < -1 ? tidx : -ERANGE;
                        goto out;
                }
                idx = tidx;
        }
        if (flags & BPF_F_AFTER) {
                tidx = bpf_mprog_pos_after(entry, &rtuple);
                if (tidx < -1 || (idx >= -1 && tidx != idx)) {
                        ret = tidx < 0 ? tidx : -ERANGE;
                        goto out;
                }
                idx = tidx;
        }
        if (idx < -1) {
                if (rtuple.prog || flags) {
                        ret = -EINVAL;
                        goto out;
                }
                idx = bpf_mprog_total(entry);
                flags = BPF_F_AFTER;
        }
        if (idx >= bpf_mprog_max()) {
                ret = -ERANGE;
                goto out;
        }
        if (flags & BPF_F_REPLACE)
                ret = bpf_mprog_replace(entry, entry_new, &ntuple, idx);
        else
                ret = bpf_mprog_insert(entry, entry_new, &ntuple, idx, flags);
out:
        bpf_mprog_tuple_put(&rtuple);
        return ret;
}

static int bpf_mprog_fetch(struct bpf_mprog_entry *entry,
                           struct bpf_tuple *tuple, int idx)
{
        int total = bpf_mprog_total(entry);
        struct bpf_mprog_cp *cp;
        struct bpf_mprog_fp *fp;
        struct bpf_prog *prog;
        struct bpf_link *link;

        if (idx == -1)
                idx = 0;
        else if (idx == total)
                idx = total - 1;
        bpf_mprog_read(entry, idx, &fp, &cp);
        prog = READ_ONCE(fp->prog);
        link = cp->link;
        /* The deletion request can either be without filled tuple in which
         * case it gets populated here based on idx, or with filled tuple
         * where the only thing we end up doing is the WARN_ON_ONCE() assert.
         * If we hit a BPF link at the given index, it must not be removed
         * from opts path.
         */
        if (link && !tuple->link)
                return -EBUSY;
        WARN_ON_ONCE(tuple->prog && tuple->prog != prog);
        WARN_ON_ONCE(tuple->link && tuple->link != link);
        tuple->prog = prog;
        tuple->link = link;
        return 0;
}

int bpf_mprog_detach(struct bpf_mprog_entry *entry,
                     struct bpf_mprog_entry **entry_new,
                     struct bpf_prog *prog, struct bpf_link *link,
                     u32 flags, u32 id_or_fd, u64 revision)
{
        struct bpf_tuple rtuple, dtuple = {
                .prog = prog,
                .link = link,
        };
        int ret, idx = -ERANGE, tidx;

        if (flags & BPF_F_REPLACE)
                return -EINVAL;
        if (revision && revision != bpf_mprog_revision(entry))
                return -ESTALE;
        if (!bpf_mprog_total(entry))
                return -ENOENT;
        ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, flags,
                                       prog ? prog->type :
                                       BPF_PROG_TYPE_UNSPEC);
        if (ret)
                return ret;
        if (dtuple.prog) {
                tidx = bpf_mprog_pos_exact(entry, &dtuple);
                if (tidx < 0) {
                        ret = tidx;
                        goto out;
                }
                idx = tidx;
        }
        if (flags & BPF_F_BEFORE) {
                tidx = bpf_mprog_pos_before(entry, &rtuple);
                if (tidx < -1 || (idx >= -1 && tidx != idx)) {
                        ret = tidx < -1 ? tidx : -ERANGE;
                        goto out;
                }
                idx = tidx;
        }
        if (flags & BPF_F_AFTER) {
                tidx = bpf_mprog_pos_after(entry, &rtuple);
                if (tidx < -1 || (idx >= -1 && tidx != idx)) {
                        ret = tidx < 0 ? tidx : -ERANGE;
                        goto out;
                }
                idx = tidx;
        }
        if (idx < -1) {
                if (rtuple.prog || flags) {
                        ret = -EINVAL;
                        goto out;
                }
                idx = bpf_mprog_total(entry);
                flags = BPF_F_AFTER;
        }
        if (idx >= bpf_mprog_max()) {
                ret = -ERANGE;
                goto out;
        }
        ret = bpf_mprog_fetch(entry, &dtuple, idx);
        if (ret)
                goto out;
        ret = bpf_mprog_delete(entry, entry_new, &dtuple, idx);
out:
        bpf_mprog_tuple_put(&rtuple);
        return ret;
}

int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr,
                    struct bpf_mprog_entry *entry)
{
        u32 __user *uprog_flags, *ulink_flags;
        u32 __user *uprog_id, *ulink_id;
        struct bpf_mprog_fp *fp;
        struct bpf_mprog_cp *cp;
        struct bpf_prog *prog;
        const u32 flags = 0;
        u32 id, count = 0;
        u64 revision = 1;
        int i, ret = 0;

        if (attr->query.query_flags || attr->query.attach_flags)
                return -EINVAL;
        if (entry) {
                revision = bpf_mprog_revision(entry);
                count = bpf_mprog_total(entry);
        }
        if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
                return -EFAULT;
        if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
                return -EFAULT;
        if (copy_to_user(&uattr->query.count, &count, sizeof(count)))
                return -EFAULT;
        uprog_id = u64_to_user_ptr(attr->query.prog_ids);
        uprog_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
        ulink_id = u64_to_user_ptr(attr->query.link_ids);
        ulink_flags = u64_to_user_ptr(attr->query.link_attach_flags);
        if (attr->query.count == 0 || !uprog_id || !count)
                return 0;
        if (attr->query.count < count) {
                count = attr->query.count;
                ret = -ENOSPC;
        }
        for (i = 0; i < bpf_mprog_max(); i++) {
                bpf_mprog_read(entry, i, &fp, &cp);
                prog = READ_ONCE(fp->prog);
                if (!prog)
                        break;
                id = prog->aux->id;
                if (copy_to_user(uprog_id + i, &id, sizeof(id)))
                        return -EFAULT;
                if (uprog_flags &&
                    copy_to_user(uprog_flags + i, &flags, sizeof(flags)))
                        return -EFAULT;
                id = cp->link ? cp->link->id : 0;
                if (ulink_id &&
                    copy_to_user(ulink_id + i, &id, sizeof(id)))
                        return -EFAULT;
                if (ulink_flags &&
                    copy_to_user(ulink_flags + i, &flags, sizeof(flags)))
                        return -EFAULT;
                if (i + 1 == count)
                        break;
        }
        return ret;
}


























  361 





























































  361 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_SYNC_CORE_H
#define _ASM_X86_SYNC_CORE_H

#include <linux/preempt.h>
#include <asm/processor.h>
#include <asm/cpufeature.h>
#include <asm/special_insns.h>

#ifdef CONFIG_X86_32
static __always_inline void iret_to_self(void)
{
        asm volatile (
                "pushfl\n\t"
                "pushl %%cs\n\t"
                "pushl $1f\n\t"
                "iret\n\t"
                "1:"
                : ASM_CALL_CONSTRAINT : : "memory");
}
#else
static __always_inline void iret_to_self(void)
{
        unsigned int tmp;

        asm volatile (
                "mov %%ss, %0\n\t"
                "pushq %q0\n\t"
                "pushq %%rsp\n\t"
                "addq $8, (%%rsp)\n\t"
                "pushfq\n\t"
                "mov %%cs, %0\n\t"
                "pushq %q0\n\t"
                "pushq $1f\n\t"
                "iretq\n\t"
                "1:"
                : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
}
#endif /* CONFIG_X86_32 */

/*
 * This function forces the icache and prefetched instruction stream to
 * catch up with reality in two very specific cases:
 *
 *  a) Text was modified using one virtual address and is about to be executed
 *     from the same physical page at a different virtual address.
 *
 *  b) Text was modified on a different CPU, may subsequently be
 *     executed on this CPU, and you want to make sure the new version
 *     gets executed.  This generally means you're calling this in an IPI.
 *
 * If you're calling this for a different reason, you're probably doing
 * it wrong.
 *
 * Like all of Linux's memory ordering operations, this is a
 * compiler barrier as well.
 */
static __always_inline void sync_core(void)
{
        /*
         * The SERIALIZE instruction is the most straightforward way to
         * do this, but it is not universally available.
         */
        if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
                serialize();
                return;
        }

        /*
         * For all other processors, there are quite a few ways to do this.
         * IRET-to-self is nice because it works on every CPU, at any CPL
         * (so it's compatible with paravirtualization), and it never exits
         * to a hypervisor.  The only downsides are that it's a bit slow
         * (it seems to be a bit more than 2x slower than the fastest
         * options) and that it unmasks NMIs.  The "push %cs" is needed,
         * because in paravirtual environments __KERNEL_CS may not be a
         * valid CS value when we do IRET directly.
         *
         * In case NMI unmasking or performance ever becomes a problem,
         * the next best option appears to be MOV-to-CR2 and an
         * unconditional jump.  That sequence also works on all CPUs,
         * but it will fault at CPL3 (i.e. Xen PV).
         *
         * CPUID is the conventional way, but it's nasty: it doesn't
         * exist on some 486-like CPUs, and it usually exits to a
         * hypervisor.
         */
        iret_to_self();
}

/*
 * Ensure that a core serializing instruction is issued before returning
 * to user-mode. x86 implements return to user-space through sysexit,
 * sysrel, and sysretq, which are not core serializing.
 */
static inline void sync_core_before_usermode(void)
{
        /* With PTI, we unconditionally serialize before running user code. */
        if (static_cpu_has(X86_FEATURE_PTI))
                return;

        /*
         * Even if we're in an interrupt, we might reschedule before returning,
         * in which case we could switch to a different thread in the same mm
         * and return using SYSRET or SYSEXIT.  Instead of trying to keep
         * track of our need to sync the core, just sync right away.
         */
        sync_core();
}

#endif /* _ASM_X86_SYNC_CORE_H */


























    5 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
// SPDX-License-Identifier: GPL-2.0-only
/* Accounting handling for netfilter. */

/*
 * (C) 2008 Krzysztof Piotr Oledzki <ole@ans.pl>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/netfilter.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/export.h>

#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_extend.h>
#include <net/netfilter/nf_conntrack_acct.h>

static bool nf_ct_acct __read_mostly;

module_param_named(acct, nf_ct_acct, bool, 0644);
MODULE_PARM_DESC(acct, "Enable connection tracking flow accounting.");

void nf_conntrack_acct_pernet_init(struct net *net)
{
        net->ct.sysctl_acct = nf_ct_acct;
}









































   20 















   19 

    1 



    1 






   20 




   20 












   20 























   20 



   20 
   20 

   20 
   20 











    9 



   10 
   10 










   27 

















   25 




   25 
   25 
    8 




   20 





    1 



    1 



   16 





    1 


    1 






    2 
    2 



    2 


    2 

    1 



    2 

    2 






   18 



    1 






   17 




















    9 















    9 








    1 




    1 








    1 















    1 



    1 



    1 




















    1 



    1 





    1 



    1 


    1 






    1 






    1 
    1 






    1 




    1 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_prio.c        Simple 3-band priority "scheduler".
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 * Fixes:       19990609: J Hadi Salim <hadi@nortelnetworks.com>:
 *              Init --  EINVAL when opt undefined
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>

struct prio_sched_data {
        int bands;
        struct tcf_proto __rcu *filter_list;
        struct tcf_block *block;
        u8  prio2band[TC_PRIO_MAX+1];
        struct Qdisc *queues[TCQ_PRIO_BANDS];
};


static struct Qdisc *
prio_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        u32 band = skb->priority;
        struct tcf_result res;
        struct tcf_proto *fl;
        int err;

        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
        if (TC_H_MAJ(skb->priority) != sch->handle) {
                fl = rcu_dereference_bh(q->filter_list);
                err = tcf_classify(skb, NULL, fl, &res, false);
#ifdef CONFIG_NET_CLS_ACT
                switch (err) {
                case TC_ACT_STOLEN:
                case TC_ACT_QUEUED:
                case TC_ACT_TRAP:
                        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
                        fallthrough;
                case TC_ACT_SHOT:
                        return NULL;
                }
#endif
                if (!fl || err < 0) {
                        if (TC_H_MAJ(band))
                                band = 0;
                        return q->queues[q->prio2band[band & TC_PRIO_MAX]];
                }
                band = res.classid;
        }
        band = TC_H_MIN(band) - 1;
        if (band >= q->bands)
                return q->queues[q->prio2band[0]];

        return q->queues[band];
}

static int
prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
{
        unsigned int len = qdisc_pkt_len(skb);
        struct Qdisc *qdisc;
        int ret;

        qdisc = prio_classify(skb, sch, &ret);
#ifdef CONFIG_NET_CLS_ACT
        if (qdisc == NULL) {

                if (ret & __NET_XMIT_BYPASS)
                        qdisc_qstats_drop(sch);
                __qdisc_drop(skb, to_free);
                return ret;
        }
#endif

        ret = qdisc_enqueue(skb, qdisc, to_free);
        if (ret == NET_XMIT_SUCCESS) {
                sch->qstats.backlog += len;
                sch->q.qlen++;
                return NET_XMIT_SUCCESS;
        }
        if (net_xmit_drop_count(ret))
                qdisc_qstats_drop(sch);
        return ret;
}

static struct sk_buff *prio_peek(struct Qdisc *sch)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        for (prio = 0; prio < q->bands; prio++) {
                struct Qdisc *qdisc = q->queues[prio];
                struct sk_buff *skb = qdisc->ops->peek(qdisc);
                if (skb)
                        return skb;
        }
        return NULL;
}

static struct sk_buff *prio_dequeue(struct Qdisc *sch)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        for (prio = 0; prio < q->bands; prio++) {
                struct Qdisc *qdisc = q->queues[prio];
                struct sk_buff *skb = qdisc_dequeue_peeked(qdisc);
                if (skb) {
                        qdisc_bstats_update(sch, skb);
                        qdisc_qstats_backlog_dec(sch, skb);
                        sch->q.qlen--;
                        return skb;
                }
        }
        return NULL;

}

static void
prio_reset(struct Qdisc *sch)
{
        int prio;
        struct prio_sched_data *q = qdisc_priv(sch);

        for (prio = 0; prio < q->bands; prio++)
                qdisc_reset(q->queues[prio]);
}

static int prio_offload(struct Qdisc *sch, struct tc_prio_qopt *qopt)
{
        struct net_device *dev = qdisc_dev(sch);
        struct tc_prio_qopt_offload opt = {
                .handle = sch->handle,
                .parent = sch->parent,
        };

        if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
                return -EOPNOTSUPP;

        if (qopt) {
                opt.command = TC_PRIO_REPLACE;
                opt.replace_params.bands = qopt->bands;
                memcpy(&opt.replace_params.priomap, qopt->priomap,
                       TC_PRIO_MAX + 1);
                opt.replace_params.qstats = &sch->qstats;
        } else {
                opt.command = TC_PRIO_DESTROY;
        }

        return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, &opt);
}

static void
prio_destroy(struct Qdisc *sch)
{
        int prio;
        struct prio_sched_data *q = qdisc_priv(sch);

        tcf_block_put(q->block);
        prio_offload(sch, NULL);
        for (prio = 0; prio < q->bands; prio++)
                qdisc_put(q->queues[prio]);
}

static int prio_tune(struct Qdisc *sch, struct nlattr *opt,
                     struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct Qdisc *queues[TCQ_PRIO_BANDS];
        int oldbands = q->bands, i;
        struct tc_prio_qopt *qopt;

        if (nla_len(opt) < sizeof(*qopt))
                return -EINVAL;
        qopt = nla_data(opt);

        if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < TCQ_MIN_PRIO_BANDS)
                return -EINVAL;

        for (i = 0; i <= TC_PRIO_MAX; i++) {
                if (qopt->priomap[i] >= qopt->bands)
                        return -EINVAL;
        }

        /* Before commit, make sure we can allocate all new qdiscs */
        for (i = oldbands; i < qopt->bands; i++) {
                queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                              TC_H_MAKE(sch->handle, i + 1),
                                              extack);
                if (!queues[i]) {
                        while (i > oldbands)
                                qdisc_put(queues[--i]);
                        return -ENOMEM;
                }
        }

        prio_offload(sch, qopt);
        sch_tree_lock(sch);
        q->bands = qopt->bands;
        memcpy(q->prio2band, qopt->priomap, TC_PRIO_MAX+1);

        for (i = q->bands; i < oldbands; i++)
                qdisc_purge_queue(q->queues[i]);

        for (i = oldbands; i < q->bands; i++) {
                q->queues[i] = queues[i];
                if (q->queues[i] != &noop_qdisc)
                        qdisc_hash_add(q->queues[i], true);
        }

        sch_tree_unlock(sch);

        for (i = q->bands; i < oldbands; i++)
                qdisc_put(q->queues[i]);
        return 0;
}

static int prio_init(struct Qdisc *sch, struct nlattr *opt,
                     struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int err;

        if (!opt)
                return -EINVAL;

        err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
        if (err)
                return err;

        return prio_tune(sch, opt, extack);
}

static int prio_dump_offload(struct Qdisc *sch)
{
        struct tc_prio_qopt_offload hw_stats = {
                .command = TC_PRIO_STATS,
                .handle = sch->handle,
                .parent = sch->parent,
                {
                        .stats = {
                                .bstats = &sch->bstats,
                                .qstats = &sch->qstats,
                        },
                },
        };

        return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_PRIO, &hw_stats);
}

static int prio_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned char *b = skb_tail_pointer(skb);
        struct tc_prio_qopt opt;
        int err;

        opt.bands = q->bands;
        memcpy(&opt.priomap, q->prio2band, TC_PRIO_MAX + 1);

        err = prio_dump_offload(sch);
        if (err)
                goto nla_put_failure;

        if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nlmsg_trim(skb, b);
        return -1;
}

static int prio_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                      struct Qdisc **old, struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct tc_prio_qopt_offload graft_offload;
        unsigned long band = arg - 1;

        if (!new) {
                new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                        TC_H_MAKE(sch->handle, arg), extack);
                if (!new)
                        new = &noop_qdisc;
                else
                        qdisc_hash_add(new, true);
        }

        *old = qdisc_replace(sch, new, &q->queues[band]);

        graft_offload.handle = sch->handle;
        graft_offload.parent = sch->parent;
        graft_offload.graft_params.band = band;
        graft_offload.graft_params.child_handle = new->handle;
        graft_offload.command = TC_PRIO_GRAFT;

        qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old,
                                   TC_SETUP_QDISC_PRIO, &graft_offload,
                                   extack);
        return 0;
}

static struct Qdisc *
prio_leaf(struct Qdisc *sch, unsigned long arg)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned long band = arg - 1;

        return q->queues[band];
}

static unsigned long prio_find(struct Qdisc *sch, u32 classid)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        unsigned long band = TC_H_MIN(classid);

        if (band - 1 >= q->bands)
                return 0;
        return band;
}

static unsigned long prio_bind(struct Qdisc *sch, unsigned long parent, u32 classid)
{
        return prio_find(sch, classid);
}


static void prio_unbind(struct Qdisc *q, unsigned long cl)
{
}

static int prio_dump_class(struct Qdisc *sch, unsigned long cl, struct sk_buff *skb,
                           struct tcmsg *tcm)
{
        struct prio_sched_data *q = qdisc_priv(sch);

        tcm->tcm_handle |= TC_H_MIN(cl);
        tcm->tcm_info = q->queues[cl-1]->handle;
        return 0;
}

static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                 struct gnet_dump *d)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        struct Qdisc *cl_q;

        cl_q = q->queues[cl - 1];
        if (gnet_stats_copy_basic(d, cl_q->cpu_bstats,
                                  &cl_q->bstats, true) < 0 ||
            qdisc_qstats_copy(d, cl_q) < 0)
                return -1;

        return 0;
}

static void prio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
        struct prio_sched_data *q = qdisc_priv(sch);
        int prio;

        if (arg->stop)
                return;

        for (prio = 0; prio < q->bands; prio++) {
                if (!tc_qdisc_stats_dump(sch, prio + 1, arg))
                        break;
        }
}

static struct tcf_block *prio_tcf_block(struct Qdisc *sch, unsigned long cl,
                                        struct netlink_ext_ack *extack)
{
        struct prio_sched_data *q = qdisc_priv(sch);

        if (cl)
                return NULL;
        return q->block;
}

static const struct Qdisc_class_ops prio_class_ops = {
        .graft                =        prio_graft,
        .leaf                =        prio_leaf,
        .find                =        prio_find,
        .walk                =        prio_walk,
        .tcf_block        =        prio_tcf_block,
        .bind_tcf        =        prio_bind,
        .unbind_tcf        =        prio_unbind,
        .dump                =        prio_dump_class,
        .dump_stats        =        prio_dump_class_stats,
};

static struct Qdisc_ops prio_qdisc_ops __read_mostly = {
        .next                =        NULL,
        .cl_ops                =        &prio_class_ops,
        .id                =        "prio",
        .priv_size        =        sizeof(struct prio_sched_data),
        .enqueue        =        prio_enqueue,
        .dequeue        =        prio_dequeue,
        .peek                =        prio_peek,
        .init                =        prio_init,
        .reset                =        prio_reset,
        .destroy        =        prio_destroy,
        .change                =        prio_tune,
        .dump                =        prio_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("prio");

static int __init prio_module_init(void)
{
        return register_qdisc(&prio_qdisc_ops);
}

static void __exit prio_module_exit(void)
{
        unregister_qdisc(&prio_qdisc_ops);
}

module_init(prio_module_init)
module_exit(prio_module_exit)

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Simple 3-band priority qdisc");





















































































































































































































    5 














    5 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _SCSI_DISK_H
#define _SCSI_DISK_H

/*
 * More than enough for everybody ;)  The huge number of majors
 * is a leftover from 16bit dev_t days, we don't really need that
 * much numberspace.
 */
#define SD_MAJORS        16

/*
 * Time out in seconds for disks and Magneto-opticals (which are slower).
 */
#define SD_TIMEOUT                (30 * HZ)
#define SD_MOD_TIMEOUT                (75 * HZ)
/*
 * Flush timeout is a multiplier over the standard device timeout which is
 * user modifiable via sysfs but initially set to SD_TIMEOUT
 */
#define SD_FLUSH_TIMEOUT_MULTIPLIER        2
#define SD_WRITE_SAME_TIMEOUT        (120 * HZ)

/*
 * Number of allowed retries
 */
#define SD_MAX_RETRIES                5
#define SD_PASSTHROUGH_RETRIES        1
#define SD_MAX_MEDIUM_TIMEOUTS        2

/*
 * Size of the initial data buffer for mode and read capacity data
 */
#define SD_BUF_SIZE                512

/*
 * Number of sectors at the end of the device to avoid multi-sector
 * accesses to in the case of last_sector_bug
 */
#define SD_LAST_BUGGY_SECTORS        8

enum {
        SD_EXT_CDB_SIZE = 32,        /* Extended CDB size */
        SD_MEMPOOL_SIZE = 2,        /* CDB pool size */
};

enum {
        SD_DEF_XFER_BLOCKS = 0xffff,
        SD_MAX_XFER_BLOCKS = 0xffffffff,
        SD_MAX_WS10_BLOCKS = 0xffff,
        SD_MAX_WS16_BLOCKS = 0x7fffff,
};

enum {
        SD_LBP_FULL = 0,        /* Full logical block provisioning */
        SD_LBP_UNMAP,                /* Use UNMAP command */
        SD_LBP_WS16,                /* Use WRITE SAME(16) with UNMAP bit */
        SD_LBP_WS10,                /* Use WRITE SAME(10) with UNMAP bit */
        SD_LBP_ZERO,                /* Use WRITE SAME(10) with zero payload */
        SD_LBP_DISABLE,                /* Discard disabled due to failed cmd */
};

enum {
        SD_ZERO_WRITE = 0,        /* Use WRITE(10/16) command */
        SD_ZERO_WS,                /* Use WRITE SAME(10/16) command */
        SD_ZERO_WS16_UNMAP,        /* Use WRITE SAME(16) with UNMAP */
        SD_ZERO_WS10_UNMAP,        /* Use WRITE SAME(10) with UNMAP */
};

/**
 * struct zoned_disk_info - Specific properties of a ZBC SCSI device.
 * @nr_zones: number of zones.
 * @zone_blocks: number of logical blocks per zone.
 *
 * This data structure holds the ZBC SCSI device properties that are retrieved
 * twice: a first time before the gendisk capacity is known and a second time
 * after the gendisk capacity is known.
 */
struct zoned_disk_info {
        u32                nr_zones;
        u32                zone_blocks;
};

struct scsi_disk {
        struct scsi_device *device;

        /*
         * disk_dev is used to show attributes in /sys/class/scsi_disk/,
         * but otherwise not really needed.  Do not use for refcounting.
         */
        struct device        disk_dev;
        struct gendisk        *disk;
        struct opal_dev *opal_dev;
#ifdef CONFIG_BLK_DEV_ZONED
        /* Updated during revalidation before the gendisk capacity is known. */
        struct zoned_disk_info        early_zone_info;
        /* Updated during revalidation after the gendisk capacity is known. */
        struct zoned_disk_info        zone_info;
        u32                zones_optimal_open;
        u32                zones_optimal_nonseq;
        u32                zones_max_open;
        /*
         * Either zero or a power of two. If not zero it means that the offset
         * between zone starting LBAs is constant.
         */
        u32                zone_starting_lba_gran;
#endif
        atomic_t        openers;
        sector_t        capacity;        /* size in logical blocks */
        int                max_retries;
        u32                min_xfer_blocks;
        u32                max_xfer_blocks;
        u32                opt_xfer_blocks;
        u32                max_ws_blocks;
        u32                max_unmap_blocks;
        u32                unmap_granularity;
        u32                unmap_alignment;

        u32                max_atomic;
        u32                atomic_alignment;
        u32                atomic_granularity;
        u32                max_atomic_with_boundary;
        u32                max_atomic_boundary;

        u32                index;
        unsigned int        physical_block_size;
        unsigned int        max_medium_access_timeouts;
        unsigned int        medium_access_timed_out;
                        /* number of permanent streams */
        u16                permanent_stream_count;
        u8                media_present;
        u8                write_prot;
        u8                protection_type;/* Data Integrity Field */
        u8                provisioning_mode;
        u8                zeroing_mode;
        u8                nr_actuators;                /* Number of actuators */
        bool                suspended;        /* Disk is suspended (stopped) */
        unsigned        ATO : 1;        /* state of disk ATO bit */
        unsigned        cache_override : 1; /* temp override of WCE,RCD */
        unsigned        WCE : 1;        /* state of disk WCE bit */
        unsigned        RCD : 1;        /* state of disk RCD bit, unused */
        unsigned        DPOFUA : 1;        /* state of disk DPOFUA bit */
        unsigned        first_scan : 1;
        unsigned        lbpme : 1;
        unsigned        lbprz : 1;
        unsigned        lbpu : 1;
        unsigned        lbpws : 1;
        unsigned        lbpws10 : 1;
        unsigned        lbpvpd : 1;
        unsigned        ws10 : 1;
        unsigned        ws16 : 1;
        unsigned        rc_basis: 2;
        unsigned        zoned: 2;
        unsigned        urswrz : 1;
        unsigned        security : 1;
        unsigned        ignore_medium_access_errors : 1;
        unsigned        rscs : 1; /* reduced stream control support */
        unsigned        use_atomic_write_boundary : 1;
};
#define to_scsi_disk(obj) container_of(obj, struct scsi_disk, disk_dev)

static inline struct scsi_disk *scsi_disk(struct gendisk *disk)
{
        return disk->private_data;
}

#define sd_printk(prefix, sdsk, fmt, a...)                                \
        (sdsk)->disk ?                                                        \
              sdev_prefix_printk(prefix, (sdsk)->device,                \
                                 (sdsk)->disk->disk_name, fmt, ##a) :        \
              sdev_printk(prefix, (sdsk)->device, fmt, ##a)

#define sd_first_printk(prefix, sdsk, fmt, a...)                        \
        do {                                                                \
                if ((sdsk)->first_scan)                                        \
                        sd_printk(prefix, sdsk, fmt, ##a);                \
        } while (0)

static inline int scsi_medium_access_command(struct scsi_cmnd *scmd)
{
        switch (scmd->cmnd[0]) {
        case READ_6:
        case READ_10:
        case READ_12:
        case READ_16:
        case SYNCHRONIZE_CACHE:
        case VERIFY:
        case VERIFY_12:
        case VERIFY_16:
        case WRITE_6:
        case WRITE_10:
        case WRITE_12:
        case WRITE_16:
        case WRITE_SAME:
        case WRITE_SAME_16:
        case UNMAP:
                return 1;
        case VARIABLE_LENGTH_CMD:
                switch (scmd->cmnd[9]) {
                case READ_32:
                case VERIFY_32:
                case WRITE_32:
                case WRITE_SAME_32:
                        return 1;
                }
        }

        return 0;
}

static inline sector_t logical_to_sectors(struct scsi_device *sdev, sector_t blocks)
{
        return blocks << (ilog2(sdev->sector_size) - 9);
}

static inline unsigned int logical_to_bytes(struct scsi_device *sdev, sector_t blocks)
{
        return blocks * sdev->sector_size;
}

static inline sector_t bytes_to_logical(struct scsi_device *sdev, unsigned int bytes)
{
        return bytes >> ilog2(sdev->sector_size);
}

static inline sector_t sectors_to_logical(struct scsi_device *sdev, sector_t sector)
{
        return sector >> (ilog2(sdev->sector_size) - 9);
}

void sd_dif_config_host(struct scsi_disk *sdkp, struct queue_limits *lim);

#ifdef CONFIG_BLK_DEV_ZONED

int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim,
                u8 buf[SD_BUF_SIZE]);
int sd_zbc_revalidate_zones(struct scsi_disk *sdkp);
blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd,
                                         unsigned char op, bool all);
unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes,
                             struct scsi_sense_hdr *sshdr);
int sd_zbc_report_zones(struct gendisk *disk, sector_t sector,
                unsigned int nr_zones, report_zones_cb cb, void *data);

#else /* CONFIG_BLK_DEV_ZONED */

static inline int sd_zbc_read_zones(struct scsi_disk *sdkp,
                struct queue_limits *lim, u8 buf[SD_BUF_SIZE])
{
        return 0;
}

static inline int sd_zbc_revalidate_zones(struct scsi_disk *sdkp)
{
        return 0;
}

static inline blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd,
                                                       unsigned char op,
                                                       bool all)
{
        return BLK_STS_TARGET;
}

static inline unsigned int sd_zbc_complete(struct scsi_cmnd *cmd,
                        unsigned int good_bytes, struct scsi_sense_hdr *sshdr)
{
        return good_bytes;
}

#define sd_zbc_report_zones NULL

#endif /* CONFIG_BLK_DEV_ZONED */

void sd_print_sense_hdr(struct scsi_disk *sdkp, struct scsi_sense_hdr *sshdr);
void sd_print_result(const struct scsi_disk *sdkp, const char *msg, int result);

#endif /* _SCSI_DISK_H */


























































    5 



















    5 









    2 

   10 

   82 













  132 


  132 

   69 


  129 



    3 







    4 




    4 
   44 




















    5 


    5 









    5 










    5 




































    1 



    8 












    3 













    8 







































  720 



  717 


   81 


  717 


  716 






  714 









  717 



  717 

  428 

  721 


   28 


   40 









  718 




  720 






  653 










  715 
























   48 






   49 









    4 







    4 


    4 









    4 







    4 














   42 
   47 


   46 


   46 


   46 



   45 



   44 


   43 



   44 



   43 


   41 


   42 


   40 
    1 



   39 


   39 


   37 
    3 



   37 


   35 
    3 



   36 


   14 












    4 
    1 


















    2 
    2 
    2 


























  121 




  100 
    1 
   22 

    3 



  102 
    1 
   16 

    4 











  103 
   13 



  116 

  110 
    6 



  110 
    5 



  103 

    1 



   11 


  112 
    4 

  112 
    4 

  111 
    5 




  115 
  111 
    3 



  110 
    5 





    1 



    4 
  110 
    1 



  111 
    1 












    8 

    3 



  104 


  101 
    8 

   96 
    4 


    4 


    4 
    4 


  103 



    1 



   93 
    4 


    4 


    4 
    3 


   98 



    1 

















   99 
   13 
   86 


   98 
    2 



   92 



    3 
    4 




   94 



    2 
    3 









   35 


   36 
   29 


   13 


   11 


    1 


    1 


    1 


    1 


    1 


    1 








    2 



    1 


    1 
















    1 



































   70 









    1 






    1 






    1 




   13 



   55 


    1 


   55 






   26 







    7 
    7 
   24 







    2 
   28 




    3 
   26 

    4 

   29 




    3 
    3 
    2 








   27 
    2 

   25 
    7 

   27 
    2 




   29 



   28 





   26 









   71 





   61 









    1 






    1 






    1 




   11 



   46 


    1 


   47 





    1 




    2 

   21 



   22 
    1 

   23 

   22 

    1 
    1 









   23 



    4 

    4 
    4 



    1 






   23 



   23 






   26 









   60 
























    4 
   45 







   53 






















   51 

    1 

   49 


    3 
    2 


   50 


    1 
    4 


   13 
   46 
   45 
    9 
   53 
    9 
   51 
    3 
   49 
    5 
   51 
    4 
   51 
    3 
   49 
    6 
   51 
    2 
   48 
    6 
   48 
   10 
   48 


   50 
    3 






   53 









    4 




    4 
    4 






    4 




    4 

    4 











    1 



    4 
    4 
    8 




    1 
    1 







   15 





    4 
   11 






    3 


    1 


    1 



    3 
    3 







    3 



    3 








   49 





   49 











   49 









  583 
  582 





  583 












  267 
  268 



  266 









 1098 




 1106 

 1088 

  565 
  565 



   20 
   20 
   20 




  250 
  250 











    5 
















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
// SPDX-License-Identifier: GPL-2.0-only
/*
 * net/core/fib_rules.c                Generic Routing Rules
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/module.h>
#include <net/net_namespace.h>
#include <net/inet_dscp.h>
#include <net/sock.h>
#include <net/fib_rules.h>
#include <net/ip_tunnels.h>
#include <linux/indirect_call_wrapper.h>

#if defined(CONFIG_IPV6) && defined(CONFIG_IPV6_MULTIPLE_TABLES)
#ifdef CONFIG_IP_MULTIPLE_TABLES
#define INDIRECT_CALL_MT(f, f2, f1, ...) \
        INDIRECT_CALL_INET(f, f2, f1, __VA_ARGS__)
#else
#define INDIRECT_CALL_MT(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
#endif
#elif defined(CONFIG_IP_MULTIPLE_TABLES)
#define INDIRECT_CALL_MT(f, f2, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)
#else
#define INDIRECT_CALL_MT(f, f2, f1, ...) f(__VA_ARGS__)
#endif

static const struct fib_kuid_range fib_kuid_range_unset = {
        KUIDT_INIT(0),
        KUIDT_INIT(~0),
};

bool fib_rule_matchall(const struct fib_rule *rule)
{
        if (READ_ONCE(rule->iifindex) || READ_ONCE(rule->oifindex) ||
            rule->mark || rule->tun_id || rule->flags)
                return false;
        if (rule->suppress_ifgroup != -1 || rule->suppress_prefixlen != -1)
                return false;
        if (!uid_eq(rule->uid_range.start, fib_kuid_range_unset.start) ||
            !uid_eq(rule->uid_range.end, fib_kuid_range_unset.end))
                return false;
        if (fib_rule_port_range_set(&rule->sport_range))
                return false;
        if (fib_rule_port_range_set(&rule->dport_range))
                return false;
        return true;
}
EXPORT_SYMBOL_GPL(fib_rule_matchall);

int fib_default_rule_add(struct fib_rules_ops *ops,
                         u32 pref, u32 table)
{
        struct fib_rule *r;

        r = kzalloc(ops->rule_size, GFP_KERNEL_ACCOUNT);
        if (r == NULL)
                return -ENOMEM;

        refcount_set(&r->refcnt, 1);
        r->action = FR_ACT_TO_TBL;
        r->pref = pref;
        r->table = table;
        r->proto = RTPROT_KERNEL;
        r->fr_net = ops->fro_net;
        r->uid_range = fib_kuid_range_unset;

        r->suppress_prefixlen = -1;
        r->suppress_ifgroup = -1;

        /* The lock is not required here, the list in unreachable
         * at the moment this function is called */
        list_add_tail(&r->list, &ops->rules_list);
        return 0;
}
EXPORT_SYMBOL(fib_default_rule_add);

static u32 fib_default_rule_pref(struct fib_rules_ops *ops)
{
        struct list_head *pos;
        struct fib_rule *rule;

        if (!list_empty(&ops->rules_list)) {
                pos = ops->rules_list.next;
                if (pos->next != &ops->rules_list) {
                        rule = list_entry(pos->next, struct fib_rule, list);
                        if (rule->pref)
                                return rule->pref - 1;
                }
        }

        return 0;
}

static void notify_rule_change(int event, struct fib_rule *rule,
                               struct fib_rules_ops *ops, struct nlmsghdr *nlh,
                               u32 pid);

static struct fib_rules_ops *lookup_rules_ops(const struct net *net,
                                              int family)
{
        struct fib_rules_ops *ops;

        rcu_read_lock();
        list_for_each_entry_rcu(ops, &net->rules_ops, list) {
                if (ops->family == family) {
                        if (!try_module_get(ops->owner))
                                ops = NULL;
                        rcu_read_unlock();
                        return ops;
                }
        }
        rcu_read_unlock();

        return NULL;
}

static void rules_ops_put(struct fib_rules_ops *ops)
{
        if (ops)
                module_put(ops->owner);
}

static void flush_route_cache(struct fib_rules_ops *ops)
{
        if (ops->flush_cache)
                ops->flush_cache(ops);
}

static int __fib_rules_register(struct fib_rules_ops *ops)
{
        int err = -EEXIST;
        struct fib_rules_ops *o;
        struct net *net;

        net = ops->fro_net;

        if (ops->rule_size < sizeof(struct fib_rule))
                return -EINVAL;

        if (ops->match == NULL || ops->configure == NULL ||
            ops->compare == NULL || ops->fill == NULL ||
            ops->action == NULL)
                return -EINVAL;

        spin_lock(&net->rules_mod_lock);
        list_for_each_entry(o, &net->rules_ops, list)
                if (ops->family == o->family)
                        goto errout;

        list_add_tail_rcu(&ops->list, &net->rules_ops);
        err = 0;
errout:
        spin_unlock(&net->rules_mod_lock);

        return err;
}

struct fib_rules_ops *
fib_rules_register(const struct fib_rules_ops *tmpl, struct net *net)
{
        struct fib_rules_ops *ops;
        int err;

        ops = kmemdup(tmpl, sizeof(*ops), GFP_KERNEL);
        if (ops == NULL)
                return ERR_PTR(-ENOMEM);

        INIT_LIST_HEAD(&ops->rules_list);
        ops->fro_net = net;

        err = __fib_rules_register(ops);
        if (err) {
                kfree(ops);
                ops = ERR_PTR(err);
        }

        return ops;
}
EXPORT_SYMBOL_GPL(fib_rules_register);

static void fib_rules_cleanup_ops(struct fib_rules_ops *ops)
{
        struct fib_rule *rule, *tmp;

        list_for_each_entry_safe(rule, tmp, &ops->rules_list, list) {
                list_del_rcu(&rule->list);
                if (ops->delete)
                        ops->delete(rule);
                fib_rule_put(rule);
        }
}

void fib_rules_unregister(struct fib_rules_ops *ops)
{
        struct net *net = ops->fro_net;

        spin_lock(&net->rules_mod_lock);
        list_del_rcu(&ops->list);
        spin_unlock(&net->rules_mod_lock);

        fib_rules_cleanup_ops(ops);
        kfree_rcu(ops, rcu);
}
EXPORT_SYMBOL_GPL(fib_rules_unregister);

static int uid_range_set(struct fib_kuid_range *range)
{
        return uid_valid(range->start) && uid_valid(range->end);
}

static struct fib_kuid_range nla_get_kuid_range(struct nlattr **tb)
{
        struct fib_rule_uid_range *in;
        struct fib_kuid_range out;

        in = (struct fib_rule_uid_range *)nla_data(tb[FRA_UID_RANGE]);

        out.start = make_kuid(current_user_ns(), in->start);
        out.end = make_kuid(current_user_ns(), in->end);

        return out;
}

static int nla_put_uid_range(struct sk_buff *skb, struct fib_kuid_range *range)
{
        struct fib_rule_uid_range out = {
                from_kuid_munged(current_user_ns(), range->start),
                from_kuid_munged(current_user_ns(), range->end)
        };

        return nla_put(skb, FRA_UID_RANGE, sizeof(out), &out);
}

static int nla_get_port_range(struct nlattr *pattr,
                              struct fib_rule_port_range *port_range)
{
        const struct fib_rule_port_range *pr = nla_data(pattr);

        if (!fib_rule_port_range_valid(pr))
                return -EINVAL;

        port_range->start = pr->start;
        port_range->end = pr->end;

        return 0;
}

static int nla_put_port_range(struct sk_buff *skb, int attrtype,
                              struct fib_rule_port_range *range)
{
        return nla_put(skb, attrtype, sizeof(*range), range);
}

static bool fib_rule_iif_match(const struct fib_rule *rule, int iifindex,
                               const struct flowi *fl)
{
        u8 iif_is_l3_master = READ_ONCE(rule->iif_is_l3_master);

        return iif_is_l3_master ? l3mdev_fib_rule_iif_match(fl, iifindex) :
                                  fl->flowi_iif == iifindex;
}

static bool fib_rule_oif_match(const struct fib_rule *rule, int oifindex,
                               const struct flowi *fl)
{
        u8 oif_is_l3_master = READ_ONCE(rule->oif_is_l3_master);

        return oif_is_l3_master ? l3mdev_fib_rule_oif_match(fl, oifindex) :
                                  fl->flowi_oif == oifindex;
}

static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
                          struct flowi *fl, int flags,
                          struct fib_lookup_arg *arg)
{
        int iifindex, oifindex, ret = 0;

        iifindex = READ_ONCE(rule->iifindex);
        if (iifindex && !fib_rule_iif_match(rule, iifindex, fl))
                goto out;

        oifindex = READ_ONCE(rule->oifindex);
        if (oifindex && !fib_rule_oif_match(rule, oifindex, fl))
                goto out;

        if ((rule->mark ^ fl->flowi_mark) & rule->mark_mask)
                goto out;

        if (rule->tun_id && (rule->tun_id != fl->flowi_tun_key.tun_id))
                goto out;

        if (rule->l3mdev && !l3mdev_fib_rule_match(rule->fr_net, fl, arg))
                goto out;

        if (uid_lt(fl->flowi_uid, rule->uid_range.start) ||
            uid_gt(fl->flowi_uid, rule->uid_range.end))
                goto out;

        ret = INDIRECT_CALL_MT(ops->match,
                               fib6_rule_match,
                               fib4_rule_match,
                               rule, fl, flags);
out:
        return (rule->flags & FIB_RULE_INVERT) ? !ret : ret;
}

int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
                     int flags, struct fib_lookup_arg *arg)
{
        struct fib_rule *rule;
        int err;

        rcu_read_lock();

        list_for_each_entry_rcu(rule, &ops->rules_list, list) {
jumped:
                if (!fib_rule_match(rule, ops, fl, flags, arg))
                        continue;

                if (rule->action == FR_ACT_GOTO) {
                        struct fib_rule *target;

                        target = rcu_dereference(rule->ctarget);
                        if (target == NULL) {
                                continue;
                        } else {
                                rule = target;
                                goto jumped;
                        }
                } else if (rule->action == FR_ACT_NOP)
                        continue;
                else
                        err = INDIRECT_CALL_MT(ops->action,
                                               fib6_rule_action,
                                               fib4_rule_action,
                                               rule, fl, flags, arg);

                if (!err && ops->suppress && INDIRECT_CALL_MT(ops->suppress,
                                                              fib6_rule_suppress,
                                                              fib4_rule_suppress,
                                                              rule, flags, arg))
                        continue;

                if (err != -EAGAIN) {
                        if ((arg->flags & FIB_LOOKUP_NOREF) ||
                            likely(refcount_inc_not_zero(&rule->refcnt))) {
                                arg->rule = rule;
                                goto out;
                        }
                        break;
                }
        }

        err = -ESRCH;
out:
        rcu_read_unlock();

        return err;
}
EXPORT_SYMBOL_GPL(fib_rules_lookup);

static int call_fib_rule_notifier(struct notifier_block *nb,
                                  enum fib_event_type event_type,
                                  struct fib_rule *rule, int family,
                                  struct netlink_ext_ack *extack)
{
        struct fib_rule_notifier_info info = {
                .info.family = family,
                .info.extack = extack,
                .rule = rule,
        };

        return call_fib_notifier(nb, event_type, &info.info);
}

static int call_fib_rule_notifiers(struct net *net,
                                   enum fib_event_type event_type,
                                   struct fib_rule *rule,
                                   struct fib_rules_ops *ops,
                                   struct netlink_ext_ack *extack)
{
        struct fib_rule_notifier_info info = {
                .info.family = ops->family,
                .info.extack = extack,
                .rule = rule,
        };

        ASSERT_RTNL_NET(net);

        /* Paired with READ_ONCE() in fib_rules_seq() */
        WRITE_ONCE(ops->fib_rules_seq, ops->fib_rules_seq + 1);
        return call_fib_notifiers(net, event_type, &info.info);
}

/* Called with rcu_read_lock() */
int fib_rules_dump(struct net *net, struct notifier_block *nb, int family,
                   struct netlink_ext_ack *extack)
{
        struct fib_rules_ops *ops;
        struct fib_rule *rule;
        int err = 0;

        ops = lookup_rules_ops(net, family);
        if (!ops)
                return -EAFNOSUPPORT;
        list_for_each_entry_rcu(rule, &ops->rules_list, list) {
                err = call_fib_rule_notifier(nb, FIB_EVENT_RULE_ADD,
                                             rule, family, extack);
                if (err)
                        break;
        }
        rules_ops_put(ops);

        return err;
}
EXPORT_SYMBOL_GPL(fib_rules_dump);

unsigned int fib_rules_seq_read(const struct net *net, int family)
{
        unsigned int fib_rules_seq;
        struct fib_rules_ops *ops;

        ops = lookup_rules_ops(net, family);
        if (!ops)
                return 0;
        /* Paired with WRITE_ONCE() in call_fib_rule_notifiers() */
        fib_rules_seq = READ_ONCE(ops->fib_rules_seq);
        rules_ops_put(ops);

        return fib_rules_seq;
}
EXPORT_SYMBOL_GPL(fib_rules_seq_read);

static struct fib_rule *rule_find(struct fib_rules_ops *ops,
                                  struct fib_rule_hdr *frh,
                                  struct nlattr **tb,
                                  struct fib_rule *rule,
                                  bool user_priority)
{
        struct fib_rule *r;

        list_for_each_entry(r, &ops->rules_list, list) {
                if (rule->action && r->action != rule->action)
                        continue;

                if (rule->table && r->table != rule->table)
                        continue;

                if (user_priority && r->pref != rule->pref)
                        continue;

                if (rule->iifname[0] &&
                    memcmp(r->iifname, rule->iifname, IFNAMSIZ))
                        continue;

                if (rule->oifname[0] &&
                    memcmp(r->oifname, rule->oifname, IFNAMSIZ))
                        continue;

                if (rule->mark && r->mark != rule->mark)
                        continue;

                if (rule->suppress_ifgroup != -1 &&
                    r->suppress_ifgroup != rule->suppress_ifgroup)
                        continue;

                if (rule->suppress_prefixlen != -1 &&
                    r->suppress_prefixlen != rule->suppress_prefixlen)
                        continue;

                if (rule->mark_mask && r->mark_mask != rule->mark_mask)
                        continue;

                if (rule->tun_id && r->tun_id != rule->tun_id)
                        continue;

                if (rule->l3mdev && r->l3mdev != rule->l3mdev)
                        continue;

                if (uid_range_set(&rule->uid_range) &&
                    (!uid_eq(r->uid_range.start, rule->uid_range.start) ||
                    !uid_eq(r->uid_range.end, rule->uid_range.end)))
                        continue;

                if (rule->ip_proto && r->ip_proto != rule->ip_proto)
                        continue;

                if (rule->proto && r->proto != rule->proto)
                        continue;

                if (fib_rule_port_range_set(&rule->sport_range) &&
                    !fib_rule_port_range_compare(&r->sport_range,
                                                 &rule->sport_range))
                        continue;

                if (rule->sport_mask && r->sport_mask != rule->sport_mask)
                        continue;

                if (fib_rule_port_range_set(&rule->dport_range) &&
                    !fib_rule_port_range_compare(&r->dport_range,
                                                 &rule->dport_range))
                        continue;

                if (rule->dport_mask && r->dport_mask != rule->dport_mask)
                        continue;

                if (!ops->compare(r, frh, tb))
                        continue;
                return r;
        }

        return NULL;
}

#ifdef CONFIG_NET_L3_MASTER_DEV
static int fib_nl2rule_l3mdev(struct nlattr *nla, struct fib_rule *nlrule,
                              struct netlink_ext_ack *extack)
{
        nlrule->l3mdev = nla_get_u8(nla);
        if (nlrule->l3mdev != 1) {
                NL_SET_ERR_MSG(extack, "Invalid l3mdev attribute");
                return -1;
        }

        return 0;
}
#else
static int fib_nl2rule_l3mdev(struct nlattr *nla, struct fib_rule *nlrule,
                              struct netlink_ext_ack *extack)
{
        NL_SET_ERR_MSG(extack, "l3mdev support is not enabled in kernel");
        return -1;
}
#endif

static int fib_nl2rule_port_mask(const struct nlattr *mask_attr,
                                 const struct fib_rule_port_range *range,
                                 u16 *port_mask,
                                 struct netlink_ext_ack *extack)
{
        if (!fib_rule_port_range_valid(range)) {
                NL_SET_ERR_MSG_ATTR(extack, mask_attr,
                                    "Cannot specify port mask without port value");
                return -EINVAL;
        }

        if (fib_rule_port_is_range(range)) {
                NL_SET_ERR_MSG_ATTR(extack, mask_attr,
                                    "Cannot specify port mask for port range");
                return -EINVAL;
        }

        if (range->start & ~nla_get_u16(mask_attr)) {
                NL_SET_ERR_MSG_ATTR(extack, mask_attr, "Invalid port mask");
                return -EINVAL;
        }

        *port_mask = nla_get_u16(mask_attr);

        return 0;
}

static int fib_nl2rule(struct net *net, struct nlmsghdr *nlh,
                       struct netlink_ext_ack *extack,
                       struct fib_rules_ops *ops,
                       struct nlattr *tb[],
                       struct fib_rule **rule,
                       bool *user_priority)
{
        struct fib_rule_hdr *frh = nlmsg_data(nlh);
        struct fib_rule *nlrule = NULL;
        int err = -EINVAL;

        if (frh->src_len)
                if (!tb[FRA_SRC] ||
                    frh->src_len > (ops->addr_size * 8) ||
                    nla_len(tb[FRA_SRC]) != ops->addr_size) {
                        NL_SET_ERR_MSG(extack, "Invalid source address");
                        goto errout;
        }

        if (frh->dst_len)
                if (!tb[FRA_DST] ||
                    frh->dst_len > (ops->addr_size * 8) ||
                    nla_len(tb[FRA_DST]) != ops->addr_size) {
                        NL_SET_ERR_MSG(extack, "Invalid dst address");
                        goto errout;
        }

        nlrule = kzalloc(ops->rule_size, GFP_KERNEL_ACCOUNT);
        if (!nlrule) {
                err = -ENOMEM;
                goto errout;
        }
        refcount_set(&nlrule->refcnt, 1);
        nlrule->fr_net = net;

        if (tb[FRA_PRIORITY]) {
                nlrule->pref = nla_get_u32(tb[FRA_PRIORITY]);
                *user_priority = true;
        }

        nlrule->proto = nla_get_u8_default(tb[FRA_PROTOCOL], RTPROT_UNSPEC);

        if (tb[FRA_IIFNAME]) {
                nlrule->iifindex = -1;
                nla_strscpy(nlrule->iifname, tb[FRA_IIFNAME], IFNAMSIZ);
        }

        if (tb[FRA_OIFNAME]) {
                nlrule->oifindex = -1;
                nla_strscpy(nlrule->oifname, tb[FRA_OIFNAME], IFNAMSIZ);
        }

        if (tb[FRA_FWMARK]) {
                nlrule->mark = nla_get_u32(tb[FRA_FWMARK]);
                if (nlrule->mark)
                        /* compatibility: if the mark value is non-zero all bits
                         * are compared unless a mask is explicitly specified.
                         */
                        nlrule->mark_mask = 0xFFFFFFFF;
        }

        if (tb[FRA_FWMASK])
                nlrule->mark_mask = nla_get_u32(tb[FRA_FWMASK]);

        if (tb[FRA_TUN_ID])
                nlrule->tun_id = nla_get_be64(tb[FRA_TUN_ID]);

        if (tb[FRA_L3MDEV] &&
            fib_nl2rule_l3mdev(tb[FRA_L3MDEV], nlrule, extack) < 0)
                goto errout_free;

        nlrule->action = frh->action;
        nlrule->flags = frh->flags;
        nlrule->table = frh_get_table(frh, tb);
        if (tb[FRA_SUPPRESS_PREFIXLEN])
                nlrule->suppress_prefixlen = nla_get_u32(tb[FRA_SUPPRESS_PREFIXLEN]);
        else
                nlrule->suppress_prefixlen = -1;

        if (tb[FRA_SUPPRESS_IFGROUP])
                nlrule->suppress_ifgroup = nla_get_u32(tb[FRA_SUPPRESS_IFGROUP]);
        else
                nlrule->suppress_ifgroup = -1;

        if (tb[FRA_GOTO]) {
                if (nlrule->action != FR_ACT_GOTO) {
                        NL_SET_ERR_MSG(extack, "Unexpected goto");
                        goto errout_free;
                }

                nlrule->target = nla_get_u32(tb[FRA_GOTO]);
        } else if (nlrule->action == FR_ACT_GOTO) {
                NL_SET_ERR_MSG(extack, "Missing goto target for action goto");
                goto errout_free;
        }

        if (nlrule->l3mdev && nlrule->table) {
                NL_SET_ERR_MSG(extack, "l3mdev and table are mutually exclusive");
                goto errout_free;
        }

        if (tb[FRA_UID_RANGE]) {
                if (current_user_ns() != net->user_ns) {
                        err = -EPERM;
                        NL_SET_ERR_MSG(extack, "No permission to set uid");
                        goto errout_free;
                }

                nlrule->uid_range = nla_get_kuid_range(tb);

                if (!uid_range_set(&nlrule->uid_range) ||
                    !uid_lte(nlrule->uid_range.start, nlrule->uid_range.end)) {
                        NL_SET_ERR_MSG(extack, "Invalid uid range");
                        goto errout_free;
                }
        } else {
                nlrule->uid_range = fib_kuid_range_unset;
        }

        if (tb[FRA_IP_PROTO])
                nlrule->ip_proto = nla_get_u8(tb[FRA_IP_PROTO]);

        if (tb[FRA_SPORT_RANGE]) {
                err = nla_get_port_range(tb[FRA_SPORT_RANGE],
                                         &nlrule->sport_range);
                if (err) {
                        NL_SET_ERR_MSG(extack, "Invalid sport range");
                        goto errout_free;
                }
                if (!fib_rule_port_is_range(&nlrule->sport_range))
                        nlrule->sport_mask = U16_MAX;
        }

        if (tb[FRA_SPORT_MASK]) {
                err = fib_nl2rule_port_mask(tb[FRA_SPORT_MASK],
                                            &nlrule->sport_range,
                                            &nlrule->sport_mask, extack);
                if (err)
                        goto errout_free;
        }

        if (tb[FRA_DPORT_RANGE]) {
                err = nla_get_port_range(tb[FRA_DPORT_RANGE],
                                         &nlrule->dport_range);
                if (err) {
                        NL_SET_ERR_MSG(extack, "Invalid dport range");
                        goto errout_free;
                }
                if (!fib_rule_port_is_range(&nlrule->dport_range))
                        nlrule->dport_mask = U16_MAX;
        }

        if (tb[FRA_DPORT_MASK]) {
                err = fib_nl2rule_port_mask(tb[FRA_DPORT_MASK],
                                            &nlrule->dport_range,
                                            &nlrule->dport_mask, extack);
                if (err)
                        goto errout_free;
        }

        *rule = nlrule;

        return 0;

errout_free:
        kfree(nlrule);
errout:
        return err;
}

static int fib_nl2rule_rtnl(struct fib_rule *nlrule,
                            struct fib_rules_ops *ops,
                            struct nlattr *tb[],
                            struct netlink_ext_ack *extack)
{
        if (!tb[FRA_PRIORITY])
                nlrule->pref = fib_default_rule_pref(ops);

        /* Backward jumps are prohibited to avoid endless loops */
        if (tb[FRA_GOTO] && nlrule->target <= nlrule->pref) {
                NL_SET_ERR_MSG(extack, "Backward goto not supported");
                return -EINVAL;
        }

        if (tb[FRA_IIFNAME]) {
                struct net_device *dev;

                dev = __dev_get_by_name(nlrule->fr_net, nlrule->iifname);
                if (dev) {
                        nlrule->iifindex = dev->ifindex;
                        nlrule->iif_is_l3_master = netif_is_l3_master(dev);
                }
        }

        if (tb[FRA_OIFNAME]) {
                struct net_device *dev;

                dev = __dev_get_by_name(nlrule->fr_net, nlrule->oifname);
                if (dev) {
                        nlrule->oifindex = dev->ifindex;
                        nlrule->oif_is_l3_master = netif_is_l3_master(dev);
                }
        }

        return 0;
}

static int rule_exists(struct fib_rules_ops *ops, struct fib_rule_hdr *frh,
                       struct nlattr **tb, struct fib_rule *rule)
{
        struct fib_rule *r;

        list_for_each_entry(r, &ops->rules_list, list) {
                if (r->action != rule->action)
                        continue;

                if (r->table != rule->table)
                        continue;

                if (r->pref != rule->pref)
                        continue;

                if (memcmp(r->iifname, rule->iifname, IFNAMSIZ))
                        continue;

                if (memcmp(r->oifname, rule->oifname, IFNAMSIZ))
                        continue;

                if (r->mark != rule->mark)
                        continue;

                if (r->suppress_ifgroup != rule->suppress_ifgroup)
                        continue;

                if (r->suppress_prefixlen != rule->suppress_prefixlen)
                        continue;

                if (r->mark_mask != rule->mark_mask)
                        continue;

                if (r->tun_id != rule->tun_id)
                        continue;

                if (r->l3mdev != rule->l3mdev)
                        continue;

                if (!uid_eq(r->uid_range.start, rule->uid_range.start) ||
                    !uid_eq(r->uid_range.end, rule->uid_range.end))
                        continue;

                if (r->ip_proto != rule->ip_proto)
                        continue;

                if (r->proto != rule->proto)
                        continue;

                if (!fib_rule_port_range_compare(&r->sport_range,
                                                 &rule->sport_range))
                        continue;

                if (r->sport_mask != rule->sport_mask)
                        continue;

                if (!fib_rule_port_range_compare(&r->dport_range,
                                                 &rule->dport_range))
                        continue;

                if (r->dport_mask != rule->dport_mask)
                        continue;

                if (!ops->compare(r, frh, tb))
                        continue;
                return 1;
        }
        return 0;
}

static const struct nla_policy fib_rule_policy[FRA_MAX + 1] = {
        [FRA_UNSPEC]        = { .strict_start_type = FRA_DPORT_RANGE + 1 },
        [FRA_IIFNAME]        = { .type = NLA_STRING, .len = IFNAMSIZ - 1 },
        [FRA_OIFNAME]        = { .type = NLA_STRING, .len = IFNAMSIZ - 1 },
        [FRA_PRIORITY]        = { .type = NLA_U32 },
        [FRA_FWMARK]        = { .type = NLA_U32 },
        [FRA_FLOW]        = { .type = NLA_U32 },
        [FRA_TUN_ID]        = { .type = NLA_U64 },
        [FRA_FWMASK]        = { .type = NLA_U32 },
        [FRA_TABLE]     = { .type = NLA_U32 },
        [FRA_SUPPRESS_PREFIXLEN] = { .type = NLA_U32 },
        [FRA_SUPPRESS_IFGROUP] = { .type = NLA_U32 },
        [FRA_GOTO]        = { .type = NLA_U32 },
        [FRA_L3MDEV]        = { .type = NLA_U8 },
        [FRA_UID_RANGE]        = { .len = sizeof(struct fib_rule_uid_range) },
        [FRA_PROTOCOL]  = { .type = NLA_U8 },
        [FRA_IP_PROTO]  = { .type = NLA_U8 },
        [FRA_SPORT_RANGE] = { .len = sizeof(struct fib_rule_port_range) },
        [FRA_DPORT_RANGE] = { .len = sizeof(struct fib_rule_port_range) },
        [FRA_DSCP]        = NLA_POLICY_MAX(NLA_U8, INET_DSCP_MASK >> 2),
        [FRA_FLOWLABEL] = { .type = NLA_BE32 },
        [FRA_FLOWLABEL_MASK] = { .type = NLA_BE32 },
        [FRA_SPORT_MASK] = { .type = NLA_U16 },
        [FRA_DPORT_MASK] = { .type = NLA_U16 },
        [FRA_DSCP_MASK] = NLA_POLICY_MASK(NLA_U8, INET_DSCP_MASK >> 2),
};

int fib_newrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
                struct netlink_ext_ack *extack, bool rtnl_held)
{
        struct fib_rule *rule = NULL, *r, *last = NULL;
        int err = -EINVAL, unresolved = 0;
        struct fib_rules_ops *ops = NULL;
        struct nlattr *tb[FRA_MAX + 1];
        bool user_priority = false;
        struct fib_rule_hdr *frh;

        frh = nlmsg_payload(nlh, sizeof(*frh));
        if (!frh) {
                NL_SET_ERR_MSG(extack, "Invalid msg length");
                goto errout;
        }

        ops = lookup_rules_ops(net, frh->family);
        if (!ops) {
                err = -EAFNOSUPPORT;
                NL_SET_ERR_MSG(extack, "Rule family not supported");
                goto errout;
        }

        err = nlmsg_parse_deprecated(nlh, sizeof(*frh), tb, FRA_MAX,
                                     fib_rule_policy, extack);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Error parsing msg");
                goto errout;
        }

        err = fib_nl2rule(net, nlh, extack, ops, tb, &rule, &user_priority);
        if (err)
                goto errout;

        if (!rtnl_held)
                rtnl_net_lock(net);

        err = fib_nl2rule_rtnl(rule, ops, tb, extack);
        if (err)
                goto errout_free;

        if ((nlh->nlmsg_flags & NLM_F_EXCL) &&
            rule_exists(ops, frh, tb, rule)) {
                err = -EEXIST;
                goto errout_free;
        }

        err = ops->configure(rule, skb, frh, tb, extack);
        if (err < 0)
                goto errout_free;

        err = call_fib_rule_notifiers(net, FIB_EVENT_RULE_ADD, rule, ops,
                                      extack);
        if (err < 0)
                goto errout_free;

        list_for_each_entry(r, &ops->rules_list, list) {
                if (r->pref == rule->target) {
                        RCU_INIT_POINTER(rule->ctarget, r);
                        break;
                }
        }

        if (rcu_dereference_protected(rule->ctarget, 1) == NULL)
                unresolved = 1;

        list_for_each_entry(r, &ops->rules_list, list) {
                if (r->pref > rule->pref)
                        break;
                last = r;
        }

        if (last)
                list_add_rcu(&rule->list, &last->list);
        else
                list_add_rcu(&rule->list, &ops->rules_list);

        if (ops->unresolved_rules) {
                /*
                 * There are unresolved goto rules in the list, check if
                 * any of them are pointing to this new rule.
                 */
                list_for_each_entry(r, &ops->rules_list, list) {
                        if (r->action == FR_ACT_GOTO &&
                            r->target == rule->pref &&
                            rtnl_dereference(r->ctarget) == NULL) {
                                rcu_assign_pointer(r->ctarget, rule);
                                if (--ops->unresolved_rules == 0)
                                        break;
                        }
                }
        }

        if (rule->action == FR_ACT_GOTO)
                ops->nr_goto_rules++;

        if (unresolved)
                ops->unresolved_rules++;

        if (rule->tun_id)
                ip_tunnel_need_metadata();

        fib_rule_get(rule);

        if (!rtnl_held)
                rtnl_net_unlock(net);

        notify_rule_change(RTM_NEWRULE, rule, ops, nlh, NETLINK_CB(skb).portid);
        fib_rule_put(rule);
        flush_route_cache(ops);
        rules_ops_put(ops);
        return 0;

errout_free:
        if (!rtnl_held)
                rtnl_net_unlock(net);
        kfree(rule);
errout:
        rules_ops_put(ops);
        return err;
}
EXPORT_SYMBOL_GPL(fib_newrule);

static int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        return fib_newrule(sock_net(skb->sk), skb, nlh, extack, false);
}

int fib_delrule(struct net *net, struct sk_buff *skb, struct nlmsghdr *nlh,
                struct netlink_ext_ack *extack, bool rtnl_held)
{
        struct fib_rule *rule = NULL, *nlrule = NULL;
        struct fib_rules_ops *ops = NULL;
        struct nlattr *tb[FRA_MAX+1];
        bool user_priority = false;
        struct fib_rule_hdr *frh;
        int err = -EINVAL;

        frh = nlmsg_payload(nlh, sizeof(*frh));
        if (!frh) {
                NL_SET_ERR_MSG(extack, "Invalid msg length");
                goto errout;
        }

        ops = lookup_rules_ops(net, frh->family);
        if (ops == NULL) {
                err = -EAFNOSUPPORT;
                NL_SET_ERR_MSG(extack, "Rule family not supported");
                goto errout;
        }

        err = nlmsg_parse_deprecated(nlh, sizeof(*frh), tb, FRA_MAX,
                                     fib_rule_policy, extack);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Error parsing msg");
                goto errout;
        }

        err = fib_nl2rule(net, nlh, extack, ops, tb, &nlrule, &user_priority);
        if (err)
                goto errout;

        if (!rtnl_held)
                rtnl_net_lock(net);

        err = fib_nl2rule_rtnl(nlrule, ops, tb, extack);
        if (err)
                goto errout_free;

        rule = rule_find(ops, frh, tb, nlrule, user_priority);
        if (!rule) {
                err = -ENOENT;
                goto errout_free;
        }

        if (rule->flags & FIB_RULE_PERMANENT) {
                err = -EPERM;
                goto errout_free;
        }

        if (ops->delete) {
                err = ops->delete(rule);
                if (err)
                        goto errout_free;
        }

        if (rule->tun_id)
                ip_tunnel_unneed_metadata();

        list_del_rcu(&rule->list);

        if (rule->action == FR_ACT_GOTO) {
                ops->nr_goto_rules--;
                if (rtnl_dereference(rule->ctarget) == NULL)
                        ops->unresolved_rules--;
        }

        /*
         * Check if this rule is a target to any of them. If so,
         * adjust to the next one with the same preference or
         * disable them. As this operation is eventually very
         * expensive, it is only performed if goto rules, except
         * current if it is goto rule, have actually been added.
         */
        if (ops->nr_goto_rules > 0) {
                struct fib_rule *n, *r;

                n = list_next_entry(rule, list);
                if (&n->list == &ops->rules_list || n->pref != rule->pref)
                        n = NULL;
                list_for_each_entry(r, &ops->rules_list, list) {
                        if (rtnl_dereference(r->ctarget) != rule)
                                continue;
                        rcu_assign_pointer(r->ctarget, n);
                        if (!n)
                                ops->unresolved_rules++;
                }
        }

        call_fib_rule_notifiers(net, FIB_EVENT_RULE_DEL, rule, ops, NULL);

        if (!rtnl_held)
                rtnl_net_unlock(net);

        notify_rule_change(RTM_DELRULE, rule, ops, nlh, NETLINK_CB(skb).portid);
        fib_rule_put(rule);
        flush_route_cache(ops);
        rules_ops_put(ops);
        kfree(nlrule);
        return 0;

errout_free:
        if (!rtnl_held)
                rtnl_net_unlock(net);
        kfree(nlrule);
errout:
        rules_ops_put(ops);
        return err;
}
EXPORT_SYMBOL_GPL(fib_delrule);

static int fib_nl_delrule(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        return fib_delrule(sock_net(skb->sk), skb, nlh, extack, false);
}

static inline size_t fib_rule_nlmsg_size(struct fib_rules_ops *ops,
                                         struct fib_rule *rule)
{
        size_t payload = NLMSG_ALIGN(sizeof(struct fib_rule_hdr))
                         + nla_total_size(IFNAMSIZ) /* FRA_IIFNAME */
                         + nla_total_size(IFNAMSIZ) /* FRA_OIFNAME */
                         + nla_total_size(4) /* FRA_PRIORITY */
                         + nla_total_size(4) /* FRA_TABLE */
                         + nla_total_size(4) /* FRA_SUPPRESS_PREFIXLEN */
                         + nla_total_size(4) /* FRA_SUPPRESS_IFGROUP */
                         + nla_total_size(4) /* FRA_FWMARK */
                         + nla_total_size(4) /* FRA_FWMASK */
                         + nla_total_size_64bit(8) /* FRA_TUN_ID */
                         + nla_total_size(sizeof(struct fib_kuid_range))
                         + nla_total_size(1) /* FRA_PROTOCOL */
                         + nla_total_size(1) /* FRA_IP_PROTO */
                         + nla_total_size(sizeof(struct fib_rule_port_range)) /* FRA_SPORT_RANGE */
                         + nla_total_size(sizeof(struct fib_rule_port_range)) /* FRA_DPORT_RANGE */
                         + nla_total_size(2) /* FRA_SPORT_MASK */
                         + nla_total_size(2); /* FRA_DPORT_MASK */

        if (ops->nlmsg_payload)
                payload += ops->nlmsg_payload(rule);

        return payload;
}

static int fib_nl_fill_rule(struct sk_buff *skb, struct fib_rule *rule,
                            u32 pid, u32 seq, int type, int flags,
                            struct fib_rules_ops *ops)
{
        struct nlmsghdr *nlh;
        struct fib_rule_hdr *frh;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*frh), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        frh = nlmsg_data(nlh);
        frh->family = ops->family;
        frh->table = rule->table < 256 ? rule->table : RT_TABLE_COMPAT;
        if (nla_put_u32(skb, FRA_TABLE, rule->table))
                goto nla_put_failure;
        if (nla_put_u32(skb, FRA_SUPPRESS_PREFIXLEN, rule->suppress_prefixlen))
                goto nla_put_failure;
        frh->res1 = 0;
        frh->res2 = 0;
        frh->action = rule->action;
        frh->flags = rule->flags;

        if (nla_put_u8(skb, FRA_PROTOCOL, rule->proto))
                goto nla_put_failure;

        if (rule->action == FR_ACT_GOTO &&
            rcu_access_pointer(rule->ctarget) == NULL)
                frh->flags |= FIB_RULE_UNRESOLVED;

        if (rule->iifname[0]) {
                if (nla_put_string(skb, FRA_IIFNAME, rule->iifname))
                        goto nla_put_failure;
                if (READ_ONCE(rule->iifindex) == -1)
                        frh->flags |= FIB_RULE_IIF_DETACHED;
        }

        if (rule->oifname[0]) {
                if (nla_put_string(skb, FRA_OIFNAME, rule->oifname))
                        goto nla_put_failure;
                if (READ_ONCE(rule->oifindex) == -1)
                        frh->flags |= FIB_RULE_OIF_DETACHED;
        }

        if ((rule->pref &&
             nla_put_u32(skb, FRA_PRIORITY, rule->pref)) ||
            (rule->mark &&
             nla_put_u32(skb, FRA_FWMARK, rule->mark)) ||
            ((rule->mark_mask || rule->mark) &&
             nla_put_u32(skb, FRA_FWMASK, rule->mark_mask)) ||
            (rule->target &&
             nla_put_u32(skb, FRA_GOTO, rule->target)) ||
            (rule->tun_id &&
             nla_put_be64(skb, FRA_TUN_ID, rule->tun_id, FRA_PAD)) ||
            (rule->l3mdev &&
             nla_put_u8(skb, FRA_L3MDEV, rule->l3mdev)) ||
            (uid_range_set(&rule->uid_range) &&
             nla_put_uid_range(skb, &rule->uid_range)) ||
            (fib_rule_port_range_set(&rule->sport_range) &&
             nla_put_port_range(skb, FRA_SPORT_RANGE, &rule->sport_range)) ||
            (rule->sport_mask && nla_put_u16(skb, FRA_SPORT_MASK,
                                             rule->sport_mask)) ||
            (fib_rule_port_range_set(&rule->dport_range) &&
             nla_put_port_range(skb, FRA_DPORT_RANGE, &rule->dport_range)) ||
            (rule->dport_mask && nla_put_u16(skb, FRA_DPORT_MASK,
                                             rule->dport_mask)) ||
            (rule->ip_proto && nla_put_u8(skb, FRA_IP_PROTO, rule->ip_proto)))
                goto nla_put_failure;

        if (rule->suppress_ifgroup != -1) {
                if (nla_put_u32(skb, FRA_SUPPRESS_IFGROUP, rule->suppress_ifgroup))
                        goto nla_put_failure;
        }

        if (ops->fill(rule, skb, frh) < 0)
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int dump_rules(struct sk_buff *skb, struct netlink_callback *cb,
                      struct fib_rules_ops *ops)
{
        int idx = 0;
        struct fib_rule *rule;
        int err = 0;

        rcu_read_lock();
        list_for_each_entry_rcu(rule, &ops->rules_list, list) {
                if (idx < cb->args[1])
                        goto skip;

                err = fib_nl_fill_rule(skb, rule, NETLINK_CB(cb->skb).portid,
                                       cb->nlh->nlmsg_seq, RTM_NEWRULE,
                                       NLM_F_MULTI, ops);
                if (err)
                        break;
skip:
                idx++;
        }
        rcu_read_unlock();
        cb->args[1] = idx;
        rules_ops_put(ops);

        return err;
}

static int fib_valid_dumprule_req(const struct nlmsghdr *nlh,
                                   struct netlink_ext_ack *extack)
{
        struct fib_rule_hdr *frh;

        frh = nlmsg_payload(nlh, sizeof(*frh));
        if (!frh) {
                NL_SET_ERR_MSG(extack, "Invalid header for fib rule dump request");
                return -EINVAL;
        }

        if (frh->dst_len || frh->src_len || frh->tos || frh->table ||
            frh->res1 || frh->res2 || frh->action || frh->flags) {
                NL_SET_ERR_MSG(extack,
                               "Invalid values in header for fib rule dump request");
                return -EINVAL;
        }

        if (nlmsg_attrlen(nlh, sizeof(*frh))) {
                NL_SET_ERR_MSG(extack, "Invalid data after header in fib rule dump request");
                return -EINVAL;
        }

        return 0;
}

static int fib_nl_dumprule(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct fib_rules_ops *ops;
        int err, idx = 0, family;

        if (cb->strict_check) {
                err = fib_valid_dumprule_req(nlh, cb->extack);

                if (err < 0)
                        return err;
        }

        family = rtnl_msg_family(nlh);
        if (family != AF_UNSPEC) {
                /* Protocol specific dump request */
                ops = lookup_rules_ops(net, family);
                if (ops == NULL)
                        return -EAFNOSUPPORT;

                return dump_rules(skb, cb, ops);
        }

        err = 0;
        rcu_read_lock();
        list_for_each_entry_rcu(ops, &net->rules_ops, list) {
                if (idx < cb->args[0] || !try_module_get(ops->owner))
                        goto skip;

                err = dump_rules(skb, cb, ops);
                if (err < 0)
                        break;

                cb->args[1] = 0;
skip:
                idx++;
        }
        rcu_read_unlock();
        cb->args[0] = idx;

        return err;
}

static void notify_rule_change(int event, struct fib_rule *rule,
                               struct fib_rules_ops *ops, struct nlmsghdr *nlh,
                               u32 pid)
{
        struct net *net;
        struct sk_buff *skb;
        int err = -ENOMEM;

        net = ops->fro_net;
        skb = nlmsg_new(fib_rule_nlmsg_size(ops, rule), GFP_KERNEL);
        if (skb == NULL)
                goto errout;

        err = fib_nl_fill_rule(skb, rule, pid, nlh->nlmsg_seq, event, 0, ops);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in fib_rule_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, pid, ops->nlgroup, nlh, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, ops->nlgroup, err);
}

static void attach_rules(struct list_head *rules, struct net_device *dev)
{
        struct fib_rule *rule;

        list_for_each_entry(rule, rules, list) {
                if (rule->iifindex == -1 &&
                    strcmp(dev->name, rule->iifname) == 0) {
                        WRITE_ONCE(rule->iifindex, dev->ifindex);
                        WRITE_ONCE(rule->iif_is_l3_master,
                                   netif_is_l3_master(dev));
                }
                if (rule->oifindex == -1 &&
                    strcmp(dev->name, rule->oifname) == 0) {
                        WRITE_ONCE(rule->oifindex, dev->ifindex);
                        WRITE_ONCE(rule->oif_is_l3_master,
                                   netif_is_l3_master(dev));
                }
        }
}

static void detach_rules(struct list_head *rules, struct net_device *dev)
{
        struct fib_rule *rule;

        list_for_each_entry(rule, rules, list) {
                if (rule->iifindex == dev->ifindex) {
                        WRITE_ONCE(rule->iifindex, -1);
                        WRITE_ONCE(rule->iif_is_l3_master, false);
                }
                if (rule->oifindex == dev->ifindex) {
                        WRITE_ONCE(rule->oifindex, -1);
                        WRITE_ONCE(rule->oif_is_l3_master, false);
                }
        }
}


static int fib_rules_event(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(dev);
        struct fib_rules_ops *ops;

        ASSERT_RTNL();

        switch (event) {
        case NETDEV_REGISTER:
                list_for_each_entry(ops, &net->rules_ops, list)
                        attach_rules(&ops->rules_list, dev);
                break;

        case NETDEV_CHANGENAME:
                list_for_each_entry(ops, &net->rules_ops, list) {
                        detach_rules(&ops->rules_list, dev);
                        attach_rules(&ops->rules_list, dev);
                }
                break;

        case NETDEV_UNREGISTER:
                list_for_each_entry(ops, &net->rules_ops, list)
                        detach_rules(&ops->rules_list, dev);
                break;
        }

        return NOTIFY_DONE;
}

static struct notifier_block fib_rules_notifier = {
        .notifier_call = fib_rules_event,
};

static int __net_init fib_rules_net_init(struct net *net)
{
        INIT_LIST_HEAD(&net->rules_ops);
        spin_lock_init(&net->rules_mod_lock);
        return 0;
}

static void __net_exit fib_rules_net_exit(struct net *net)
{
        WARN_ON_ONCE(!list_empty(&net->rules_ops));
}

static struct pernet_operations fib_rules_net_ops = {
        .init = fib_rules_net_init,
        .exit = fib_rules_net_exit,
};

static const struct rtnl_msg_handler fib_rules_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWRULE, .doit = fib_nl_newrule,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_DELRULE, .doit = fib_nl_delrule,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_GETRULE, .dumpit = fib_nl_dumprule,
         .flags = RTNL_FLAG_DUMP_UNLOCKED},
};

static int __init fib_rules_init(void)
{
        int err;

        rtnl_register_many(fib_rules_rtnl_msg_handlers);

        err = register_pernet_subsys(&fib_rules_net_ops);
        if (err < 0)
                goto fail;

        err = register_netdevice_notifier(&fib_rules_notifier);
        if (err < 0)
                goto fail_unregister;

        return 0;

fail_unregister:
        unregister_pernet_subsys(&fib_rules_net_ops);
fail:
        rtnl_unregister_many(fib_rules_rtnl_msg_handlers);
        return err;
}

subsys_initcall(fib_rules_init);















































    1 


    1 























   10 














































































































































   90 





  790 






















   44 


   42 



































































































    1 





    1 











  792 









  795 

    4 














  790 



  789 
    2 
    4 
























  792 








    4 

























  794 







  763 










   67 




































   51 































































   79 



    7 






















   78 












































































   10 












   10 


























   39 


















































   39 
   38 
   39 

















   63 
   51 


   38 















    1 



    1 





















   10 







    2 






















    5 










    4 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
// SPDX-License-Identifier: GPL-2.0
/*
 *  inode.c - part of debugfs, a tiny little debug file system
 *
 *  Copyright (C) 2004,2019 Greg Kroah-Hartman <greg@kroah.com>
 *  Copyright (C) 2004 IBM Inc.
 *  Copyright (C) 2019 Linux Foundation <gregkh@linuxfoundation.org>
 *
 *  debugfs is for people to use instead of /proc or /sys.
 *  See ./Documentation/core-api/kernel-api.rst for more details.
 */

#define pr_fmt(fmt)        "debugfs: " fmt

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/kobject.h>
#include <linux/namei.h>
#include <linux/debugfs.h>
#include <linux/fsnotify.h>
#include <linux/string.h>
#include <linux/seq_file.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/security.h>

#include "internal.h"

#define DEBUGFS_DEFAULT_MODE        0700

static struct vfsmount *debugfs_mount;
static int debugfs_mount_count;
static bool debugfs_registered;
static unsigned int debugfs_allow __ro_after_init = DEFAULT_DEBUGFS_ALLOW_BITS;

/*
 * Don't allow access attributes to be changed whilst the kernel is locked down
 * so that we can use the file mode as part of a heuristic to determine whether
 * to lock down individual files.
 */
static int debugfs_setattr(struct mnt_idmap *idmap,
                           struct dentry *dentry, struct iattr *ia)
{
        int ret;

        if (ia->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) {
                ret = security_locked_down(LOCKDOWN_DEBUGFS);
                if (ret)
                        return ret;
        }
        return simple_setattr(&nop_mnt_idmap, dentry, ia);
}

static const struct inode_operations debugfs_file_inode_operations = {
        .setattr        = debugfs_setattr,
};
static const struct inode_operations debugfs_dir_inode_operations = {
        .lookup                = simple_lookup,
        .setattr        = debugfs_setattr,
};
static const struct inode_operations debugfs_symlink_inode_operations = {
        .get_link        = simple_get_link,
        .setattr        = debugfs_setattr,
};

static struct inode *debugfs_get_inode(struct super_block *sb)
{
        struct inode *inode = new_inode(sb);
        if (inode) {
                inode->i_ino = get_next_ino();
                simple_inode_init_ts(inode);
        }
        return inode;
}

struct debugfs_fs_info {
        kuid_t uid;
        kgid_t gid;
        umode_t mode;
        /* Opt_* bitfield. */
        unsigned int opts;
};

enum {
        Opt_uid,
        Opt_gid,
        Opt_mode,
        Opt_source,
};

static const struct fs_parameter_spec debugfs_param_specs[] = {
        fsparam_gid        ("gid",                Opt_gid),
        fsparam_u32oct        ("mode",        Opt_mode),
        fsparam_uid        ("uid",                Opt_uid),
        fsparam_string        ("source",        Opt_source),
        {}
};

static int debugfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        struct debugfs_fs_info *opts = fc->s_fs_info;
        struct fs_parse_result result;
        int opt;

        opt = fs_parse(fc, debugfs_param_specs, param, &result);
        if (opt < 0) {
                /*
                * We might like to report bad mount options here; but
                * traditionally debugfs has ignored all mount options
                */
                if (opt == -ENOPARAM)
                        return 0;

                return opt;
        }

        switch (opt) {
        case Opt_uid:
                opts->uid = result.uid;
                break;
        case Opt_gid:
                opts->gid = result.gid;
                break;
        case Opt_mode:
                opts->mode = result.uint_32 & S_IALLUGO;
                break;
        case Opt_source:
                if (fc->source)
                        return invalfc(fc, "Multiple sources specified");
                fc->source = param->string;
                param->string = NULL;
                break;
        /*
         * We might like to report bad mount options here;
         * but traditionally debugfs has ignored all mount options
         */
        }

        opts->opts |= BIT(opt);

        return 0;
}

static void _debugfs_apply_options(struct super_block *sb, bool remount)
{
        struct debugfs_fs_info *fsi = sb->s_fs_info;
        struct inode *inode = d_inode(sb->s_root);

        /*
         * On remount, only reset mode/uid/gid if they were provided as mount
         * options.
         */

        if (!remount || fsi->opts & BIT(Opt_mode)) {
                inode->i_mode &= ~S_IALLUGO;
                inode->i_mode |= fsi->mode;
        }

        if (!remount || fsi->opts & BIT(Opt_uid))
                inode->i_uid = fsi->uid;

        if (!remount || fsi->opts & BIT(Opt_gid))
                inode->i_gid = fsi->gid;
}

static void debugfs_apply_options(struct super_block *sb)
{
        _debugfs_apply_options(sb, false);
}

static void debugfs_apply_options_remount(struct super_block *sb)
{
        _debugfs_apply_options(sb, true);
}

static int debugfs_reconfigure(struct fs_context *fc)
{
        struct super_block *sb = fc->root->d_sb;
        struct debugfs_fs_info *sb_opts = sb->s_fs_info;
        struct debugfs_fs_info *new_opts = fc->s_fs_info;

        if (!new_opts)
                return 0;

        sync_filesystem(sb);

        /* structure copy of new mount options to sb */
        *sb_opts = *new_opts;
        debugfs_apply_options_remount(sb);

        return 0;
}

static int debugfs_show_options(struct seq_file *m, struct dentry *root)
{
        struct debugfs_fs_info *fsi = root->d_sb->s_fs_info;

        if (!uid_eq(fsi->uid, GLOBAL_ROOT_UID))
                seq_printf(m, ",uid=%u",
                           from_kuid_munged(&init_user_ns, fsi->uid));
        if (!gid_eq(fsi->gid, GLOBAL_ROOT_GID))
                seq_printf(m, ",gid=%u",
                           from_kgid_munged(&init_user_ns, fsi->gid));
        if (fsi->mode != DEBUGFS_DEFAULT_MODE)
                seq_printf(m, ",mode=%o", fsi->mode);

        return 0;
}

static struct kmem_cache *debugfs_inode_cachep __ro_after_init;

static void init_once(void *foo)
{
        struct debugfs_inode_info *info = foo;
        inode_init_once(&info->vfs_inode);
}

static struct inode *debugfs_alloc_inode(struct super_block *sb)
{
        struct debugfs_inode_info *info;
        info = alloc_inode_sb(sb, debugfs_inode_cachep, GFP_KERNEL);
        if (!info)
                return NULL;
        return &info->vfs_inode;
}

static void debugfs_free_inode(struct inode *inode)
{
        if (S_ISLNK(inode->i_mode))
                kfree(inode->i_link);
        kmem_cache_free(debugfs_inode_cachep, DEBUGFS_I(inode));
}

static const struct super_operations debugfs_super_operations = {
        .statfs                = simple_statfs,
        .show_options        = debugfs_show_options,
        .alloc_inode        = debugfs_alloc_inode,
        .free_inode        = debugfs_free_inode,
};

static void debugfs_release_dentry(struct dentry *dentry)
{
        struct debugfs_fsdata *fsd = dentry->d_fsdata;

        if (fsd) {
                WARN_ON(!list_empty(&fsd->cancellations));
                mutex_destroy(&fsd->cancellations_mtx);
        }
        kfree(fsd);
}

static struct vfsmount *debugfs_automount(struct path *path)
{
        struct inode *inode = path->dentry->d_inode;

        return DEBUGFS_I(inode)->automount(path->dentry, inode->i_private);
}

static const struct dentry_operations debugfs_dops = {
        .d_release = debugfs_release_dentry,
        .d_automount = debugfs_automount,
};

static int debugfs_fill_super(struct super_block *sb, struct fs_context *fc)
{
        static const struct tree_descr debug_files[] = {{""}};
        int err;

        err = simple_fill_super(sb, DEBUGFS_MAGIC, debug_files);
        if (err)
                return err;

        sb->s_op = &debugfs_super_operations;
        set_default_d_op(sb, &debugfs_dops);
        sb->s_d_flags |= DCACHE_DONTCACHE;

        debugfs_apply_options(sb);

        return 0;
}

static int debugfs_get_tree(struct fs_context *fc)
{
        int err;

        if (!(debugfs_allow & DEBUGFS_ALLOW_API))
                return -EPERM;

        err = get_tree_single(fc, debugfs_fill_super);
        if (err)
                return err;

        return debugfs_reconfigure(fc);
}

static void debugfs_free_fc(struct fs_context *fc)
{
        kfree(fc->s_fs_info);
}

static const struct fs_context_operations debugfs_context_ops = {
        .free                = debugfs_free_fc,
        .parse_param        = debugfs_parse_param,
        .get_tree        = debugfs_get_tree,
        .reconfigure        = debugfs_reconfigure,
};

static int debugfs_init_fs_context(struct fs_context *fc)
{
        struct debugfs_fs_info *fsi;

        fsi = kzalloc(sizeof(struct debugfs_fs_info), GFP_KERNEL);
        if (!fsi)
                return -ENOMEM;

        fsi->mode = DEBUGFS_DEFAULT_MODE;

        fc->s_fs_info = fsi;
        fc->ops = &debugfs_context_ops;
        return 0;
}

static struct file_system_type debug_fs_type = {
        .owner =        THIS_MODULE,
        .name =                "debugfs",
        .init_fs_context = debugfs_init_fs_context,
        .parameters =        debugfs_param_specs,
        .kill_sb =        kill_litter_super,
};
MODULE_ALIAS_FS("debugfs");

/**
 * debugfs_lookup() - look up an existing debugfs file
 * @name: a pointer to a string containing the name of the file to look up.
 * @parent: a pointer to the parent dentry of the file.
 *
 * This function will return a pointer to a dentry if it succeeds.  If the file
 * doesn't exist or an error occurs, %NULL will be returned.  The returned
 * dentry must be passed to dput() when it is no longer needed.
 *
 * If debugfs is not enabled in the kernel, the value -%ENODEV will be
 * returned.
 */
struct dentry *debugfs_lookup(const char *name, struct dentry *parent)
{
        struct dentry *dentry;

        if (!debugfs_initialized() || IS_ERR_OR_NULL(name) || IS_ERR(parent))
                return NULL;

        if (!parent)
                parent = debugfs_mount->mnt_root;

        dentry = lookup_noperm_positive_unlocked(&QSTR(name), parent);
        if (IS_ERR(dentry))
                return NULL;
        return dentry;
}
EXPORT_SYMBOL_GPL(debugfs_lookup);

static struct dentry *debugfs_start_creating(const char *name,
                                             struct dentry *parent)
{
        struct dentry *dentry;
        int error;

        if (!(debugfs_allow & DEBUGFS_ALLOW_API))
                return ERR_PTR(-EPERM);

        if (!debugfs_initialized())
                return ERR_PTR(-ENOENT);

        pr_debug("creating file '%s'\n", name);

        if (IS_ERR(parent))
                return parent;

        error = simple_pin_fs(&debug_fs_type, &debugfs_mount,
                              &debugfs_mount_count);
        if (error) {
                pr_err("Unable to pin filesystem for file '%s'\n", name);
                return ERR_PTR(error);
        }

        /* If the parent is not specified, we create it in the root.
         * We need the root dentry to do this, which is in the super
         * block. A pointer to that is in the struct vfsmount that we
         * have around.
         */
        if (!parent)
                parent = debugfs_mount->mnt_root;

        dentry = simple_start_creating(parent, name);
        if (IS_ERR(dentry)) {
                if (dentry == ERR_PTR(-EEXIST))
                        pr_err("'%s' already exists in '%pd'\n", name, parent);
                simple_release_fs(&debugfs_mount, &debugfs_mount_count);
        }
        return dentry;
}

static struct dentry *failed_creating(struct dentry *dentry)
{
        inode_unlock(d_inode(dentry->d_parent));
        dput(dentry);
        simple_release_fs(&debugfs_mount, &debugfs_mount_count);
        return ERR_PTR(-ENOMEM);
}

static struct dentry *end_creating(struct dentry *dentry)
{
        inode_unlock(d_inode(dentry->d_parent));
        return dentry;
}

static struct dentry *__debugfs_create_file(const char *name, umode_t mode,
                                struct dentry *parent, void *data,
                                const void *aux,
                                const struct file_operations *proxy_fops,
                                const void *real_fops)
{
        struct dentry *dentry;
        struct inode *inode;

        if (!(mode & S_IFMT))
                mode |= S_IFREG;
        BUG_ON(!S_ISREG(mode));
        dentry = debugfs_start_creating(name, parent);

        if (IS_ERR(dentry))
                return dentry;

        if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
                failed_creating(dentry);
                return ERR_PTR(-EPERM);
        }

        inode = debugfs_get_inode(dentry->d_sb);
        if (unlikely(!inode)) {
                pr_err("out of free dentries, can not create file '%s'\n",
                       name);
                return failed_creating(dentry);
        }

        inode->i_mode = mode;
        inode->i_private = data;

        inode->i_op = &debugfs_file_inode_operations;
        if (!real_fops)
                proxy_fops = &debugfs_noop_file_operations;
        inode->i_fop = proxy_fops;
        DEBUGFS_I(inode)->raw = real_fops;
        DEBUGFS_I(inode)->aux = (void *)aux;

        d_instantiate(dentry, inode);
        fsnotify_create(d_inode(dentry->d_parent), dentry);
        return end_creating(dentry);
}

struct dentry *debugfs_create_file_full(const char *name, umode_t mode,
                                        struct dentry *parent, void *data,
                                        const void *aux,
                                        const struct file_operations *fops)
{
        return __debugfs_create_file(name, mode, parent, data, aux,
                                &debugfs_full_proxy_file_operations,
                                fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_file_full);

struct dentry *debugfs_create_file_short(const char *name, umode_t mode,
                                        struct dentry *parent, void *data,
                                        const void *aux,
                                        const struct debugfs_short_fops *fops)
{
        return __debugfs_create_file(name, mode, parent, data, aux,
                                &debugfs_full_short_proxy_file_operations,
                                fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_file_short);

/**
 * debugfs_create_file_unsafe - create a file in the debugfs filesystem
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @data: a pointer to something that the caller will want to get to later
 *        on.  The inode.i_private pointer will point to this value on
 *        the open() call.
 * @fops: a pointer to a struct file_operations that should be used for
 *        this file.
 *
 * debugfs_create_file_unsafe() is completely analogous to
 * debugfs_create_file(), the only difference being that the fops
 * handed it will not get protected against file removals by the
 * debugfs core.
 *
 * It is your responsibility to protect your struct file_operation
 * methods against file removals by means of debugfs_file_get()
 * and debugfs_file_put(). ->open() is still protected by
 * debugfs though.
 *
 * Any struct file_operations defined by means of
 * DEFINE_DEBUGFS_ATTRIBUTE() is protected against file removals and
 * thus, may be used here.
 */
struct dentry *debugfs_create_file_unsafe(const char *name, umode_t mode,
                                   struct dentry *parent, void *data,
                                   const struct file_operations *fops)
{

        return __debugfs_create_file(name, mode, parent, data, NULL,
                                &debugfs_open_proxy_file_operations,
                                fops);
}
EXPORT_SYMBOL_GPL(debugfs_create_file_unsafe);

/**
 * debugfs_create_file_size - create a file in the debugfs filesystem
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @data: a pointer to something that the caller will want to get to later
 *        on.  The inode.i_private pointer will point to this value on
 *        the open() call.
 * @fops: a pointer to a struct file_operations that should be used for
 *        this file.
 * @file_size: initial file size
 *
 * This is the basic "create a file" function for debugfs.  It allows for a
 * wide range of flexibility in creating a file, or a directory (if you want
 * to create a directory, the debugfs_create_dir() function is
 * recommended to be used instead.)
 */
void debugfs_create_file_size(const char *name, umode_t mode,
                              struct dentry *parent, void *data,
                              const struct file_operations *fops,
                              loff_t file_size)
{
        struct dentry *de = debugfs_create_file(name, mode, parent, data, fops);

        if (!IS_ERR(de))
                d_inode(de)->i_size = file_size;
}
EXPORT_SYMBOL_GPL(debugfs_create_file_size);

/**
 * debugfs_create_dir - create a directory in the debugfs filesystem
 * @name: a pointer to a string containing the name of the directory to
 *        create.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is NULL, then the
 *          directory will be created in the root of the debugfs filesystem.
 *
 * This function creates a directory in debugfs with the given name.
 *
 * This function will return a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the debugfs_remove() function when the file is
 * to be removed (no automatic cleanup happens if your module is unloaded,
 * you are responsible here.)  If an error occurs, ERR_PTR(-ERROR) will be
 * returned.
 *
 * If debugfs is not enabled in the kernel, the value -%ENODEV will be
 * returned.
 *
 * NOTE: it's expected that most callers should _ignore_ the errors returned
 * by this function. Other debugfs functions handle the fact that the "dentry"
 * passed to them could be an error and they don't crash in that case.
 * Drivers should generally work fine even if debugfs fails to init anyway.
 */
struct dentry *debugfs_create_dir(const char *name, struct dentry *parent)
{
        struct dentry *dentry = debugfs_start_creating(name, parent);
        struct inode *inode;

        if (IS_ERR(dentry))
                return dentry;

        if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
                failed_creating(dentry);
                return ERR_PTR(-EPERM);
        }

        inode = debugfs_get_inode(dentry->d_sb);
        if (unlikely(!inode)) {
                pr_err("out of free dentries, can not create directory '%s'\n",
                       name);
                return failed_creating(dentry);
        }

        inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO;
        inode->i_op = &debugfs_dir_inode_operations;
        inode->i_fop = &simple_dir_operations;

        /* directory inodes start off with i_nlink == 2 (for "." entry) */
        inc_nlink(inode);
        d_instantiate(dentry, inode);
        inc_nlink(d_inode(dentry->d_parent));
        fsnotify_mkdir(d_inode(dentry->d_parent), dentry);
        return end_creating(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_create_dir);

/**
 * debugfs_create_automount - create automount point in the debugfs filesystem
 * @name: a pointer to a string containing the name of the file to create.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is NULL, then the
 *          file will be created in the root of the debugfs filesystem.
 * @f: function to be called when pathname resolution steps on that one.
 * @data: opaque argument to pass to f().
 *
 * @f should return what ->d_automount() would.
 */
struct dentry *debugfs_create_automount(const char *name,
                                        struct dentry *parent,
                                        debugfs_automount_t f,
                                        void *data)
{
        struct dentry *dentry = debugfs_start_creating(name, parent);
        struct inode *inode;

        if (IS_ERR(dentry))
                return dentry;

        if (!(debugfs_allow & DEBUGFS_ALLOW_API)) {
                failed_creating(dentry);
                return ERR_PTR(-EPERM);
        }

        inode = debugfs_get_inode(dentry->d_sb);
        if (unlikely(!inode)) {
                pr_err("out of free dentries, can not create automount '%s'\n",
                       name);
                return failed_creating(dentry);
        }

        make_empty_dir_inode(inode);
        inode->i_flags |= S_AUTOMOUNT;
        inode->i_private = data;
        DEBUGFS_I(inode)->automount = f;
        /* directory inodes start off with i_nlink == 2 (for "." entry) */
        inc_nlink(inode);
        d_instantiate(dentry, inode);
        inc_nlink(d_inode(dentry->d_parent));
        fsnotify_mkdir(d_inode(dentry->d_parent), dentry);
        return end_creating(dentry);
}
EXPORT_SYMBOL(debugfs_create_automount);

/**
 * debugfs_create_symlink- create a symbolic link in the debugfs filesystem
 * @name: a pointer to a string containing the name of the symbolic link to
 *        create.
 * @parent: a pointer to the parent dentry for this symbolic link.  This
 *          should be a directory dentry if set.  If this parameter is NULL,
 *          then the symbolic link will be created in the root of the debugfs
 *          filesystem.
 * @target: a pointer to a string containing the path to the target of the
 *          symbolic link.
 *
 * This function creates a symbolic link with the given name in debugfs that
 * links to the given target path.
 *
 * This function will return a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the debugfs_remove() function when the symbolic
 * link is to be removed (no automatic cleanup happens if your module is
 * unloaded, you are responsible here.)  If an error occurs, ERR_PTR(-ERROR)
 * will be returned.
 *
 * If debugfs is not enabled in the kernel, the value -%ENODEV will be
 * returned.
 */
struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent,
                                      const char *target)
{
        struct dentry *dentry;
        struct inode *inode;
        char *link = kstrdup(target, GFP_KERNEL);
        if (!link)
                return ERR_PTR(-ENOMEM);

        dentry = debugfs_start_creating(name, parent);
        if (IS_ERR(dentry)) {
                kfree(link);
                return dentry;
        }

        inode = debugfs_get_inode(dentry->d_sb);
        if (unlikely(!inode)) {
                pr_err("out of free dentries, can not create symlink '%s'\n",
                       name);
                kfree(link);
                return failed_creating(dentry);
        }
        inode->i_mode = S_IFLNK | S_IRWXUGO;
        inode->i_op = &debugfs_symlink_inode_operations;
        inode->i_link = link;
        d_instantiate(dentry, inode);
        return end_creating(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_create_symlink);

static void __debugfs_file_removed(struct dentry *dentry)
{
        struct debugfs_fsdata *fsd;

        /*
         * Paired with the closing smp_mb() implied by a successful
         * cmpxchg() in debugfs_file_get(): either
         * debugfs_file_get() must see a dead dentry or we must see a
         * debugfs_fsdata instance at ->d_fsdata here (or both).
         */
        smp_mb();
        fsd = READ_ONCE(dentry->d_fsdata);
        if (!fsd)
                return;

        /* if this was the last reference, we're done */
        if (refcount_dec_and_test(&fsd->active_users))
                return;

        /*
         * If there's still a reference, the code that obtained it can
         * be in different states:
         *  - The common case of not using cancellations, or already
         *    after debugfs_leave_cancellation(), where we just need
         *    to wait for debugfs_file_put() which signals the completion;
         *  - inside a cancellation section, i.e. between
         *    debugfs_enter_cancellation() and debugfs_leave_cancellation(),
         *    in which case we need to trigger the ->cancel() function,
         *    and then wait for debugfs_file_put() just like in the
         *    previous case;
         *  - before debugfs_enter_cancellation() (but obviously after
         *    debugfs_file_get()), in which case we may not see the
         *    cancellation in the list on the first round of the loop,
         *    but debugfs_enter_cancellation() signals the completion
         *    after adding it, so this code gets woken up to call the
         *    ->cancel() function.
         */
        while (refcount_read(&fsd->active_users)) {
                struct debugfs_cancellation *c;

                /*
                 * Lock the cancellations. Note that the cancellations
                 * structs are meant to be on the stack, so we need to
                 * ensure we either use them here or don't touch them,
                 * and debugfs_leave_cancellation() will wait for this
                 * to be finished processing before exiting one. It may
                 * of course win and remove the cancellation, but then
                 * chances are we never even got into this bit, we only
                 * do if the refcount isn't zero already.
                 */
                mutex_lock(&fsd->cancellations_mtx);
                while ((c = list_first_entry_or_null(&fsd->cancellations,
                                                     typeof(*c), list))) {
                        list_del_init(&c->list);
                        c->cancel(dentry, c->cancel_data);
                }
                mutex_unlock(&fsd->cancellations_mtx);

                wait_for_completion(&fsd->active_users_drained);
        }
}

static void remove_one(struct dentry *victim)
{
        if (d_is_reg(victim))
                __debugfs_file_removed(victim);
        simple_release_fs(&debugfs_mount, &debugfs_mount_count);
}

/**
 * debugfs_remove - recursively removes a directory
 * @dentry: a pointer to a the dentry of the directory to be removed.  If this
 *          parameter is NULL or an error value, nothing will be done.
 *
 * This function recursively removes a directory tree in debugfs that
 * was previously created with a call to another debugfs function
 * (like debugfs_create_file() or variants thereof.)
 *
 * This function is required to be called in order for the file to be
 * removed, no automatic cleanup of files will happen when a module is
 * removed, you are responsible here.
 */
void debugfs_remove(struct dentry *dentry)
{
        if (IS_ERR_OR_NULL(dentry))
                return;

        simple_pin_fs(&debug_fs_type, &debugfs_mount, &debugfs_mount_count);
        simple_recursive_removal(dentry, remove_one);
        simple_release_fs(&debugfs_mount, &debugfs_mount_count);
}
EXPORT_SYMBOL_GPL(debugfs_remove);

/**
 * debugfs_lookup_and_remove - lookup a directory or file and recursively remove it
 * @name: a pointer to a string containing the name of the item to look up.
 * @parent: a pointer to the parent dentry of the item.
 *
 * This is the equlivant of doing something like
 * debugfs_remove(debugfs_lookup(..)) but with the proper reference counting
 * handled for the directory being looked up.
 */
void debugfs_lookup_and_remove(const char *name, struct dentry *parent)
{
        struct dentry *dentry;

        dentry = debugfs_lookup(name, parent);
        if (!dentry)
                return;

        debugfs_remove(dentry);
        dput(dentry);
}
EXPORT_SYMBOL_GPL(debugfs_lookup_and_remove);

/**
 * debugfs_change_name - rename a file/directory in the debugfs filesystem
 * @dentry: dentry of an object to be renamed.
 * @fmt: format for new name
 *
 * This function renames a file/directory in debugfs.  The target must not
 * exist for rename to succeed.
 *
 * This function will return 0 on success and -E... on failure.
 *
 * If debugfs is not enabled in the kernel, the value -%ENODEV will be
 * returned.
 */
int __printf(2, 3) debugfs_change_name(struct dentry *dentry, const char *fmt, ...)
{
        int error = 0;
        const char *new_name;
        struct name_snapshot old_name;
        struct dentry *parent, *target;
        struct inode *dir;
        va_list ap;

        if (IS_ERR_OR_NULL(dentry))
                return 0;

        va_start(ap, fmt);
        new_name = kvasprintf_const(GFP_KERNEL, fmt, ap);
        va_end(ap);
        if (!new_name)
                return -ENOMEM;

        parent = dget_parent(dentry);
        dir = d_inode(parent);
        inode_lock(dir);

        take_dentry_name_snapshot(&old_name, dentry);

        if (WARN_ON_ONCE(dentry->d_parent != parent)) {
                error = -EINVAL;
                goto out;
        }
        if (strcmp(old_name.name.name, new_name) == 0)
                goto out;
        target = lookup_noperm(&QSTR(new_name), parent);
        if (IS_ERR(target)) {
                error = PTR_ERR(target);
                goto out;
        }
        if (d_really_is_positive(target)) {
                dput(target);
                error = -EINVAL;
                goto out;
        }
        simple_rename_timestamp(dir, dentry, dir, target);
        d_move(dentry, target);
        dput(target);
        fsnotify_move(dir, dir, &old_name.name, d_is_dir(dentry), NULL, dentry);
out:
        release_dentry_name_snapshot(&old_name);
        inode_unlock(dir);
        dput(parent);
        kfree_const(new_name);
        return error;
}
EXPORT_SYMBOL_GPL(debugfs_change_name);

/**
 * debugfs_initialized - Tells whether debugfs has been registered
 */
bool debugfs_initialized(void)
{
        return debugfs_registered;
}
EXPORT_SYMBOL_GPL(debugfs_initialized);

static int __init debugfs_kernel(char *str)
{
        if (str) {
                if (!strcmp(str, "on"))
                        debugfs_allow = DEBUGFS_ALLOW_API | DEBUGFS_ALLOW_MOUNT;
                else if (!strcmp(str, "no-mount"))
                        debugfs_allow = DEBUGFS_ALLOW_API;
                else if (!strcmp(str, "off"))
                        debugfs_allow = 0;
        }

        return 0;
}
early_param("debugfs", debugfs_kernel);
static int __init debugfs_init(void)
{
        int retval;

        if (!(debugfs_allow & DEBUGFS_ALLOW_MOUNT))
                return -EPERM;

        retval = sysfs_create_mount_point(kernel_kobj, "debug");
        if (retval)
                return retval;

        debugfs_inode_cachep = kmem_cache_create("debugfs_inode_cache",
                                sizeof(struct debugfs_inode_info), 0,
                                SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
                                init_once);
        if (debugfs_inode_cachep == NULL) {
                sysfs_remove_mount_point(kernel_kobj, "debug");
                return -ENOMEM;
        }

        retval = register_filesystem(&debug_fs_type);
        if (retval) { // Really not going to happen
                sysfs_remove_mount_point(kernel_kobj, "debug");
                kmem_cache_destroy(debugfs_inode_cachep);
                return retval;
        }
        debugfs_registered = true;
        return 0;
}
core_initcall(debugfs_init);












































    2 





    2 






























    5 













    6 





    6 










    2 





    6 
    1 


    1 
    6 



    2 



    1 





    1 







    9 












    9 




    9 

    2 

    1 

    1 






    4 
    5 





    4 
    6 








    5 





    5 



    3 
    2 





    1 
    4 







    5 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
// SPDX-License-Identifier: GPL-2.0
/*
 * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
 *
 * Copyright 2018 Google LLC
 */

/*
 * "NHPoly1305" is the main component of Adiantum hashing.
 * Specifically, it is the calculation
 *
 *        H_L ← Poly1305_{K_L}(NH_{K_N}(pad_{128}(L)))
 *
 * from the procedure in section 6.4 of the Adiantum paper [1].  It is an
 * ε-almost-∆-universal (ε-∆U) hash function for equal-length inputs over
 * Z/(2^{128}Z), where the "∆" operation is addition.  It hashes 1024-byte
 * chunks of the input with the NH hash function [2], reducing the input length
 * by 32x.  The resulting NH digests are evaluated as a polynomial in
 * GF(2^{130}-5), like in the Poly1305 MAC [3].  Note that the polynomial
 * evaluation by itself would suffice to achieve the ε-∆U property; NH is used
 * for performance since it's over twice as fast as Poly1305.
 *
 * This is *not* a cryptographic hash function; do not use it as such!
 *
 * [1] Adiantum: length-preserving encryption for entry-level processors
 *     (https://eprint.iacr.org/2018/720.pdf)
 * [2] UMAC: Fast and Secure Message Authentication
 *     (https://fastcrypto.org/umac/umac_proc.pdf)
 * [3] The Poly1305-AES message-authentication code
 *     (https://cr.yp.to/mac/poly1305-20050329.pdf)
 */

#include <linux/unaligned.h>
#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/poly1305.h>
#include <crypto/nhpoly1305.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/module.h>

static void nh_generic(const u32 *key, const u8 *message, size_t message_len,
                       __le64 hash[NH_NUM_PASSES])
{
        u64 sums[4] = { 0, 0, 0, 0 };

        BUILD_BUG_ON(NH_PAIR_STRIDE != 2);
        BUILD_BUG_ON(NH_NUM_PASSES != 4);

        while (message_len) {
                u32 m0 = get_unaligned_le32(message + 0);
                u32 m1 = get_unaligned_le32(message + 4);
                u32 m2 = get_unaligned_le32(message + 8);
                u32 m3 = get_unaligned_le32(message + 12);

                sums[0] += (u64)(u32)(m0 + key[ 0]) * (u32)(m2 + key[ 2]);
                sums[1] += (u64)(u32)(m0 + key[ 4]) * (u32)(m2 + key[ 6]);
                sums[2] += (u64)(u32)(m0 + key[ 8]) * (u32)(m2 + key[10]);
                sums[3] += (u64)(u32)(m0 + key[12]) * (u32)(m2 + key[14]);
                sums[0] += (u64)(u32)(m1 + key[ 1]) * (u32)(m3 + key[ 3]);
                sums[1] += (u64)(u32)(m1 + key[ 5]) * (u32)(m3 + key[ 7]);
                sums[2] += (u64)(u32)(m1 + key[ 9]) * (u32)(m3 + key[11]);
                sums[3] += (u64)(u32)(m1 + key[13]) * (u32)(m3 + key[15]);
                key += NH_MESSAGE_UNIT / sizeof(key[0]);
                message += NH_MESSAGE_UNIT;
                message_len -= NH_MESSAGE_UNIT;
        }

        hash[0] = cpu_to_le64(sums[0]);
        hash[1] = cpu_to_le64(sums[1]);
        hash[2] = cpu_to_le64(sums[2]);
        hash[3] = cpu_to_le64(sums[3]);
}

/* Pass the next NH hash value through Poly1305 */
static void process_nh_hash_value(struct nhpoly1305_state *state,
                                  const struct nhpoly1305_key *key)
{
        BUILD_BUG_ON(NH_HASH_BYTES % POLY1305_BLOCK_SIZE != 0);

        poly1305_core_blocks(&state->poly_state, &key->poly_key, state->nh_hash,
                             NH_HASH_BYTES / POLY1305_BLOCK_SIZE, 1);
}

/*
 * Feed the next portion of the source data, as a whole number of 16-byte
 * "NH message units", through NH and Poly1305.  Each NH hash is taken over
 * 1024 bytes, except possibly the final one which is taken over a multiple of
 * 16 bytes up to 1024.  Also, in the case where data is passed in misaligned
 * chunks, we combine partial hashes; the end result is the same either way.
 */
static void nhpoly1305_units(struct nhpoly1305_state *state,
                             const struct nhpoly1305_key *key,
                             const u8 *src, unsigned int srclen, nh_t nh_fn)
{
        do {
                unsigned int bytes;

                if (state->nh_remaining == 0) {
                        /* Starting a new NH message */
                        bytes = min_t(unsigned int, srclen, NH_MESSAGE_BYTES);
                        nh_fn(key->nh_key, src, bytes, state->nh_hash);
                        state->nh_remaining = NH_MESSAGE_BYTES - bytes;
                } else {
                        /* Continuing a previous NH message */
                        __le64 tmp_hash[NH_NUM_PASSES];
                        unsigned int pos;
                        int i;

                        pos = NH_MESSAGE_BYTES - state->nh_remaining;
                        bytes = min(srclen, state->nh_remaining);
                        nh_fn(&key->nh_key[pos / 4], src, bytes, tmp_hash);
                        for (i = 0; i < NH_NUM_PASSES; i++)
                                le64_add_cpu(&state->nh_hash[i],
                                             le64_to_cpu(tmp_hash[i]));
                        state->nh_remaining -= bytes;
                }
                if (state->nh_remaining == 0)
                        process_nh_hash_value(state, key);
                src += bytes;
                srclen -= bytes;
        } while (srclen);
}

int crypto_nhpoly1305_setkey(struct crypto_shash *tfm,
                             const u8 *key, unsigned int keylen)
{
        struct nhpoly1305_key *ctx = crypto_shash_ctx(tfm);
        int i;

        if (keylen != NHPOLY1305_KEY_SIZE)
                return -EINVAL;

        poly1305_core_setkey(&ctx->poly_key, key);
        key += POLY1305_BLOCK_SIZE;

        for (i = 0; i < NH_KEY_WORDS; i++)
                ctx->nh_key[i] = get_unaligned_le32(key + i * sizeof(u32));

        return 0;
}
EXPORT_SYMBOL(crypto_nhpoly1305_setkey);

int crypto_nhpoly1305_init(struct shash_desc *desc)
{
        struct nhpoly1305_state *state = shash_desc_ctx(desc);

        poly1305_core_init(&state->poly_state);
        state->buflen = 0;
        state->nh_remaining = 0;
        return 0;
}
EXPORT_SYMBOL(crypto_nhpoly1305_init);

int crypto_nhpoly1305_update_helper(struct shash_desc *desc,
                                    const u8 *src, unsigned int srclen,
                                    nh_t nh_fn)
{
        struct nhpoly1305_state *state = shash_desc_ctx(desc);
        const struct nhpoly1305_key *key = crypto_shash_ctx(desc->tfm);
        unsigned int bytes;

        if (state->buflen) {
                bytes = min(srclen, (int)NH_MESSAGE_UNIT - state->buflen);
                memcpy(&state->buffer[state->buflen], src, bytes);
                state->buflen += bytes;
                if (state->buflen < NH_MESSAGE_UNIT)
                        return 0;
                nhpoly1305_units(state, key, state->buffer, NH_MESSAGE_UNIT,
                                 nh_fn);
                state->buflen = 0;
                src += bytes;
                srclen -= bytes;
        }

        if (srclen >= NH_MESSAGE_UNIT) {
                bytes = round_down(srclen, NH_MESSAGE_UNIT);
                nhpoly1305_units(state, key, src, bytes, nh_fn);
                src += bytes;
                srclen -= bytes;
        }

        if (srclen) {
                memcpy(state->buffer, src, srclen);
                state->buflen = srclen;
        }
        return 0;
}
EXPORT_SYMBOL(crypto_nhpoly1305_update_helper);

int crypto_nhpoly1305_update(struct shash_desc *desc,
                             const u8 *src, unsigned int srclen)
{
        return crypto_nhpoly1305_update_helper(desc, src, srclen, nh_generic);
}
EXPORT_SYMBOL(crypto_nhpoly1305_update);

int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, nh_t nh_fn)
{
        struct nhpoly1305_state *state = shash_desc_ctx(desc);
        const struct nhpoly1305_key *key = crypto_shash_ctx(desc->tfm);

        if (state->buflen) {
                memset(&state->buffer[state->buflen], 0,
                       NH_MESSAGE_UNIT - state->buflen);
                nhpoly1305_units(state, key, state->buffer, NH_MESSAGE_UNIT,
                                 nh_fn);
        }

        if (state->nh_remaining)
                process_nh_hash_value(state, key);

        poly1305_core_emit(&state->poly_state, NULL, dst);
        return 0;
}
EXPORT_SYMBOL(crypto_nhpoly1305_final_helper);

int crypto_nhpoly1305_final(struct shash_desc *desc, u8 *dst)
{
        return crypto_nhpoly1305_final_helper(desc, dst, nh_generic);
}
EXPORT_SYMBOL(crypto_nhpoly1305_final);

static struct shash_alg nhpoly1305_alg = {
        .base.cra_name                = "nhpoly1305",
        .base.cra_driver_name        = "nhpoly1305-generic",
        .base.cra_priority        = 100,
        .base.cra_ctxsize        = sizeof(struct nhpoly1305_key),
        .base.cra_module        = THIS_MODULE,
        .digestsize                = POLY1305_DIGEST_SIZE,
        .init                        = crypto_nhpoly1305_init,
        .update                        = crypto_nhpoly1305_update,
        .final                        = crypto_nhpoly1305_final,
        .setkey                        = crypto_nhpoly1305_setkey,
        .descsize                = sizeof(struct nhpoly1305_state),
};

static int __init nhpoly1305_mod_init(void)
{
        return crypto_register_shash(&nhpoly1305_alg);
}

static void __exit nhpoly1305_mod_exit(void)
{
        crypto_unregister_shash(&nhpoly1305_alg);
}

module_init(nhpoly1305_mod_init);
module_exit(nhpoly1305_mod_exit);

MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
MODULE_ALIAS_CRYPTO("nhpoly1305");
MODULE_ALIAS_CRYPTO("nhpoly1305-generic");






































































    5 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * xfrm_proc.c
 *
 * Copyright (C)2006-2007 USAGI/WIDE Project
 *
 * Authors:        Masahide NAKAMURA <nakam@linux-ipv6.org>
 */
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/export.h>
#include <net/snmp.h>
#include <net/xfrm.h>

static const struct snmp_mib xfrm_mib_list[] = {
        SNMP_MIB_ITEM("XfrmInError", LINUX_MIB_XFRMINERROR),
        SNMP_MIB_ITEM("XfrmInBufferError", LINUX_MIB_XFRMINBUFFERERROR),
        SNMP_MIB_ITEM("XfrmInHdrError", LINUX_MIB_XFRMINHDRERROR),
        SNMP_MIB_ITEM("XfrmInNoStates", LINUX_MIB_XFRMINNOSTATES),
        SNMP_MIB_ITEM("XfrmInStateProtoError", LINUX_MIB_XFRMINSTATEPROTOERROR),
        SNMP_MIB_ITEM("XfrmInStateModeError", LINUX_MIB_XFRMINSTATEMODEERROR),
        SNMP_MIB_ITEM("XfrmInStateSeqError", LINUX_MIB_XFRMINSTATESEQERROR),
        SNMP_MIB_ITEM("XfrmInStateExpired", LINUX_MIB_XFRMINSTATEEXPIRED),
        SNMP_MIB_ITEM("XfrmInStateMismatch", LINUX_MIB_XFRMINSTATEMISMATCH),
        SNMP_MIB_ITEM("XfrmInStateInvalid", LINUX_MIB_XFRMINSTATEINVALID),
        SNMP_MIB_ITEM("XfrmInTmplMismatch", LINUX_MIB_XFRMINTMPLMISMATCH),
        SNMP_MIB_ITEM("XfrmInNoPols", LINUX_MIB_XFRMINNOPOLS),
        SNMP_MIB_ITEM("XfrmInPolBlock", LINUX_MIB_XFRMINPOLBLOCK),
        SNMP_MIB_ITEM("XfrmInPolError", LINUX_MIB_XFRMINPOLERROR),
        SNMP_MIB_ITEM("XfrmOutError", LINUX_MIB_XFRMOUTERROR),
        SNMP_MIB_ITEM("XfrmOutBundleGenError", LINUX_MIB_XFRMOUTBUNDLEGENERROR),
        SNMP_MIB_ITEM("XfrmOutBundleCheckError", LINUX_MIB_XFRMOUTBUNDLECHECKERROR),
        SNMP_MIB_ITEM("XfrmOutNoStates", LINUX_MIB_XFRMOUTNOSTATES),
        SNMP_MIB_ITEM("XfrmOutStateProtoError", LINUX_MIB_XFRMOUTSTATEPROTOERROR),
        SNMP_MIB_ITEM("XfrmOutStateModeError", LINUX_MIB_XFRMOUTSTATEMODEERROR),
        SNMP_MIB_ITEM("XfrmOutStateSeqError", LINUX_MIB_XFRMOUTSTATESEQERROR),
        SNMP_MIB_ITEM("XfrmOutStateExpired", LINUX_MIB_XFRMOUTSTATEEXPIRED),
        SNMP_MIB_ITEM("XfrmOutPolBlock", LINUX_MIB_XFRMOUTPOLBLOCK),
        SNMP_MIB_ITEM("XfrmOutPolDead", LINUX_MIB_XFRMOUTPOLDEAD),
        SNMP_MIB_ITEM("XfrmOutPolError", LINUX_MIB_XFRMOUTPOLERROR),
        SNMP_MIB_ITEM("XfrmFwdHdrError", LINUX_MIB_XFRMFWDHDRERROR),
        SNMP_MIB_ITEM("XfrmOutStateInvalid", LINUX_MIB_XFRMOUTSTATEINVALID),
        SNMP_MIB_ITEM("XfrmAcquireError", LINUX_MIB_XFRMACQUIREERROR),
        SNMP_MIB_ITEM("XfrmOutStateDirError", LINUX_MIB_XFRMOUTSTATEDIRERROR),
        SNMP_MIB_ITEM("XfrmInStateDirError", LINUX_MIB_XFRMINSTATEDIRERROR),
        SNMP_MIB_ITEM("XfrmInIptfsError", LINUX_MIB_XFRMINIPTFSERROR),
        SNMP_MIB_ITEM("XfrmOutNoQueueSpace", LINUX_MIB_XFRMOUTNOQSPACE),
};

static int xfrm_statistics_seq_show(struct seq_file *seq, void *v)
{
        unsigned long buff[ARRAY_SIZE(xfrm_mib_list)];
        const int cnt = ARRAY_SIZE(xfrm_mib_list);
        struct net *net = seq->private;
        int i;

        memset(buff, 0, sizeof(buff));

        xfrm_state_update_stats(net);
        snmp_get_cpu_field_batch_cnt(buff, xfrm_mib_list, cnt,
                                     net->mib.xfrm_statistics);
        for (i = 0; i < cnt; i++)
                seq_printf(seq, "%-24s\t%lu\n", xfrm_mib_list[i].name,
                                                buff[i]);

        return 0;
}

int __net_init xfrm_proc_init(struct net *net)
{
        if (!proc_create_net_single("xfrm_stat", 0444, net->proc_net,
                         xfrm_statistics_seq_show, NULL))
                return -ENOMEM;
        return 0;
}

void xfrm_proc_fini(struct net *net)
{
        remove_proc_entry("xfrm_stat", net->proc_net);
}












































































































    5 



    5 
    5 









































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/virtio.h>
#include <linux/spinlock.h>
#include <linux/virtio_config.h>
#include <linux/virtio_anchor.h>
#include <linux/module.h>
#include <linux/idr.h>
#include <linux/of.h>
#include <uapi/linux/virtio_ids.h>

/* Unique numbering for virtio devices. */
static DEFINE_IDA(virtio_index_ida);

static ssize_t device_show(struct device *_d,
                           struct device_attribute *attr, char *buf)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        return sysfs_emit(buf, "0x%04x\n", dev->id.device);
}
static DEVICE_ATTR_RO(device);

static ssize_t vendor_show(struct device *_d,
                           struct device_attribute *attr, char *buf)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        return sysfs_emit(buf, "0x%04x\n", dev->id.vendor);
}
static DEVICE_ATTR_RO(vendor);

static ssize_t status_show(struct device *_d,
                           struct device_attribute *attr, char *buf)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        return sysfs_emit(buf, "0x%08x\n", dev->config->get_status(dev));
}
static DEVICE_ATTR_RO(status);

static ssize_t modalias_show(struct device *_d,
                             struct device_attribute *attr, char *buf)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        return sysfs_emit(buf, "virtio:d%08Xv%08X\n",
                       dev->id.device, dev->id.vendor);
}
static DEVICE_ATTR_RO(modalias);

static ssize_t features_show(struct device *_d,
                             struct device_attribute *attr, char *buf)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        unsigned int i;
        ssize_t len = 0;

        /* We actually represent this as a bitstring, as it could be
         * arbitrary length in future. */
        for (i = 0; i < VIRTIO_FEATURES_MAX; i++)
                len += sysfs_emit_at(buf, len, "%c",
                               __virtio_test_bit(dev, i) ? '1' : '0');
        len += sysfs_emit_at(buf, len, "\n");
        return len;
}
static DEVICE_ATTR_RO(features);

static struct attribute *virtio_dev_attrs[] = {
        &dev_attr_device.attr,
        &dev_attr_vendor.attr,
        &dev_attr_status.attr,
        &dev_attr_modalias.attr,
        &dev_attr_features.attr,
        NULL,
};
ATTRIBUTE_GROUPS(virtio_dev);

static inline int virtio_id_match(const struct virtio_device *dev,
                                  const struct virtio_device_id *id)
{
        if (id->device != dev->id.device && id->device != VIRTIO_DEV_ANY_ID)
                return 0;

        return id->vendor == VIRTIO_DEV_ANY_ID || id->vendor == dev->id.vendor;
}

/* This looks through all the IDs a driver claims to support.  If any of them
 * match, we return 1 and the kernel will call virtio_dev_probe(). */
static int virtio_dev_match(struct device *_dv, const struct device_driver *_dr)
{
        unsigned int i;
        struct virtio_device *dev = dev_to_virtio(_dv);
        const struct virtio_device_id *ids;

        ids = drv_to_virtio(_dr)->id_table;
        for (i = 0; ids[i].device; i++)
                if (virtio_id_match(dev, &ids[i]))
                        return 1;
        return 0;
}

static int virtio_uevent(const struct device *_dv, struct kobj_uevent_env *env)
{
        const struct virtio_device *dev = dev_to_virtio(_dv);

        return add_uevent_var(env, "MODALIAS=virtio:d%08Xv%08X",
                              dev->id.device, dev->id.vendor);
}

void virtio_check_driver_offered_feature(const struct virtio_device *vdev,
                                         unsigned int fbit)
{
        unsigned int i;
        struct virtio_driver *drv = drv_to_virtio(vdev->dev.driver);

        for (i = 0; i < drv->feature_table_size; i++)
                if (drv->feature_table[i] == fbit)
                        return;

        if (drv->feature_table_legacy) {
                for (i = 0; i < drv->feature_table_size_legacy; i++)
                        if (drv->feature_table_legacy[i] == fbit)
                                return;
        }

        BUG();
}
EXPORT_SYMBOL_GPL(virtio_check_driver_offered_feature);

static void __virtio_config_changed(struct virtio_device *dev)
{
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

        if (!dev->config_core_enabled || dev->config_driver_disabled)
                dev->config_change_pending = true;
        else if (drv && drv->config_changed) {
                drv->config_changed(dev);
                dev->config_change_pending = false;
        }
}

void virtio_config_changed(struct virtio_device *dev)
{
        unsigned long flags;

        spin_lock_irqsave(&dev->config_lock, flags);
        __virtio_config_changed(dev);
        spin_unlock_irqrestore(&dev->config_lock, flags);
}
EXPORT_SYMBOL_GPL(virtio_config_changed);

/**
 * virtio_config_driver_disable - disable config change reporting by drivers
 * @dev: the device to disable
 *
 * This is only allowed to be called by a driver and disabling can't
 * be nested.
 */
void virtio_config_driver_disable(struct virtio_device *dev)
{
        spin_lock_irq(&dev->config_lock);
        dev->config_driver_disabled = true;
        spin_unlock_irq(&dev->config_lock);
}
EXPORT_SYMBOL_GPL(virtio_config_driver_disable);

/**
 * virtio_config_driver_enable - enable config change reporting by drivers
 * @dev: the device to enable
 *
 * This is only allowed to be called by a driver and enabling can't
 * be nested.
 */
void virtio_config_driver_enable(struct virtio_device *dev)
{
        spin_lock_irq(&dev->config_lock);
        dev->config_driver_disabled = false;
        if (dev->config_change_pending)
                __virtio_config_changed(dev);
        spin_unlock_irq(&dev->config_lock);
}
EXPORT_SYMBOL_GPL(virtio_config_driver_enable);

static void virtio_config_core_disable(struct virtio_device *dev)
{
        spin_lock_irq(&dev->config_lock);
        dev->config_core_enabled = false;
        spin_unlock_irq(&dev->config_lock);
}

static void virtio_config_core_enable(struct virtio_device *dev)
{
        spin_lock_irq(&dev->config_lock);
        dev->config_core_enabled = true;
        if (dev->config_change_pending)
                __virtio_config_changed(dev);
        spin_unlock_irq(&dev->config_lock);
}

void virtio_add_status(struct virtio_device *dev, unsigned int status)
{
        might_sleep();
        dev->config->set_status(dev, dev->config->get_status(dev) | status);
}
EXPORT_SYMBOL_GPL(virtio_add_status);

/* Do some validation, then set FEATURES_OK */
static int virtio_features_ok(struct virtio_device *dev)
{
        unsigned int status;

        might_sleep();

        if (virtio_check_mem_acc_cb(dev)) {
                if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) {
                        dev_warn(&dev->dev,
                                 "device must provide VIRTIO_F_VERSION_1\n");
                        return -ENODEV;
                }

                if (!virtio_has_feature(dev, VIRTIO_F_ACCESS_PLATFORM)) {
                        dev_warn(&dev->dev,
                                 "device must provide VIRTIO_F_ACCESS_PLATFORM\n");
                        return -ENODEV;
                }
        }

        if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
                return 0;

        virtio_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
        status = dev->config->get_status(dev);
        if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
                dev_err(&dev->dev, "virtio: device refuses features: %x\n",
                        status);
                return -ENODEV;
        }
        return 0;
}

/**
 * virtio_reset_device - quiesce device for removal
 * @dev: the device to reset
 *
 * Prevents device from sending interrupts and accessing memory.
 *
 * Generally used for cleanup during driver / device removal.
 *
 * Once this has been invoked, caller must ensure that
 * virtqueue_notify / virtqueue_kick are not in progress.
 *
 * Note: this guarantees that vq callbacks are not in progress, however caller
 * is responsible for preventing access from other contexts, such as a system
 * call/workqueue/bh.  Invoking virtio_break_device then flushing any such
 * contexts is one way to handle that.
 * */
void virtio_reset_device(struct virtio_device *dev)
{
#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
        /*
         * The below virtio_synchronize_cbs() guarantees that any
         * interrupt for this line arriving after
         * virtio_synchronize_vqs() has completed is guaranteed to see
         * vq->broken as true.
         */
        virtio_break_device(dev);
        virtio_synchronize_cbs(dev);
#endif

        dev->config->reset(dev);
}
EXPORT_SYMBOL_GPL(virtio_reset_device);

static int virtio_dev_probe(struct device *_d)
{
        int err, i;
        struct virtio_device *dev = dev_to_virtio(_d);
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
        u64 device_features[VIRTIO_FEATURES_DWORDS];
        u64 driver_features[VIRTIO_FEATURES_DWORDS];
        u64 driver_features_legacy;

        /* We have a driver! */
        virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);

        /* Figure out what features the device supports. */
        virtio_get_features(dev, device_features);

        /* Figure out what features the driver supports. */
        virtio_features_zero(driver_features);
        for (i = 0; i < drv->feature_table_size; i++) {
                unsigned int f = drv->feature_table[i];
                if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_MAX))
                        virtio_features_set_bit(driver_features, f);
        }

        /* Some drivers have a separate feature table for virtio v1.0 */
        if (drv->feature_table_legacy) {
                driver_features_legacy = 0;
                for (i = 0; i < drv->feature_table_size_legacy; i++) {
                        unsigned int f = drv->feature_table_legacy[i];
                        if (!WARN_ON_ONCE(f >= 64))
                                driver_features_legacy |= (1ULL << f);
                }
        } else {
                driver_features_legacy = driver_features[0];
        }

        if (virtio_features_test_bit(device_features, VIRTIO_F_VERSION_1)) {
                for (i = 0; i < VIRTIO_FEATURES_DWORDS; ++i)
                        dev->features_array[i] = driver_features[i] &
                                                 device_features[i];
        } else {
                virtio_features_from_u64(dev->features_array,
                                         driver_features_legacy &
                                         device_features[0]);
        }

        /* When debugging, user may filter some features by hand. */
        virtio_debug_device_filter_features(dev);

        /* Transport features always preserved to pass to finalize_features. */
        for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++)
                if (virtio_features_test_bit(device_features, i))
                        __virtio_set_bit(dev, i);

        err = dev->config->finalize_features(dev);
        if (err)
                goto err;

        if (drv->validate) {
                u64 features[VIRTIO_FEATURES_DWORDS];

                virtio_features_copy(features, dev->features_array);
                err = drv->validate(dev);
                if (err)
                        goto err;

                /* Did validation change any features? Then write them again. */
                if (!virtio_features_equal(features, dev->features_array)) {
                        err = dev->config->finalize_features(dev);
                        if (err)
                                goto err;
                }
        }

        err = virtio_features_ok(dev);
        if (err)
                goto err;

        err = drv->probe(dev);
        if (err)
                goto err;

        /* If probe didn't do it, mark device DRIVER_OK ourselves. */
        if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
                virtio_device_ready(dev);

        if (drv->scan)
                drv->scan(dev);

        virtio_config_core_enable(dev);

        return 0;

err:
        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
        return err;

}

static void virtio_dev_remove(struct device *_d)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

        virtio_config_core_disable(dev);

        drv->remove(dev);

        /* Driver should have reset device. */
        WARN_ON_ONCE(dev->config->get_status(dev));

        /* Acknowledge the device's existence again. */
        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

        of_node_put(dev->dev.of_node);
}

/*
 * virtio_irq_get_affinity - get IRQ affinity mask for device
 * @_d: ptr to dev structure
 * @irq_vec: interrupt vector number
 *
 * Return the CPU affinity mask for @_d and @irq_vec.
 */
static const struct cpumask *virtio_irq_get_affinity(struct device *_d,
                                                     unsigned int irq_vec)
{
        struct virtio_device *dev = dev_to_virtio(_d);

        if (!dev->config->get_vq_affinity)
                return NULL;

        return dev->config->get_vq_affinity(dev, irq_vec);
}

static void virtio_dev_shutdown(struct device *_d)
{
        struct virtio_device *dev = dev_to_virtio(_d);
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

        /*
         * Stop accesses to or from the device.
         * We only need to do it if there's a driver - no accesses otherwise.
         */
        if (!drv)
                return;

        /* If the driver has its own shutdown method, use that. */
        if (drv->shutdown) {
                drv->shutdown(dev);
                return;
        }

        /*
         * Some devices get wedged if you kick them after they are
         * reset. Mark all vqs as broken to make sure we don't.
         */
        virtio_break_device(dev);
        /*
         * Guarantee that any callback will see vq->broken as true.
         */
        virtio_synchronize_cbs(dev);
        /*
         * As IOMMUs are reset on shutdown, this will block device access to memory.
         * Some devices get wedged if this happens, so reset to make sure it does not.
         */
        dev->config->reset(dev);
}

static const struct bus_type virtio_bus = {
        .name  = "virtio",
        .match = virtio_dev_match,
        .dev_groups = virtio_dev_groups,
        .uevent = virtio_uevent,
        .probe = virtio_dev_probe,
        .remove = virtio_dev_remove,
        .irq_get_affinity = virtio_irq_get_affinity,
        .shutdown = virtio_dev_shutdown,
};

int __register_virtio_driver(struct virtio_driver *driver, struct module *owner)
{
        /* Catch this early. */
        BUG_ON(driver->feature_table_size && !driver->feature_table);
        driver->driver.bus = &virtio_bus;
        driver->driver.owner = owner;

        return driver_register(&driver->driver);
}
EXPORT_SYMBOL_GPL(__register_virtio_driver);

void unregister_virtio_driver(struct virtio_driver *driver)
{
        driver_unregister(&driver->driver);
}
EXPORT_SYMBOL_GPL(unregister_virtio_driver);

static int virtio_device_of_init(struct virtio_device *dev)
{
        struct device_node *np, *pnode = dev_of_node(dev->dev.parent);
        char compat[] = "virtio,deviceXXXXXXXX";
        int ret, count;

        if (!pnode)
                return 0;

        count = of_get_available_child_count(pnode);
        if (!count)
                return 0;

        /* There can be only 1 child node */
        if (WARN_ON(count > 1))
                return -EINVAL;

        np = of_get_next_available_child(pnode, NULL);
        if (WARN_ON(!np))
                return -ENODEV;

        ret = snprintf(compat, sizeof(compat), "virtio,device%x", dev->id.device);
        BUG_ON(ret >= sizeof(compat));

        /*
         * On powerpc/pseries virtio devices are PCI devices so PCI
         * vendor/device ids play the role of the "compatible" property.
         * Simply don't init of_node in this case.
         */
        if (!of_device_is_compatible(np, compat)) {
                ret = 0;
                goto out;
        }

        dev->dev.of_node = np;
        return 0;

out:
        of_node_put(np);
        return ret;
}

/**
 * register_virtio_device - register virtio device
 * @dev        : virtio device to be registered
 *
 * On error, the caller must call put_device on &@dev->dev (and not kfree),
 * as another code path may have obtained a reference to @dev.
 *
 * Returns: 0 on success, -error on failure
 */
int register_virtio_device(struct virtio_device *dev)
{
        int err;

        dev->dev.bus = &virtio_bus;
        device_initialize(&dev->dev);

        /* Assign a unique device index and hence name. */
        err = ida_alloc(&virtio_index_ida, GFP_KERNEL);
        if (err < 0)
                goto out;

        dev->index = err;
        err = dev_set_name(&dev->dev, "virtio%u", dev->index);
        if (err)
                goto out_ida_remove;

        err = virtio_device_of_init(dev);
        if (err)
                goto out_ida_remove;

        spin_lock_init(&dev->config_lock);
        dev->config_driver_disabled = false;
        dev->config_core_enabled = false;
        dev->config_change_pending = false;

        INIT_LIST_HEAD(&dev->vqs);
        spin_lock_init(&dev->vqs_list_lock);

        /* We always start by resetting the device, in case a previous
         * driver messed it up.  This also tests that code path a little. */
        virtio_reset_device(dev);

        /* Acknowledge that we've seen the device. */
        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

        virtio_debug_device_init(dev);

        /*
         * device_add() causes the bus infrastructure to look for a matching
         * driver.
         */
        err = device_add(&dev->dev);
        if (err)
                goto out_of_node_put;

        return 0;

out_of_node_put:
        of_node_put(dev->dev.of_node);
out_ida_remove:
        ida_free(&virtio_index_ida, dev->index);
out:
        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
        return err;
}
EXPORT_SYMBOL_GPL(register_virtio_device);

bool is_virtio_device(struct device *dev)
{
        return dev->bus == &virtio_bus;
}
EXPORT_SYMBOL_GPL(is_virtio_device);

void unregister_virtio_device(struct virtio_device *dev)
{
        int index = dev->index; /* save for after device release */

        device_unregister(&dev->dev);
        virtio_debug_device_exit(dev);
        ida_free(&virtio_index_ida, index);
}
EXPORT_SYMBOL_GPL(unregister_virtio_device);

static int virtio_device_restore_priv(struct virtio_device *dev, bool restore)
{
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
        int ret;

        /* We always start by resetting the device, in case a previous
         * driver messed it up. */
        virtio_reset_device(dev);

        /* Acknowledge that we've seen the device. */
        virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);

        /* Maybe driver failed before freeze.
         * Restore the failed status, for debugging. */
        if (dev->failed)
                virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);

        if (!drv)
                return 0;

        /* We have a driver! */
        virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);

        ret = dev->config->finalize_features(dev);
        if (ret)
                goto err;

        ret = virtio_features_ok(dev);
        if (ret)
                goto err;

        if (restore) {
                if (drv->restore) {
                        ret = drv->restore(dev);
                        if (ret)
                                goto err;
                }
        } else {
                ret = drv->reset_done(dev);
                if (ret)
                        goto err;
        }

        /* If restore didn't do it, mark device DRIVER_OK ourselves. */
        if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
                virtio_device_ready(dev);

        virtio_config_core_enable(dev);

        return 0;

err:
        virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
        return ret;
}

#ifdef CONFIG_PM_SLEEP
int virtio_device_freeze(struct virtio_device *dev)
{
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
        int ret;

        virtio_config_core_disable(dev);

        dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;

        if (drv && drv->freeze) {
                ret = drv->freeze(dev);
                if (ret) {
                        virtio_config_core_enable(dev);
                        return ret;
                }
        }

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_device_freeze);

int virtio_device_restore(struct virtio_device *dev)
{
        return virtio_device_restore_priv(dev, true);
}
EXPORT_SYMBOL_GPL(virtio_device_restore);
#endif

int virtio_device_reset_prepare(struct virtio_device *dev)
{
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);
        int ret;

        if (!drv || !drv->reset_prepare)
                return -EOPNOTSUPP;

        virtio_config_core_disable(dev);

        dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;

        ret = drv->reset_prepare(dev);
        if (ret) {
                virtio_config_core_enable(dev);
                return ret;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(virtio_device_reset_prepare);

int virtio_device_reset_done(struct virtio_device *dev)
{
        struct virtio_driver *drv = drv_to_virtio(dev->dev.driver);

        if (!drv || !drv->reset_done)
                return -EOPNOTSUPP;

        return virtio_device_restore_priv(dev, false);
}
EXPORT_SYMBOL_GPL(virtio_device_reset_done);

static int virtio_init(void)
{
        BUILD_BUG_ON(offsetof(struct virtio_device, features) !=
                     offsetof(struct virtio_device, features_array[0]));

        if (bus_register(&virtio_bus) != 0)
                panic("virtio bus registration failed");
        virtio_debug_init();
        return 0;
}

static void __exit virtio_exit(void)
{
        virtio_debug_exit();
        bus_unregister(&virtio_bus);
        ida_destroy(&virtio_index_ida);
}
core_initcall(virtio_init);
module_exit(virtio_exit);

MODULE_DESCRIPTION("Virtio core interface");
MODULE_LICENSE("GPL");

























































































    5 


    5 




























    5 




    5 




    5 




    5 

    5 






















































































































































   72 

























    8 








    5 








    6 







    6 



    6 







  257 


  253 

  256 









    1 




   13 


    2 


   11 

   11 
    1 
   11 


















    5 






    5 


























    5 







    5 





    5 















































































  289 
  291 




















   82 





   79 















    5 









    5 
























  582 





  584 

  586 

























































































































    1 


    1 
    1 










    4 


    4 







   25 


    8 


    4 

    1 
   15 



   19 




   17 





   17 

    5 




   12 


   17 




























    5 


















    5 








    1 


    1 





    1 


    1 


    1 



    1 

















    1 








    1 






















    8 














    8 



















    4 






    1 



    2 









    1 









    7 












    7 
    5 


    2 


    1 







    1 



    1 
    2 




    1 




    1 




















    1 





    2 

    2 












































    1 



    4 







    1 








    4 










    7 














    1 
    6 
    1 



    1 

    1 



    7 






    7 


    7 














    7 


























































































































































































































































































































   70 






   69 






   72 
   72 



   69 







   69 
   69 




    1 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
// SPDX-License-Identifier: GPL-2.0-only
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/workqueue.h>
#include <linux/rtnetlink.h>
#include <linux/cache.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/idr.h>
#include <linux/rculist.h>
#include <linux/nsproxy.h>
#include <linux/fs.h>
#include <linux/proc_ns.h>
#include <linux/file.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/net_namespace.h>
#include <linux/sched/task.h>
#include <linux/uidgid.h>
#include <linux/proc_fs.h>
#include <linux/nstree.h>

#include <net/aligned_data.h>
#include <net/sock.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>

/*
 *        Our network namespace constructor/destructor lists
 */

static LIST_HEAD(pernet_list);
static struct list_head *first_device = &pernet_list;

LIST_HEAD(net_namespace_list);
EXPORT_SYMBOL_GPL(net_namespace_list);

/* Protects net_namespace_list. Nests iside rtnl_lock() */
DECLARE_RWSEM(net_rwsem);
EXPORT_SYMBOL_GPL(net_rwsem);

#ifdef CONFIG_KEYS
static struct key_tag init_net_key_domain = { .usage = REFCOUNT_INIT(1) };
#endif

struct net init_net;
EXPORT_SYMBOL(init_net);

static bool init_net_initialized;
/*
 * pernet_ops_rwsem: protects: pernet_list, net_generic_ids,
 * init_net_initialized and first_device pointer.
 * This is internal net namespace object. Please, don't use it
 * outside.
 */
DECLARE_RWSEM(pernet_ops_rwsem);

#define MIN_PERNET_OPS_ID        \
        ((sizeof(struct net_generic) + sizeof(void *) - 1) / sizeof(void *))

#define INITIAL_NET_GEN_PTRS        13 /* +1 for len +2 for rcu_head */

static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;

static struct net_generic *net_alloc_generic(void)
{
        unsigned int gen_ptrs = READ_ONCE(max_gen_ptrs);
        unsigned int generic_size;
        struct net_generic *ng;

        generic_size = offsetof(struct net_generic, ptr[gen_ptrs]);

        ng = kzalloc(generic_size, GFP_KERNEL);
        if (ng)
                ng->s.len = gen_ptrs;

        return ng;
}

static int net_assign_generic(struct net *net, unsigned int id, void *data)
{
        struct net_generic *ng, *old_ng;

        BUG_ON(id < MIN_PERNET_OPS_ID);

        old_ng = rcu_dereference_protected(net->gen,
                                           lockdep_is_held(&pernet_ops_rwsem));
        if (old_ng->s.len > id) {
                old_ng->ptr[id] = data;
                return 0;
        }

        ng = net_alloc_generic();
        if (!ng)
                return -ENOMEM;

        /*
         * Some synchronisation notes:
         *
         * The net_generic explores the net->gen array inside rcu
         * read section. Besides once set the net->gen->ptr[x]
         * pointer never changes (see rules in netns/generic.h).
         *
         * That said, we simply duplicate this array and schedule
         * the old copy for kfree after a grace period.
         */

        memcpy(&ng->ptr[MIN_PERNET_OPS_ID], &old_ng->ptr[MIN_PERNET_OPS_ID],
               (old_ng->s.len - MIN_PERNET_OPS_ID) * sizeof(void *));
        ng->ptr[id] = data;

        rcu_assign_pointer(net->gen, ng);
        kfree_rcu(old_ng, s.rcu);
        return 0;
}

static int ops_init(const struct pernet_operations *ops, struct net *net)
{
        struct net_generic *ng;
        int err = -ENOMEM;
        void *data = NULL;

        if (ops->id) {
                data = kzalloc(ops->size, GFP_KERNEL);
                if (!data)
                        goto out;

                err = net_assign_generic(net, *ops->id, data);
                if (err)
                        goto cleanup;
        }
        err = 0;
        if (ops->init)
                err = ops->init(net);
        if (!err)
                return 0;

        if (ops->id) {
                ng = rcu_dereference_protected(net->gen,
                                               lockdep_is_held(&pernet_ops_rwsem));
                ng->ptr[*ops->id] = NULL;
        }

cleanup:
        kfree(data);

out:
        return err;
}

static void ops_pre_exit_list(const struct pernet_operations *ops,
                              struct list_head *net_exit_list)
{
        struct net *net;

        if (ops->pre_exit) {
                list_for_each_entry(net, net_exit_list, exit_list)
                        ops->pre_exit(net);
        }
}

static void ops_exit_rtnl_list(const struct list_head *ops_list,
                               const struct pernet_operations *ops,
                               struct list_head *net_exit_list)
{
        const struct pernet_operations *saved_ops = ops;
        LIST_HEAD(dev_kill_list);
        struct net *net;

        rtnl_lock();

        list_for_each_entry(net, net_exit_list, exit_list) {
                __rtnl_net_lock(net);

                ops = saved_ops;
                list_for_each_entry_continue_reverse(ops, ops_list, list) {
                        if (ops->exit_rtnl)
                                ops->exit_rtnl(net, &dev_kill_list);
                }

                __rtnl_net_unlock(net);
        }

        unregister_netdevice_many(&dev_kill_list);

        rtnl_unlock();
}

static void ops_exit_list(const struct pernet_operations *ops,
                          struct list_head *net_exit_list)
{
        if (ops->exit) {
                struct net *net;

                list_for_each_entry(net, net_exit_list, exit_list) {
                        ops->exit(net);
                        cond_resched();
                }
        }

        if (ops->exit_batch)
                ops->exit_batch(net_exit_list);
}

static void ops_free_list(const struct pernet_operations *ops,
                          struct list_head *net_exit_list)
{
        struct net *net;

        if (ops->id) {
                list_for_each_entry(net, net_exit_list, exit_list)
                        kfree(net_generic(net, *ops->id));
        }
}

static void ops_undo_list(const struct list_head *ops_list,
                          const struct pernet_operations *ops,
                          struct list_head *net_exit_list,
                          bool expedite_rcu)
{
        const struct pernet_operations *saved_ops;
        bool hold_rtnl = false;

        if (!ops)
                ops = list_entry(ops_list, typeof(*ops), list);

        saved_ops = ops;

        list_for_each_entry_continue_reverse(ops, ops_list, list) {
                hold_rtnl |= !!ops->exit_rtnl;
                ops_pre_exit_list(ops, net_exit_list);
        }

        /* Another CPU might be rcu-iterating the list, wait for it.
         * This needs to be before calling the exit() notifiers, so the
         * rcu_barrier() after ops_undo_list() isn't sufficient alone.
         * Also the pre_exit() and exit() methods need this barrier.
         */
        if (expedite_rcu)
                synchronize_rcu_expedited();
        else
                synchronize_rcu();

        if (hold_rtnl)
                ops_exit_rtnl_list(ops_list, saved_ops, net_exit_list);

        ops = saved_ops;
        list_for_each_entry_continue_reverse(ops, ops_list, list)
                ops_exit_list(ops, net_exit_list);

        ops = saved_ops;
        list_for_each_entry_continue_reverse(ops, ops_list, list)
                ops_free_list(ops, net_exit_list);
}

static void ops_undo_single(struct pernet_operations *ops,
                            struct list_head *net_exit_list)
{
        LIST_HEAD(ops_list);

        list_add(&ops->list, &ops_list);
        ops_undo_list(&ops_list, NULL, net_exit_list, false);
        list_del(&ops->list);
}

/* should be called with nsid_lock held */
static int alloc_netid(struct net *net, struct net *peer, int reqid)
{
        int min = 0, max = 0;

        if (reqid >= 0) {
                min = reqid;
                max = reqid + 1;
        }

        return idr_alloc(&net->netns_ids, peer, min, max, GFP_ATOMIC);
}

/* This function is used by idr_for_each(). If net is equal to peer, the
 * function returns the id so that idr_for_each() stops. Because we cannot
 * returns the id 0 (idr_for_each() will not stop), we return the magic value
 * NET_ID_ZERO (-1) for it.
 */
#define NET_ID_ZERO -1
static int net_eq_idr(int id, void *net, void *peer)
{
        if (net_eq(net, peer))
                return id ? : NET_ID_ZERO;
        return 0;
}

/* Must be called from RCU-critical section or with nsid_lock held */
static int __peernet2id(const struct net *net, struct net *peer)
{
        int id = idr_for_each(&net->netns_ids, net_eq_idr, peer);

        /* Magic value for id 0. */
        if (id == NET_ID_ZERO)
                return 0;
        if (id > 0)
                return id;

        return NETNSA_NSID_NOT_ASSIGNED;
}

static void rtnl_net_notifyid(struct net *net, int cmd, int id, u32 portid,
                              struct nlmsghdr *nlh, gfp_t gfp);
/* This function returns the id of a peer netns. If no id is assigned, one will
 * be allocated and returned.
 */
int peernet2id_alloc(struct net *net, struct net *peer, gfp_t gfp)
{
        int id;

        if (!check_net(net))
                return NETNSA_NSID_NOT_ASSIGNED;

        spin_lock(&net->nsid_lock);
        id = __peernet2id(net, peer);
        if (id >= 0) {
                spin_unlock(&net->nsid_lock);
                return id;
        }

        /* When peer is obtained from RCU lists, we may race with
         * its cleanup. Check whether it's alive, and this guarantees
         * we never hash a peer back to net->netns_ids, after it has
         * just been idr_remove()'d from there in cleanup_net().
         */
        if (!maybe_get_net(peer)) {
                spin_unlock(&net->nsid_lock);
                return NETNSA_NSID_NOT_ASSIGNED;
        }

        id = alloc_netid(net, peer, -1);
        spin_unlock(&net->nsid_lock);

        put_net(peer);
        if (id < 0)
                return NETNSA_NSID_NOT_ASSIGNED;

        rtnl_net_notifyid(net, RTM_NEWNSID, id, 0, NULL, gfp);

        return id;
}
EXPORT_SYMBOL_GPL(peernet2id_alloc);

/* This function returns, if assigned, the id of a peer netns. */
int peernet2id(const struct net *net, struct net *peer)
{
        int id;

        rcu_read_lock();
        id = __peernet2id(net, peer);
        rcu_read_unlock();

        return id;
}
EXPORT_SYMBOL(peernet2id);

/* This function returns true is the peer netns has an id assigned into the
 * current netns.
 */
bool peernet_has_id(const struct net *net, struct net *peer)
{
        return peernet2id(net, peer) >= 0;
}

struct net *get_net_ns_by_id(const struct net *net, int id)
{
        struct net *peer;

        if (id < 0)
                return NULL;

        rcu_read_lock();
        peer = idr_find(&net->netns_ids, id);
        if (peer)
                peer = maybe_get_net(peer);
        rcu_read_unlock();

        return peer;
}
EXPORT_SYMBOL_GPL(get_net_ns_by_id);

static __net_init void preinit_net_sysctl(struct net *net)
{
        net->core.sysctl_somaxconn = SOMAXCONN;
        /* Limits per socket sk_omem_alloc usage.
         * TCP zerocopy regular usage needs 128 KB.
         */
        net->core.sysctl_optmem_max = 128 * 1024;
        net->core.sysctl_txrehash = SOCK_TXREHASH_ENABLED;
        net->core.sysctl_tstamp_allow_data = 1;
}

/* init code that must occur even if setup_net() is not called. */
static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns)
{
        int ret;

        ret = ns_common_init(net);
        if (ret)
                return ret;

        refcount_set(&net->passive, 1);
        ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt");
        ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt");

        get_random_bytes(&net->hash_mix, sizeof(u32));
        net->dev_base_seq = 1;
        net->user_ns = user_ns;

        idr_init(&net->netns_ids);
        spin_lock_init(&net->nsid_lock);
        mutex_init(&net->ipv4.ra_mutex);

#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
        mutex_init(&net->rtnl_mutex);
        lock_set_cmp_fn(&net->rtnl_mutex, rtnl_net_lock_cmp_fn, NULL);
#endif

        INIT_LIST_HEAD(&net->ptype_all);
        INIT_LIST_HEAD(&net->ptype_specific);
        preinit_net_sysctl(net);
        return 0;
}

/*
 * setup_net runs the initializers for the network namespace object.
 */
static __net_init int setup_net(struct net *net)
{
        /* Must be called with pernet_ops_rwsem held */
        const struct pernet_operations *ops;
        LIST_HEAD(net_exit_list);
        int error = 0;

        net->net_cookie = ns_tree_gen_id(&net->ns);

        list_for_each_entry(ops, &pernet_list, list) {
                error = ops_init(ops, net);
                if (error < 0)
                        goto out_undo;
        }
        down_write(&net_rwsem);
        list_add_tail_rcu(&net->list, &net_namespace_list);
        up_write(&net_rwsem);
        ns_tree_add_raw(net);
out:
        return error;

out_undo:
        /* Walk through the list backwards calling the exit functions
         * for the pernet modules whose init functions did not fail.
         */
        list_add(&net->exit_list, &net_exit_list);
        ops_undo_list(&pernet_list, ops, &net_exit_list, false);
        rcu_barrier();
        goto out;
}

#ifdef CONFIG_NET_NS
static struct ucounts *inc_net_namespaces(struct user_namespace *ns)
{
        return inc_ucount(ns, current_euid(), UCOUNT_NET_NAMESPACES);
}

static void dec_net_namespaces(struct ucounts *ucounts)
{
        dec_ucount(ucounts, UCOUNT_NET_NAMESPACES);
}

static struct kmem_cache *net_cachep __ro_after_init;
static struct workqueue_struct *netns_wq;

static struct net *net_alloc(void)
{
        struct net *net = NULL;
        struct net_generic *ng;

        ng = net_alloc_generic();
        if (!ng)
                goto out;

        net = kmem_cache_zalloc(net_cachep, GFP_KERNEL);
        if (!net)
                goto out_free;

#ifdef CONFIG_KEYS
        net->key_domain = kzalloc(sizeof(struct key_tag), GFP_KERNEL);
        if (!net->key_domain)
                goto out_free_2;
        refcount_set(&net->key_domain->usage, 1);
#endif

        rcu_assign_pointer(net->gen, ng);
out:
        return net;

#ifdef CONFIG_KEYS
out_free_2:
        kmem_cache_free(net_cachep, net);
        net = NULL;
#endif
out_free:
        kfree(ng);
        goto out;
}

static LLIST_HEAD(defer_free_list);

static void net_complete_free(void)
{
        struct llist_node *kill_list;
        struct net *net, *next;

        /* Get the list of namespaces to free from last round. */
        kill_list = llist_del_all(&defer_free_list);

        llist_for_each_entry_safe(net, next, kill_list, defer_free_list)
                kmem_cache_free(net_cachep, net);

}

void net_passive_dec(struct net *net)
{
        if (refcount_dec_and_test(&net->passive)) {
                kfree(rcu_access_pointer(net->gen));

                /* There should not be any trackers left there. */
                ref_tracker_dir_exit(&net->notrefcnt_tracker);

                /* Wait for an extra rcu_barrier() before final free. */
                llist_add(&net->defer_free_list, &defer_free_list);
        }
}

void net_drop_ns(void *p)
{
        struct net *net = (struct net *)p;

        if (net)
                net_passive_dec(net);
}

struct net *copy_net_ns(u64 flags,
                        struct user_namespace *user_ns, struct net *old_net)
{
        struct ucounts *ucounts;
        struct net *net;
        int rv;

        if (!(flags & CLONE_NEWNET))
                return get_net(old_net);

        ucounts = inc_net_namespaces(user_ns);
        if (!ucounts)
                return ERR_PTR(-ENOSPC);

        net = net_alloc();
        if (!net) {
                rv = -ENOMEM;
                goto dec_ucounts;
        }

        rv = preinit_net(net, user_ns);
        if (rv < 0)
                goto dec_ucounts;
        net->ucounts = ucounts;
        get_user_ns(user_ns);

        rv = down_read_killable(&pernet_ops_rwsem);
        if (rv < 0)
                goto put_userns;

        rv = setup_net(net);

        up_read(&pernet_ops_rwsem);

        if (rv < 0) {
put_userns:
                ns_common_free(net);
#ifdef CONFIG_KEYS
                key_remove_domain(net->key_domain);
#endif
                put_user_ns(user_ns);
                net_passive_dec(net);
dec_ucounts:
                dec_net_namespaces(ucounts);
                return ERR_PTR(rv);
        }
        return net;
}

/**
 * net_ns_get_ownership - get sysfs ownership data for @net
 * @net: network namespace in question (can be NULL)
 * @uid: kernel user ID for sysfs objects
 * @gid: kernel group ID for sysfs objects
 *
 * Returns the uid/gid pair of root in the user namespace associated with the
 * given network namespace.
 */
void net_ns_get_ownership(const struct net *net, kuid_t *uid, kgid_t *gid)
{
        if (net) {
                kuid_t ns_root_uid = make_kuid(net->user_ns, 0);
                kgid_t ns_root_gid = make_kgid(net->user_ns, 0);

                if (uid_valid(ns_root_uid))
                        *uid = ns_root_uid;

                if (gid_valid(ns_root_gid))
                        *gid = ns_root_gid;
        } else {
                *uid = GLOBAL_ROOT_UID;
                *gid = GLOBAL_ROOT_GID;
        }
}
EXPORT_SYMBOL_GPL(net_ns_get_ownership);

static void unhash_nsid(struct net *net, struct net *last)
{
        struct net *tmp;
        /* This function is only called from cleanup_net() work,
         * and this work is the only process, that may delete
         * a net from net_namespace_list. So, when the below
         * is executing, the list may only grow. Thus, we do not
         * use for_each_net_rcu() or net_rwsem.
         */
        for_each_net(tmp) {
                int id;

                spin_lock(&tmp->nsid_lock);
                id = __peernet2id(tmp, net);
                if (id >= 0)
                        idr_remove(&tmp->netns_ids, id);
                spin_unlock(&tmp->nsid_lock);
                if (id >= 0)
                        rtnl_net_notifyid(tmp, RTM_DELNSID, id, 0, NULL,
                                          GFP_KERNEL);
                if (tmp == last)
                        break;
        }
        spin_lock(&net->nsid_lock);
        idr_destroy(&net->netns_ids);
        spin_unlock(&net->nsid_lock);
}

static LLIST_HEAD(cleanup_list);

struct task_struct *cleanup_net_task;

static void cleanup_net(struct work_struct *work)
{
        struct llist_node *net_kill_list;
        struct net *net, *tmp, *last;
        LIST_HEAD(net_exit_list);

        WRITE_ONCE(cleanup_net_task, current);

        /* Atomically snapshot the list of namespaces to cleanup */
        net_kill_list = llist_del_all(&cleanup_list);

        down_read(&pernet_ops_rwsem);

        /* Don't let anyone else find us. */
        down_write(&net_rwsem);
        llist_for_each_entry(net, net_kill_list, cleanup_list) {
                ns_tree_remove(net);
                list_del_rcu(&net->list);
        }
        /* Cache last net. After we unlock rtnl, no one new net
         * added to net_namespace_list can assign nsid pointer
         * to a net from net_kill_list (see peernet2id_alloc()).
         * So, we skip them in unhash_nsid().
         *
         * Note, that unhash_nsid() does not delete nsid links
         * between net_kill_list's nets, as they've already
         * deleted from net_namespace_list. But, this would be
         * useless anyway, as netns_ids are destroyed there.
         */
        last = list_last_entry(&net_namespace_list, struct net, list);
        up_write(&net_rwsem);

        llist_for_each_entry(net, net_kill_list, cleanup_list) {
                unhash_nsid(net, last);
                list_add_tail(&net->exit_list, &net_exit_list);
        }

        ops_undo_list(&pernet_list, NULL, &net_exit_list, true);

        up_read(&pernet_ops_rwsem);

        /* Ensure there are no outstanding rcu callbacks using this
         * network namespace.
         */
        rcu_barrier();

        net_complete_free();

        /* Finally it is safe to free my network namespace structure */
        list_for_each_entry_safe(net, tmp, &net_exit_list, exit_list) {
                list_del_init(&net->exit_list);
                ns_common_free(net);
                dec_net_namespaces(net->ucounts);
#ifdef CONFIG_KEYS
                key_remove_domain(net->key_domain);
#endif
                put_user_ns(net->user_ns);
                net_passive_dec(net);
        }
        WRITE_ONCE(cleanup_net_task, NULL);
}

/**
 * net_ns_barrier - wait until concurrent net_cleanup_work is done
 *
 * cleanup_net runs from work queue and will first remove namespaces
 * from the global list, then run net exit functions.
 *
 * Call this in module exit path to make sure that all netns
 * ->exit ops have been invoked before the function is removed.
 */
void net_ns_barrier(void)
{
        down_write(&pernet_ops_rwsem);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL(net_ns_barrier);

static DECLARE_WORK(net_cleanup_work, cleanup_net);

void __put_net(struct net *net)
{
        ref_tracker_dir_exit(&net->refcnt_tracker);
        /* Cleanup the network namespace in process context */
        if (llist_add(&net->cleanup_list, &cleanup_list))
                queue_work(netns_wq, &net_cleanup_work);
}
EXPORT_SYMBOL_GPL(__put_net);

/**
 * get_net_ns - increment the refcount of the network namespace
 * @ns: common namespace (net)
 *
 * Returns the net's common namespace or ERR_PTR() if ref is zero.
 */
struct ns_common *get_net_ns(struct ns_common *ns)
{
        struct net *net;

        net = maybe_get_net(container_of(ns, struct net, ns));
        if (net)
                return &net->ns;
        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(get_net_ns);

struct net *get_net_ns_by_fd(int fd)
{
        CLASS(fd, f)(fd);

        if (fd_empty(f))
                return ERR_PTR(-EBADF);

        if (proc_ns_file(fd_file(f))) {
                struct ns_common *ns = get_proc_ns(file_inode(fd_file(f)));
                if (ns->ops == &netns_operations)
                        return get_net(container_of(ns, struct net, ns));
        }

        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(get_net_ns_by_fd);
#endif

struct net *get_net_ns_by_pid(pid_t pid)
{
        struct task_struct *tsk;
        struct net *net;

        /* Lookup the network namespace */
        net = ERR_PTR(-ESRCH);
        rcu_read_lock();
        tsk = find_task_by_vpid(pid);
        if (tsk) {
                struct nsproxy *nsproxy;
                task_lock(tsk);
                nsproxy = tsk->nsproxy;
                if (nsproxy)
                        net = get_net(nsproxy->net_ns);
                task_unlock(tsk);
        }
        rcu_read_unlock();
        return net;
}
EXPORT_SYMBOL_GPL(get_net_ns_by_pid);

#ifdef CONFIG_NET_NS_REFCNT_TRACKER
static void net_ns_net_debugfs(struct net *net)
{
        ref_tracker_dir_symlink(&net->refcnt_tracker, "netns-%llx-%u-refcnt",
                                net->net_cookie, net->ns.inum);
        ref_tracker_dir_symlink(&net->notrefcnt_tracker, "netns-%llx-%u-notrefcnt",
                                net->net_cookie, net->ns.inum);
}

static int __init init_net_debugfs(void)
{
        ref_tracker_dir_debugfs(&init_net.refcnt_tracker);
        ref_tracker_dir_debugfs(&init_net.notrefcnt_tracker);
        net_ns_net_debugfs(&init_net);
        return 0;
}
late_initcall(init_net_debugfs);
#else
static void net_ns_net_debugfs(struct net *net)
{
}
#endif

static __net_init int net_ns_net_init(struct net *net)
{
        net_ns_net_debugfs(net);
        return 0;
}

static struct pernet_operations __net_initdata net_ns_ops = {
        .init = net_ns_net_init,
};

static const struct nla_policy rtnl_net_policy[NETNSA_MAX + 1] = {
        [NETNSA_NONE]                = { .type = NLA_UNSPEC },
        [NETNSA_NSID]                = { .type = NLA_S32 },
        [NETNSA_PID]                = { .type = NLA_U32 },
        [NETNSA_FD]                = { .type = NLA_U32 },
        [NETNSA_TARGET_NSID]        = { .type = NLA_S32 },
};

static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[NETNSA_MAX + 1];
        struct nlattr *nla;
        struct net *peer;
        int nsid, err;

        err = nlmsg_parse_deprecated(nlh, sizeof(struct rtgenmsg), tb,
                                     NETNSA_MAX, rtnl_net_policy, extack);
        if (err < 0)
                return err;
        if (!tb[NETNSA_NSID]) {
                NL_SET_ERR_MSG(extack, "nsid is missing");
                return -EINVAL;
        }
        nsid = nla_get_s32(tb[NETNSA_NSID]);

        if (tb[NETNSA_PID]) {
                peer = get_net_ns_by_pid(nla_get_u32(tb[NETNSA_PID]));
                nla = tb[NETNSA_PID];
        } else if (tb[NETNSA_FD]) {
                peer = get_net_ns_by_fd(nla_get_u32(tb[NETNSA_FD]));
                nla = tb[NETNSA_FD];
        } else {
                NL_SET_ERR_MSG(extack, "Peer netns reference is missing");
                return -EINVAL;
        }
        if (IS_ERR(peer)) {
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack, "Peer netns reference is invalid");
                return PTR_ERR(peer);
        }

        spin_lock(&net->nsid_lock);
        if (__peernet2id(net, peer) >= 0) {
                spin_unlock(&net->nsid_lock);
                err = -EEXIST;
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack,
                               "Peer netns already has a nsid assigned");
                goto out;
        }

        err = alloc_netid(net, peer, nsid);
        spin_unlock(&net->nsid_lock);
        if (err >= 0) {
                rtnl_net_notifyid(net, RTM_NEWNSID, err, NETLINK_CB(skb).portid,
                                  nlh, GFP_KERNEL);
                err = 0;
        } else if (err == -ENOSPC && nsid >= 0) {
                err = -EEXIST;
                NL_SET_BAD_ATTR(extack, tb[NETNSA_NSID]);
                NL_SET_ERR_MSG(extack, "The specified nsid is already used");
        }
out:
        put_net(peer);
        return err;
}

static int rtnl_net_get_size(void)
{
        return NLMSG_ALIGN(sizeof(struct rtgenmsg))
               + nla_total_size(sizeof(s32)) /* NETNSA_NSID */
               + nla_total_size(sizeof(s32)) /* NETNSA_CURRENT_NSID */
               ;
}

struct net_fill_args {
        u32 portid;
        u32 seq;
        int flags;
        int cmd;
        int nsid;
        bool add_ref;
        int ref_nsid;
};

static int rtnl_net_fill(struct sk_buff *skb, struct net_fill_args *args)
{
        struct nlmsghdr *nlh;
        struct rtgenmsg *rth;

        nlh = nlmsg_put(skb, args->portid, args->seq, args->cmd, sizeof(*rth),
                        args->flags);
        if (!nlh)
                return -EMSGSIZE;

        rth = nlmsg_data(nlh);
        rth->rtgen_family = AF_UNSPEC;

        if (nla_put_s32(skb, NETNSA_NSID, args->nsid))
                goto nla_put_failure;

        if (args->add_ref &&
            nla_put_s32(skb, NETNSA_CURRENT_NSID, args->ref_nsid))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int rtnl_net_valid_getid_req(struct sk_buff *skb,
                                    const struct nlmsghdr *nlh,
                                    struct nlattr **tb,
                                    struct netlink_ext_ack *extack)
{
        int i, err;

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct rtgenmsg),
                                              tb, NETNSA_MAX, rtnl_net_policy,
                                              extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct rtgenmsg), tb,
                                            NETNSA_MAX, rtnl_net_policy,
                                            extack);
        if (err)
                return err;

        for (i = 0; i <= NETNSA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETNSA_PID:
                case NETNSA_FD:
                case NETNSA_NSID:
                case NETNSA_TARGET_NSID:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in peer netns getid request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_net_getid(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[NETNSA_MAX + 1];
        struct net_fill_args fillargs = {
                .portid = NETLINK_CB(skb).portid,
                .seq = nlh->nlmsg_seq,
                .cmd = RTM_NEWNSID,
        };
        struct net *peer, *target = net;
        struct nlattr *nla;
        struct sk_buff *msg;
        int err;

        err = rtnl_net_valid_getid_req(skb, nlh, tb, extack);
        if (err < 0)
                return err;
        if (tb[NETNSA_PID]) {
                peer = get_net_ns_by_pid(nla_get_u32(tb[NETNSA_PID]));
                nla = tb[NETNSA_PID];
        } else if (tb[NETNSA_FD]) {
                peer = get_net_ns_by_fd(nla_get_u32(tb[NETNSA_FD]));
                nla = tb[NETNSA_FD];
        } else if (tb[NETNSA_NSID]) {
                peer = get_net_ns_by_id(net, nla_get_s32(tb[NETNSA_NSID]));
                if (!peer)
                        peer = ERR_PTR(-ENOENT);
                nla = tb[NETNSA_NSID];
        } else {
                NL_SET_ERR_MSG(extack, "Peer netns reference is missing");
                return -EINVAL;
        }

        if (IS_ERR(peer)) {
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack, "Peer netns reference is invalid");
                return PTR_ERR(peer);
        }

        if (tb[NETNSA_TARGET_NSID]) {
                int id = nla_get_s32(tb[NETNSA_TARGET_NSID]);

                target = rtnl_get_net_ns_capable(NETLINK_CB(skb).sk, id);
                if (IS_ERR(target)) {
                        NL_SET_BAD_ATTR(extack, tb[NETNSA_TARGET_NSID]);
                        NL_SET_ERR_MSG(extack,
                                       "Target netns reference is invalid");
                        err = PTR_ERR(target);
                        goto out;
                }
                fillargs.add_ref = true;
                fillargs.ref_nsid = peernet2id(net, peer);
        }

        msg = nlmsg_new(rtnl_net_get_size(), GFP_KERNEL);
        if (!msg) {
                err = -ENOMEM;
                goto out;
        }

        fillargs.nsid = peernet2id(target, peer);
        err = rtnl_net_fill(msg, &fillargs);
        if (err < 0)
                goto err_out;

        err = rtnl_unicast(msg, net, NETLINK_CB(skb).portid);
        goto out;

err_out:
        nlmsg_free(msg);
out:
        if (fillargs.add_ref)
                put_net(target);
        put_net(peer);
        return err;
}

struct rtnl_net_dump_cb {
        struct net *tgt_net;
        struct net *ref_net;
        struct sk_buff *skb;
        struct net_fill_args fillargs;
        int idx;
        int s_idx;
};

/* Runs in RCU-critical section. */
static int rtnl_net_dumpid_one(int id, void *peer, void *data)
{
        struct rtnl_net_dump_cb *net_cb = (struct rtnl_net_dump_cb *)data;
        int ret;

        if (net_cb->idx < net_cb->s_idx)
                goto cont;

        net_cb->fillargs.nsid = id;
        if (net_cb->fillargs.add_ref)
                net_cb->fillargs.ref_nsid = __peernet2id(net_cb->ref_net, peer);
        ret = rtnl_net_fill(net_cb->skb, &net_cb->fillargs);
        if (ret < 0)
                return ret;

cont:
        net_cb->idx++;
        return 0;
}

static int rtnl_valid_dump_net_req(const struct nlmsghdr *nlh, struct sock *sk,
                                   struct rtnl_net_dump_cb *net_cb,
                                   struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[NETNSA_MAX + 1];
        int err, i;

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct rtgenmsg), tb,
                                            NETNSA_MAX, rtnl_net_policy,
                                            extack);
        if (err < 0)
                return err;

        for (i = 0; i <= NETNSA_MAX; i++) {
                if (!tb[i])
                        continue;

                if (i == NETNSA_TARGET_NSID) {
                        struct net *net;

                        net = rtnl_get_net_ns_capable(sk, nla_get_s32(tb[i]));
                        if (IS_ERR(net)) {
                                NL_SET_BAD_ATTR(extack, tb[i]);
                                NL_SET_ERR_MSG(extack,
                                               "Invalid target network namespace id");
                                return PTR_ERR(net);
                        }
                        net_cb->fillargs.add_ref = true;
                        net_cb->ref_net = net_cb->tgt_net;
                        net_cb->tgt_net = net;
                } else {
                        NL_SET_BAD_ATTR(extack, tb[i]);
                        NL_SET_ERR_MSG(extack,
                                       "Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_net_dumpid(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rtnl_net_dump_cb net_cb = {
                .tgt_net = sock_net(skb->sk),
                .skb = skb,
                .fillargs = {
                        .portid = NETLINK_CB(cb->skb).portid,
                        .seq = cb->nlh->nlmsg_seq,
                        .flags = NLM_F_MULTI,
                        .cmd = RTM_NEWNSID,
                },
                .idx = 0,
                .s_idx = cb->args[0],
        };
        int err = 0;

        if (cb->strict_check) {
                err = rtnl_valid_dump_net_req(cb->nlh, skb->sk, &net_cb, cb);
                if (err < 0)
                        goto end;
        }

        rcu_read_lock();
        idr_for_each(&net_cb.tgt_net->netns_ids, rtnl_net_dumpid_one, &net_cb);
        rcu_read_unlock();

        cb->args[0] = net_cb.idx;
end:
        if (net_cb.fillargs.add_ref)
                put_net(net_cb.tgt_net);
        return err;
}

static void rtnl_net_notifyid(struct net *net, int cmd, int id, u32 portid,
                              struct nlmsghdr *nlh, gfp_t gfp)
{
        struct net_fill_args fillargs = {
                .portid = portid,
                .seq = nlh ? nlh->nlmsg_seq : 0,
                .cmd = cmd,
                .nsid = id,
        };
        struct sk_buff *msg;
        int err = -ENOMEM;

        msg = nlmsg_new(rtnl_net_get_size(), gfp);
        if (!msg)
                goto out;

        err = rtnl_net_fill(msg, &fillargs);
        if (err < 0)
                goto err_out;

        rtnl_notify(msg, net, portid, RTNLGRP_NSID, nlh, gfp);
        return;

err_out:
        nlmsg_free(msg);
out:
        rtnl_set_sk_err(net, RTNLGRP_NSID, err);
}

#ifdef CONFIG_NET_NS
static void __init netns_ipv4_struct_check(void)
{
        /* TX readonly hotpath cache lines */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_early_retrans);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_tso_win_divisor);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_tso_rtt_log);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_autocorking);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_min_snd_mss);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_notsent_lowat);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_limit_output_bytes);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_min_rtt_wlen);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_wmem);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_ip_fwd_use_pmtu);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_tx, 33);

        /* TXRX readonly hotpath cache lines */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_txrx,
                                      sysctl_tcp_moderate_rcvbuf);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_txrx, 1);

        /* RX readonly hotpath cache line */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_ip_early_demux);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_early_demux);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_l3mdev_accept);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_reordering);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_rmem);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_rx, 22);
}
#endif

static const struct rtnl_msg_handler net_ns_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNSID, .doit = rtnl_net_newid,
         .flags = RTNL_FLAG_DOIT_UNLOCKED},
        {.msgtype = RTM_GETNSID, .doit = rtnl_net_getid,
         .dumpit = rtnl_net_dumpid,
         .flags = RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
};

void __init net_ns_init(void)
{
        struct net_generic *ng;

#ifdef CONFIG_NET_NS
        netns_ipv4_struct_check();
        net_cachep = kmem_cache_create("net_namespace", sizeof(struct net),
                                        SMP_CACHE_BYTES,
                                        SLAB_PANIC|SLAB_ACCOUNT, NULL);

        /* Create workqueue for cleanup */
        netns_wq = create_singlethread_workqueue("netns");
        if (!netns_wq)
                panic("Could not create netns workq");
#endif

        ng = net_alloc_generic();
        if (!ng)
                panic("Could not allocate generic netns");

        rcu_assign_pointer(init_net.gen, ng);

#ifdef CONFIG_KEYS
        init_net.key_domain = &init_net_key_domain;
#endif
        /*
         * This currently cannot fail as the initial network namespace
         * has a static inode number.
         */
        if (preinit_net(&init_net, &init_user_ns))
                panic("Could not preinitialize the initial network namespace");

        down_write(&pernet_ops_rwsem);
        if (setup_net(&init_net))
                panic("Could not setup the initial network namespace");

        init_net_initialized = true;
        up_write(&pernet_ops_rwsem);

        if (register_pernet_subsys(&net_ns_ops))
                panic("Could not register network namespace subsystems");

        rtnl_register_many(net_ns_rtnl_msg_handlers);
}

#ifdef CONFIG_NET_NS
static int __register_pernet_operations(struct list_head *list,
                                        struct pernet_operations *ops)
{
        LIST_HEAD(net_exit_list);
        struct net *net;
        int error;

        list_add_tail(&ops->list, list);
        if (ops->init || ops->id) {
                /* We held write locked pernet_ops_rwsem, and parallel
                 * setup_net() and cleanup_net() are not possible.
                 */
                for_each_net(net) {
                        error = ops_init(ops, net);
                        if (error)
                                goto out_undo;
                        list_add_tail(&net->exit_list, &net_exit_list);
                }
        }
        return 0;

out_undo:
        /* If I have an error cleanup all namespaces I initialized */
        list_del(&ops->list);
        ops_undo_single(ops, &net_exit_list);
        return error;
}

static void __unregister_pernet_operations(struct pernet_operations *ops)
{
        LIST_HEAD(net_exit_list);
        struct net *net;

        /* See comment in __register_pernet_operations() */
        for_each_net(net)
                list_add_tail(&net->exit_list, &net_exit_list);

        list_del(&ops->list);
        ops_undo_single(ops, &net_exit_list);
}

#else

static int __register_pernet_operations(struct list_head *list,
                                        struct pernet_operations *ops)
{
        if (!init_net_initialized) {
                list_add_tail(&ops->list, list);
                return 0;
        }

        return ops_init(ops, &init_net);
}

static void __unregister_pernet_operations(struct pernet_operations *ops)
{
        if (!init_net_initialized) {
                list_del(&ops->list);
        } else {
                LIST_HEAD(net_exit_list);

                list_add(&init_net.exit_list, &net_exit_list);
                ops_undo_single(ops, &net_exit_list);
        }
}

#endif /* CONFIG_NET_NS */

static DEFINE_IDA(net_generic_ids);

static int register_pernet_operations(struct list_head *list,
                                      struct pernet_operations *ops)
{
        int error;

        if (WARN_ON(!!ops->id ^ !!ops->size))
                return -EINVAL;

        if (ops->id) {
                error = ida_alloc_min(&net_generic_ids, MIN_PERNET_OPS_ID,
                                GFP_KERNEL);
                if (error < 0)
                        return error;
                *ops->id = error;
                /* This does not require READ_ONCE as writers already hold
                 * pernet_ops_rwsem. But WRITE_ONCE is needed to protect
                 * net_alloc_generic.
                 */
                WRITE_ONCE(max_gen_ptrs, max(max_gen_ptrs, *ops->id + 1));
        }
        error = __register_pernet_operations(list, ops);
        if (error) {
                rcu_barrier();
                if (ops->id)
                        ida_free(&net_generic_ids, *ops->id);
        }

        return error;
}

static void unregister_pernet_operations(struct pernet_operations *ops)
{
        __unregister_pernet_operations(ops);
        rcu_barrier();
        if (ops->id)
                ida_free(&net_generic_ids, *ops->id);
}

/**
 *      register_pernet_subsys - register a network namespace subsystem
 *        @ops:  pernet operations structure for the subsystem
 *
 *        Register a subsystem which has init and exit functions
 *        that are called when network namespaces are created and
 *        destroyed respectively.
 *
 *        When registered all network namespace init functions are
 *        called for every existing network namespace.  Allowing kernel
 *        modules to have a race free view of the set of network namespaces.
 *
 *        When a new network namespace is created all of the init
 *        methods are called in the order in which they were registered.
 *
 *        When a network namespace is destroyed all of the exit methods
 *        are called in the reverse of the order with which they were
 *        registered.
 */
int register_pernet_subsys(struct pernet_operations *ops)
{
        int error;
        down_write(&pernet_ops_rwsem);
        error =  register_pernet_operations(first_device, ops);
        up_write(&pernet_ops_rwsem);
        return error;
}
EXPORT_SYMBOL_GPL(register_pernet_subsys);

/**
 *      unregister_pernet_subsys - unregister a network namespace subsystem
 *        @ops: pernet operations structure to manipulate
 *
 *        Remove the pernet operations structure from the list to be
 *        used when network namespaces are created or destroyed.  In
 *        addition run the exit method for all existing network
 *        namespaces.
 */
void unregister_pernet_subsys(struct pernet_operations *ops)
{
        down_write(&pernet_ops_rwsem);
        unregister_pernet_operations(ops);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL_GPL(unregister_pernet_subsys);

/**
 *      register_pernet_device - register a network namespace device
 *        @ops:  pernet operations structure for the subsystem
 *
 *        Register a device which has init and exit functions
 *        that are called when network namespaces are created and
 *        destroyed respectively.
 *
 *        When registered all network namespace init functions are
 *        called for every existing network namespace.  Allowing kernel
 *        modules to have a race free view of the set of network namespaces.
 *
 *        When a new network namespace is created all of the init
 *        methods are called in the order in which they were registered.
 *
 *        When a network namespace is destroyed all of the exit methods
 *        are called in the reverse of the order with which they were
 *        registered.
 */
int register_pernet_device(struct pernet_operations *ops)
{
        int error;
        down_write(&pernet_ops_rwsem);
        error = register_pernet_operations(&pernet_list, ops);
        if (!error && (first_device == &pernet_list))
                first_device = &ops->list;
        up_write(&pernet_ops_rwsem);
        return error;
}
EXPORT_SYMBOL_GPL(register_pernet_device);

/**
 *      unregister_pernet_device - unregister a network namespace netdevice
 *        @ops: pernet operations structure to manipulate
 *
 *        Remove the pernet operations structure from the list to be
 *        used when network namespaces are created or destroyed.  In
 *        addition run the exit method for all existing network
 *        namespaces.
 */
void unregister_pernet_device(struct pernet_operations *ops)
{
        down_write(&pernet_ops_rwsem);
        if (&ops->list == first_device)
                first_device = first_device->next;
        unregister_pernet_operations(ops);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL_GPL(unregister_pernet_device);

#ifdef CONFIG_NET_NS
static struct ns_common *netns_get(struct task_struct *task)
{
        struct net *net = NULL;
        struct nsproxy *nsproxy;

        task_lock(task);
        nsproxy = task->nsproxy;
        if (nsproxy)
                net = get_net(nsproxy->net_ns);
        task_unlock(task);

        return net ? &net->ns : NULL;
}

static void netns_put(struct ns_common *ns)
{
        put_net(to_net_ns(ns));
}

static int netns_install(struct nsset *nsset, struct ns_common *ns)
{
        struct nsproxy *nsproxy = nsset->nsproxy;
        struct net *net = to_net_ns(ns);

        if (!ns_capable(net->user_ns, CAP_SYS_ADMIN) ||
            !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        put_net(nsproxy->net_ns);
        nsproxy->net_ns = get_net(net);
        return 0;
}

static struct user_namespace *netns_owner(struct ns_common *ns)
{
        return to_net_ns(ns)->user_ns;
}

const struct proc_ns_operations netns_operations = {
        .name                = "net",
        .get                = netns_get,
        .put                = netns_put,
        .install        = netns_install,
        .owner                = netns_owner,
};
#endif


















































 1807 
 1886 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_USER_H
#define _LINUX_SCHED_USER_H

#include <linux/uidgid.h>
#include <linux/atomic.h>
#include <linux/percpu_counter.h>
#include <linux/refcount.h>
#include <linux/ratelimit.h>

/*
 * Some day this will be a full-fledged user tracking system..
 */
struct user_struct {
        refcount_t __count;        /* reference count */
#ifdef CONFIG_EPOLL
        struct percpu_counter epoll_watches; /* The number of file descriptors currently watched */
#endif
        unsigned long unix_inflight;        /* How many files in flight in unix sockets */
        atomic_long_t pipe_bufs;  /* how many pages are allocated in pipe buffers */

        /* Hash table maintenance information */
        struct hlist_node uidhash_node;
        kuid_t uid;

#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \
        defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \
        defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD)
        atomic_long_t locked_vm;
#endif
#ifdef CONFIG_WATCH_QUEUE
        atomic_t nr_watches;        /* The number of watches this user currently has */
#endif

        /* Miscellaneous per-user rate limit */
        struct ratelimit_state ratelimit;
};

extern int uids_sysfs_init(void);

extern struct user_struct *find_user(kuid_t);

extern struct user_struct root_user;
#define INIT_USER (&root_user)


/* per-UID process charging. */
extern struct user_struct * alloc_uid(kuid_t);
static inline struct user_struct *get_uid(struct user_struct *u)
{
        refcount_inc(&u->__count);
        return u;
}
extern void free_uid(struct user_struct *);

#endif /* _LINUX_SCHED_USER_H */


















































































































   24 


   23 

   16 


   24 














































































































































   25 




    1 









   24 










    5 



   18 


















   19 

   19 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * lib/textsearch.c        Generic text search interface
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 *                 Pablo Neira Ayuso <pablo@netfilter.org>
 *
 * ==========================================================================
 */

/**
 * DOC: ts_intro
 * INTRODUCTION
 *
 *   The textsearch infrastructure provides text searching facilities for
 *   both linear and non-linear data. Individual search algorithms are
 *   implemented in modules and chosen by the user.
 *
 * ARCHITECTURE
 *
 * .. code-block:: none
 *
 *     User
 *     +----------------+
 *     |        finish()|<--------------(6)-----------------+
 *     |get_next_block()|<--------------(5)---------------+ |
 *     |                |                     Algorithm   | |
 *     |                |                    +------------------------------+
 *     |                |                    |  init()   find()   destroy() |
 *     |                |                    +------------------------------+
 *     |                |       Core API           ^       ^          ^
 *     |                |      +---------------+  (2)     (4)        (8)
 *     |             (1)|----->| prepare()     |---+       |          |
 *     |             (3)|----->| find()/next() |-----------+          |
 *     |             (7)|----->| destroy()     |----------------------+
 *     +----------------+      +---------------+
 *
 *   (1) User configures a search by calling textsearch_prepare() specifying
 *       the search parameters such as the pattern and algorithm name.
 *   (2) Core requests the algorithm to allocate and initialize a search
 *       configuration according to the specified parameters.
 *   (3) User starts the search(es) by calling textsearch_find() or
 *       textsearch_next() to fetch subsequent occurrences. A state variable
 *       is provided to the algorithm to store persistent variables.
 *   (4) Core eventually resets the search offset and forwards the find()
 *       request to the algorithm.
 *   (5) Algorithm calls get_next_block() provided by the user continuously
 *       to fetch the data to be searched in block by block.
 *   (6) Algorithm invokes finish() after the last call to get_next_block
 *       to clean up any leftovers from get_next_block. (Optional)
 *   (7) User destroys the configuration by calling textsearch_destroy().
 *   (8) Core notifies the algorithm to destroy algorithm specific
 *       allocations. (Optional)
 *
 * USAGE
 *
 *   Before a search can be performed, a configuration must be created
 *   by calling textsearch_prepare() specifying the searching algorithm,
 *   the pattern to look for and flags. As a flag, you can set TS_IGNORECASE
 *   to perform case insensitive matching. But it might slow down
 *   performance of algorithm, so you should use it at own your risk.
 *   The returned configuration may then be used for an arbitrary
 *   amount of times and even in parallel as long as a separate struct
 *   ts_state variable is provided to every instance.
 *
 *   The actual search is performed by either calling
 *   textsearch_find_continuous() for linear data or by providing
 *   an own get_next_block() implementation and
 *   calling textsearch_find(). Both functions return
 *   the position of the first occurrence of the pattern or UINT_MAX if
 *   no match was found. Subsequent occurrences can be found by calling
 *   textsearch_next() regardless of the linearity of the data.
 *
 *   Once you're done using a configuration it must be given back via
 *   textsearch_destroy.
 *
 * EXAMPLE::
 *
 *   int pos;
 *   struct ts_config *conf;
 *   struct ts_state state;
 *   const char *pattern = "chicken";
 *   const char *example = "We dance the funky chicken";
 *
 *   conf = textsearch_prepare("kmp", pattern, strlen(pattern),
 *                             GFP_KERNEL, TS_AUTOLOAD);
 *   if (IS_ERR(conf)) {
 *       err = PTR_ERR(conf);
 *       goto errout;
 *   }
 *
 *   pos = textsearch_find_continuous(conf, &state, example, strlen(example));
 *   if (pos != UINT_MAX)
 *       panic("Oh my god, dancing chickens at %d\n", pos);
 *
 *   textsearch_destroy(conf);
 */
/* ========================================================================== */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/err.h>
#include <linux/textsearch.h>
#include <linux/slab.h>

static LIST_HEAD(ts_ops);
static DEFINE_SPINLOCK(ts_mod_lock);

static inline struct ts_ops *lookup_ts_algo(const char *name)
{
        struct ts_ops *o;

        rcu_read_lock();
        list_for_each_entry_rcu(o, &ts_ops, list) {
                if (!strcmp(name, o->name)) {
                        if (!try_module_get(o->owner))
                                o = NULL;
                        rcu_read_unlock();
                        return o;
                }
        }
        rcu_read_unlock();

        return NULL;
}

/**
 * textsearch_register - register a textsearch module
 * @ops: operations lookup table
 *
 * This function must be called by textsearch modules to announce
 * their presence. The specified &@ops must have %name set to a
 * unique identifier and the callbacks find(), init(), get_pattern(),
 * and get_pattern_len() must be implemented.
 *
 * Returns 0 or -EEXISTS if another module has already registered
 * with same name.
 */
int textsearch_register(struct ts_ops *ops)
{
        int err = -EEXIST;
        struct ts_ops *o;

        if (ops->name == NULL || ops->find == NULL || ops->init == NULL ||
            ops->get_pattern == NULL || ops->get_pattern_len == NULL)
                return -EINVAL;

        spin_lock(&ts_mod_lock);
        list_for_each_entry(o, &ts_ops, list) {
                if (!strcmp(ops->name, o->name))
                        goto errout;
        }

        list_add_tail_rcu(&ops->list, &ts_ops);
        err = 0;
errout:
        spin_unlock(&ts_mod_lock);
        return err;
}
EXPORT_SYMBOL(textsearch_register);

/**
 * textsearch_unregister - unregister a textsearch module
 * @ops: operations lookup table
 *
 * This function must be called by textsearch modules to announce
 * their disappearance for examples when the module gets unloaded.
 * The &ops parameter must be the same as the one during the
 * registration.
 *
 * Returns 0 on success or -ENOENT if no matching textsearch
 * registration was found.
 */
int textsearch_unregister(struct ts_ops *ops)
{
        int err = 0;
        struct ts_ops *o;

        spin_lock(&ts_mod_lock);
        list_for_each_entry(o, &ts_ops, list) {
                if (o == ops) {
                        list_del_rcu(&o->list);
                        goto out;
                }
        }

        err = -ENOENT;
out:
        spin_unlock(&ts_mod_lock);
        return err;
}
EXPORT_SYMBOL(textsearch_unregister);

struct ts_linear_state
{
        unsigned int        len;
        const void        *data;
};

static unsigned int get_linear_data(unsigned int consumed, const u8 **dst,
                                    struct ts_config *conf,
                                    struct ts_state *state)
{
        struct ts_linear_state *st = (struct ts_linear_state *) state->cb;

        if (likely(consumed < st->len)) {
                *dst = st->data + consumed;
                return st->len - consumed;
        }

        return 0;
}

/**
 * textsearch_find_continuous - search a pattern in continuous/linear data
 * @conf: search configuration
 * @state: search state
 * @data: data to search in
 * @len: length of data
 *
 * A simplified version of textsearch_find() for continuous/linear data.
 * Call textsearch_next() to retrieve subsequent matches.
 *
 * Returns the position of first occurrence of the pattern or
 * %UINT_MAX if no occurrence was found.
 */
unsigned int textsearch_find_continuous(struct ts_config *conf,
                                        struct ts_state *state,
                                        const void *data, unsigned int len)
{
        struct ts_linear_state *st = (struct ts_linear_state *) state->cb;

        conf->get_next_block = get_linear_data;
        st->data = data;
        st->len = len;

        return textsearch_find(conf, state);
}
EXPORT_SYMBOL(textsearch_find_continuous);

/**
 * textsearch_prepare - Prepare a search
 * @algo: name of search algorithm
 * @pattern: pattern data
 * @len: length of pattern
 * @gfp_mask: allocation mask
 * @flags: search flags
 *
 * Looks up the search algorithm module and creates a new textsearch
 * configuration for the specified pattern.
 *
 * Note: The format of the pattern may not be compatible between
 *       the various search algorithms.
 *
 * Returns a new textsearch configuration according to the specified
 * parameters or a ERR_PTR(). If a zero length pattern is passed, this
 * function returns EINVAL.
 */
struct ts_config *textsearch_prepare(const char *algo, const void *pattern,
                                     unsigned int len, gfp_t gfp_mask, int flags)
{
        int err = -ENOENT;
        struct ts_config *conf;
        struct ts_ops *ops;
        
        if (len == 0)
                return ERR_PTR(-EINVAL);

        ops = lookup_ts_algo(algo);
#ifdef CONFIG_MODULES
        /*
         * Why not always autoload you may ask. Some users are
         * in a situation where requesting a module may deadlock,
         * especially when the module is located on a NFS mount.
         */
        if (ops == NULL && flags & TS_AUTOLOAD) {
                request_module("ts_%s", algo);
                ops = lookup_ts_algo(algo);
        }
#endif

        if (ops == NULL)
                goto errout;

        conf = ops->init(pattern, len, gfp_mask, flags);
        if (IS_ERR(conf)) {
                err = PTR_ERR(conf);
                goto errout;
        }

        conf->ops = ops;
        return conf;

errout:
        if (ops)
                module_put(ops->owner);
                
        return ERR_PTR(err);
}
EXPORT_SYMBOL(textsearch_prepare);

/**
 * textsearch_destroy - destroy a search configuration
 * @conf: search configuration
 *
 * Releases all references of the configuration and frees
 * up the memory.
 */
void textsearch_destroy(struct ts_config *conf)
{
        if (conf->ops) {
                if (conf->ops->destroy)
                        conf->ops->destroy(conf);
                module_put(conf->ops->owner);
        }

        kfree(conf);
}
EXPORT_SYMBOL(textsearch_destroy);











































































    4 





























    3 

































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_WAIT_BIT_H
#define _LINUX_WAIT_BIT_H

/*
 * Linux wait-bit related types and methods:
 */
#include <linux/wait.h>

struct wait_bit_key {
        unsigned long                *flags;
        int                        bit_nr;
        unsigned long                timeout;
};

struct wait_bit_queue_entry {
        struct wait_bit_key        key;
        struct wait_queue_entry        wq_entry;
};

#define __WAIT_BIT_KEY_INITIALIZER(word, bit)                                        \
        { .flags = word, .bit_nr = bit, }

typedef int wait_bit_action_f(struct wait_bit_key *key, int mode);

void __wake_up_bit(struct wait_queue_head *wq_head, unsigned long *word, int bit);
int __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
int __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, wait_bit_action_f *action, unsigned int mode);
void wake_up_bit(unsigned long *word, int bit);
int out_of_line_wait_on_bit(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode);
int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode, unsigned long timeout);
int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode);
struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit);
extern void __init wait_bit_init(void);

int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);

#define DEFINE_WAIT_BIT(name, word, bit)                                        \
        struct wait_bit_queue_entry name = {                                        \
                .key = __WAIT_BIT_KEY_INITIALIZER(word, bit),                        \
                .wq_entry = {                                                        \
                        .private        = current,                                \
                        .func                = wake_bit_function,                        \
                        .entry                =                                        \
                                LIST_HEAD_INIT((name).wq_entry.entry),                \
                },                                                                \
        }

extern int bit_wait(struct wait_bit_key *key, int mode);
extern int bit_wait_io(struct wait_bit_key *key, int mode);
extern int bit_wait_timeout(struct wait_bit_key *key, int mode);

/**
 * wait_on_bit - wait for a bit to be cleared
 * @word: the address containing the bit being waited on
 * @bit: the bit at that address being waited on
 * @mode: the task state to sleep in
 *
 * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
 * to be cleared.  The clearing of the bit must be signalled with
 * wake_up_bit(), often as clear_and_wake_up_bit().
 *
 * The process will wait on a waitqueue selected by hash from a shared
 * pool.  It will only be woken on a wake_up for the target bit, even
 * if other processes on the same queue are waiting for other bits.
 *
 * Returned value will be zero if the bit was cleared in which case the
 * call has ACQUIRE semantics, or %-EINTR if the process received a
 * signal and the mode permitted wake up on that signal.
 */
static inline int
wait_on_bit(unsigned long *word, int bit, unsigned mode)
{
        might_sleep();
        if (!test_bit_acquire(bit, word))
                return 0;
        return out_of_line_wait_on_bit(word, bit,
                                       bit_wait,
                                       mode);
}

/**
 * wait_on_bit_io - wait for a bit to be cleared
 * @word: the address containing the bit being waited on
 * @bit: the bit at that address being waited on
 * @mode: the task state to sleep in
 *
 * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
 * to be cleared.  The clearing of the bit must be signalled with
 * wake_up_bit(), often as clear_and_wake_up_bit().
 *
 * This is similar to wait_on_bit(), but calls io_schedule() instead of
 * schedule() for the actual waiting.
 *
 * Returned value will be zero if the bit was cleared in which case the
 * call has ACQUIRE semantics, or %-EINTR if the process received a
 * signal and the mode permitted wake up on that signal.
 */
static inline int
wait_on_bit_io(unsigned long *word, int bit, unsigned mode)
{
        might_sleep();
        if (!test_bit_acquire(bit, word))
                return 0;
        return out_of_line_wait_on_bit(word, bit,
                                       bit_wait_io,
                                       mode);
}

/**
 * wait_on_bit_timeout - wait for a bit to be cleared or a timeout to elapse
 * @word: the address containing the bit being waited on
 * @bit: the bit at that address being waited on
 * @mode: the task state to sleep in
 * @timeout: timeout, in jiffies
 *
 * Wait for the given bit in an unsigned long or bitmap (see
 * DECLARE_BITMAP()) to be cleared, or for a timeout to expire.  The
 * clearing of the bit must be signalled with wake_up_bit(), often as
 * clear_and_wake_up_bit().
 *
 * This is similar to wait_on_bit(), except it also takes a timeout
 * parameter.
 *
 * Returned value will be zero if the bit was cleared in which case the
 * call has ACQUIRE semantics, or %-EINTR if the process received a
 * signal and the mode permitted wake up on that signal, or %-EAGAIN if the
 * timeout elapsed.
 */
static inline int
wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode,
                    unsigned long timeout)
{
        might_sleep();
        if (!test_bit_acquire(bit, word))
                return 0;
        return out_of_line_wait_on_bit_timeout(word, bit,
                                               bit_wait_timeout,
                                               mode, timeout);
}

/**
 * wait_on_bit_action - wait for a bit to be cleared
 * @word: the address containing the bit waited on
 * @bit: the bit at that address being waited on
 * @action: the function used to sleep, which may take special actions
 * @mode: the task state to sleep in
 *
 * Wait for the given bit in an unsigned long or bitmap (see DECLARE_BITMAP())
 * to be cleared.  The clearing of the bit must be signalled with
 * wake_up_bit(), often as clear_and_wake_up_bit().
 *
 * This is similar to wait_on_bit(), but calls @action() instead of
 * schedule() for the actual waiting.
 *
 * Returned value will be zero if the bit was cleared in which case the
 * call has ACQUIRE semantics, or the error code returned by @action if
 * that call returned non-zero.
 */
static inline int
wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action,
                   unsigned mode)
{
        might_sleep();
        if (!test_bit_acquire(bit, word))
                return 0;
        return out_of_line_wait_on_bit(word, bit, action, mode);
}

/**
 * wait_on_bit_lock - wait for a bit to be cleared, then set it
 * @word: the address containing the bit being waited on
 * @bit: the bit of the word being waited on and set
 * @mode: the task state to sleep in
 *
 * Wait for the given bit in an unsigned long or bitmap (see
 * DECLARE_BITMAP()) to be cleared.  The clearing of the bit must be
 * signalled with wake_up_bit(), often as clear_and_wake_up_bit().  As
 * soon as it is clear, atomically set it and return.
 *
 * This is similar to wait_on_bit(), but sets the bit before returning.
 *
 * Returned value will be zero if the bit was successfully set in which
 * case the call has the same memory sequencing semantics as
 * test_and_clear_bit(), or %-EINTR if the process received a signal and
 * the mode permitted wake up on that signal.
 */
static inline int
wait_on_bit_lock(unsigned long *word, int bit, unsigned mode)
{
        might_sleep();
        if (!test_and_set_bit(bit, word))
                return 0;
        return out_of_line_wait_on_bit_lock(word, bit, bit_wait, mode);
}

/**
 * wait_on_bit_lock_io - wait for a bit to be cleared, then set it
 * @word: the address containing the bit being waited on
 * @bit: the bit of the word being waited on and set
 * @mode: the task state to sleep in
 *
 * Wait for the given bit in an unsigned long or bitmap (see
 * DECLARE_BITMAP()) to be cleared.  The clearing of the bit must be
 * signalled with wake_up_bit(), often as clear_and_wake_up_bit().  As
 * soon as it is clear, atomically set it and return.
 *
 * This is similar to wait_on_bit_lock(), but calls io_schedule() instead
 * of schedule().
 *
 * Returns zero if the bit was (eventually) found to be clear and was
 * set.  Returns non-zero if a signal was delivered to the process and
 * the @mode allows that signal to wake the process.
 */
static inline int
wait_on_bit_lock_io(unsigned long *word, int bit, unsigned mode)
{
        might_sleep();
        if (!test_and_set_bit(bit, word))
                return 0;
        return out_of_line_wait_on_bit_lock(word, bit, bit_wait_io, mode);
}

/**
 * wait_on_bit_lock_action - wait for a bit to be cleared, then set it
 * @word: the address containing the bit being waited on
 * @bit: the bit of the word being waited on and set
 * @action: the function used to sleep, which may take special actions
 * @mode: the task state to sleep in
 *
 * This is similar to wait_on_bit_lock(), but calls @action() instead of
 * schedule() for the actual waiting.
 *
 * Returned value will be zero if the bit was successfully set in which
 * case the call has the same memory sequencing semantics as
 * test_and_clear_bit(), or the error code returned by @action if that
 * call returned non-zero.
 */
static inline int
wait_on_bit_lock_action(unsigned long *word, int bit, wait_bit_action_f *action,
                        unsigned mode)
{
        might_sleep();
        if (!test_and_set_bit(bit, word))
                return 0;
        return out_of_line_wait_on_bit_lock(word, bit, action, mode);
}

extern void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags);
extern void wake_up_var(void *var);
extern wait_queue_head_t *__var_waitqueue(void *p);

#define ___wait_var_event(var, condition, state, exclusive, ret, cmd)        \
({                                                                        \
        __label__ __out;                                                \
        struct wait_queue_head *__wq_head = __var_waitqueue(var);        \
        struct wait_bit_queue_entry __wbq_entry;                        \
        long __ret = ret; /* explicit shadow */                                \
                                                                        \
        init_wait_var_entry(&__wbq_entry, var,                                \
                            exclusive ? WQ_FLAG_EXCLUSIVE : 0);                \
        for (;;) {                                                        \
                long __int = prepare_to_wait_event(__wq_head,                \
                                                   &__wbq_entry.wq_entry, \
                                                   state);                \
                if (condition)                                                \
                        break;                                                \
                                                                        \
                if (___wait_is_interruptible(state) && __int) {                \
                        __ret = __int;                                        \
                        goto __out;                                        \
                }                                                        \
                                                                        \
                cmd;                                                        \
        }                                                                \
        finish_wait(__wq_head, &__wbq_entry.wq_entry);                        \
__out:        __ret;                                                                \
})

#define __wait_var_event(var, condition)                                \
        ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                          schedule())
#define __wait_var_event_io(var, condition)                                \
        ___wait_var_event(var, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                          io_schedule())

/**
 * wait_var_event - wait for a variable to be updated and notified
 * @var: the address of variable being waited on
 * @condition: the condition to wait for
 *
 * Wait for a @condition to be true, only re-checking when a wake up is
 * received for the given @var (an arbitrary kernel address which need
 * not be directly related to the given condition, but usually is).
 *
 * The process will wait on a waitqueue selected by hash from a shared
 * pool.  It will only be woken on a wake_up for the given address.
 *
 * The condition should normally use smp_load_acquire() or a similarly
 * ordered access to ensure that any changes to memory made before the
 * condition became true will be visible after the wait completes.
 */
#define wait_var_event(var, condition)                                        \
do {                                                                        \
        might_sleep();                                                        \
        if (condition)                                                        \
                break;                                                        \
        __wait_var_event(var, condition);                                \
} while (0)

/**
 * wait_var_event_io - wait for a variable to be updated and notified
 * @var: the address of variable being waited on
 * @condition: the condition to wait for
 *
 * Wait for an IO related @condition to be true, only re-checking when a
 * wake up is received for the given @var (an arbitrary kernel address
 * which need not be directly related to the given condition, but
 * usually is).
 *
 * The process will wait on a waitqueue selected by hash from a shared
 * pool.  It will only be woken on a wake_up for the given address.
 *
 * This is similar to wait_var_event(), but calls io_schedule() instead
 * of schedule().
 *
 * The condition should normally use smp_load_acquire() or a similarly
 * ordered access to ensure that any changes to memory made before the
 * condition became true will be visible after the wait completes.
 */
#define wait_var_event_io(var, condition)                                \
do {                                                                        \
        might_sleep();                                                        \
        if (condition)                                                        \
                break;                                                        \
        __wait_var_event_io(var, condition);                                \
} while (0)

#define __wait_var_event_killable(var, condition)                        \
        ___wait_var_event(var, condition, TASK_KILLABLE, 0, 0,                \
                          schedule())

/**
 * wait_var_event_killable - wait for a variable to be updated and notified
 * @var: the address of variable being waited on
 * @condition: the condition to wait for
 *
 * Wait for a @condition to be true or a fatal signal to be received,
 * only re-checking the condition when a wake up is received for the given
 * @var (an arbitrary kernel address which need not be directly related
 * to the given condition, but usually is).
 *
 * This is similar to wait_var_event() but returns a value which is
 * 0 if the condition became true, or %-ERESTARTSYS if a fatal signal
 * was received.
 *
 * The condition should normally use smp_load_acquire() or a similarly
 * ordered access to ensure that any changes to memory made before the
 * condition became true will be visible after the wait completes.
 */
#define wait_var_event_killable(var, condition)                                \
({                                                                        \
        int __ret = 0;                                                        \
        might_sleep();                                                        \
        if (!(condition))                                                \
                __ret = __wait_var_event_killable(var, condition);        \
        __ret;                                                                \
})

#define __wait_var_event_timeout(var, condition, timeout)                \
        ___wait_var_event(var, ___wait_cond_timeout(condition),                \
                          TASK_UNINTERRUPTIBLE, 0, timeout,                \
                          __ret = schedule_timeout(__ret))

/**
 * wait_var_event_timeout - wait for a variable to be updated or a timeout to expire
 * @var: the address of variable being waited on
 * @condition: the condition to wait for
 * @timeout: maximum time to wait in jiffies
 *
 * Wait for a @condition to be true or a timeout to expire, only
 * re-checking the condition when a wake up is received for the given
 * @var (an arbitrary kernel address which need not be directly related
 * to the given condition, but usually is).
 *
 * This is similar to wait_var_event() but returns a value which is 0 if
 * the timeout expired and the condition was still false, or the
 * remaining time left in the timeout (but at least 1) if the condition
 * was found to be true.
 *
 * The condition should normally use smp_load_acquire() or a similarly
 * ordered access to ensure that any changes to memory made before the
 * condition became true will be visible after the wait completes.
 */
#define wait_var_event_timeout(var, condition, timeout)                        \
({                                                                        \
        long __ret = timeout;                                                \
        might_sleep();                                                        \
        if (!___wait_cond_timeout(condition))                                \
                __ret = __wait_var_event_timeout(var, condition, timeout); \
        __ret;                                                                \
})

#define __wait_var_event_interruptible(var, condition)                        \
        ___wait_var_event(var, condition, TASK_INTERRUPTIBLE, 0, 0,        \
                          schedule())

/**
 * wait_var_event_killable - wait for a variable to be updated and notified
 * @var: the address of variable being waited on
 * @condition: the condition to wait for
 *
 * Wait for a @condition to be true or a signal to be received, only
 * re-checking the condition when a wake up is received for the given
 * @var (an arbitrary kernel address which need not be directly related
 * to the given condition, but usually is).
 *
 * This is similar to wait_var_event() but returns a value which is 0 if
 * the condition became true, or %-ERESTARTSYS if a signal was received.
 *
 * The condition should normally use smp_load_acquire() or a similarly
 * ordered access to ensure that any changes to memory made before the
 * condition became true will be visible after the wait completes.
 */
#define wait_var_event_interruptible(var, condition)                        \
({                                                                        \
        int __ret = 0;                                                        \
        might_sleep();                                                        \
        if (!(condition))                                                \
                __ret = __wait_var_event_interruptible(var, condition);        \
        __ret;                                                                \
})

/**
 * wait_var_event_any_lock - wait for a variable to be updated under a lock
 * @var: the address of the variable being waited on
 * @condition: condition to wait for
 * @lock: the object that is locked to protect updates to the variable
 * @type: prefix on lock and unlock operations
 * @state: waiting state, %TASK_UNINTERRUPTIBLE etc.
 *
 * Wait for a condition which can only be reliably tested while holding
 * a lock.  The variables assessed in the condition will normal be updated
 * under the same lock, and the wake up should be signalled with
 * wake_up_var_locked() under the same lock.
 *
 * This is similar to wait_var_event(), but assumes a lock is held
 * while calling this function and while updating the variable.
 *
 * This must be called while the given lock is held and the lock will be
 * dropped when schedule() is called to wait for a wake up, and will be
 * reclaimed before testing the condition again.  The functions used to
 * unlock and lock the object are constructed by appending _unlock and _lock
 * to @type.
 *
 * Return %-ERESTARTSYS if a signal arrives which is allowed to interrupt
 * the wait according to @state.
 */
#define wait_var_event_any_lock(var, condition, lock, type, state)        \
({                                                                        \
        int __ret = 0;                                                        \
        if (!(condition))                                                \
                __ret = ___wait_var_event(var, condition, state, 0, 0,        \
                                          type ## _unlock(lock);        \
                                          schedule();                        \
                                          type ## _lock(lock));                \
        __ret;                                                                \
})

/**
 * wait_var_event_spinlock - wait for a variable to be updated under a spinlock
 * @var: the address of the variable being waited on
 * @condition: condition to wait for
 * @lock: the spinlock which protects updates to the variable
 *
 * Wait for a condition which can only be reliably tested while holding
 * a spinlock.  The variables assessed in the condition will normal be updated
 * under the same spinlock, and the wake up should be signalled with
 * wake_up_var_locked() under the same spinlock.
 *
 * This is similar to wait_var_event(), but assumes a spinlock is held
 * while calling this function and while updating the variable.
 *
 * This must be called while the given lock is held and the lock will be
 * dropped when schedule() is called to wait for a wake up, and will be
 * reclaimed before testing the condition again.
 */
#define wait_var_event_spinlock(var, condition, lock)                        \
        wait_var_event_any_lock(var, condition, lock, spin, TASK_UNINTERRUPTIBLE)

/**
 * wait_var_event_mutex - wait for a variable to be updated under a mutex
 * @var: the address of the variable being waited on
 * @condition: condition to wait for
 * @mutex: the mutex which protects updates to the variable
 *
 * Wait for a condition which can only be reliably tested while holding
 * a mutex.  The variables assessed in the condition will normal be
 * updated under the same mutex, and the wake up should be signalled
 * with wake_up_var_locked() under the same mutex.
 *
 * This is similar to wait_var_event(), but assumes a mutex is held
 * while calling this function and while updating the variable.
 *
 * This must be called while the given mutex is held and the mutex will be
 * dropped when schedule() is called to wait for a wake up, and will be
 * reclaimed before testing the condition again.
 */
#define wait_var_event_mutex(var, condition, lock)                        \
        wait_var_event_any_lock(var, condition, lock, mutex, TASK_UNINTERRUPTIBLE)

/**
 * wake_up_var_protected - wake up waiters for a variable asserting that it is safe
 * @var: the address of the variable being waited on
 * @cond: the condition which afirms this is safe
 *
 * When waking waiters which use wait_var_event_any_lock() the waker must be
 * holding the reelvant lock to avoid races.  This version of wake_up_var()
 * asserts that the relevant lock is held and so no barrier is needed.
 * The @cond is only tested when CONFIG_LOCKDEP is enabled.
 */
#define wake_up_var_protected(var, cond)                                \
do {                                                                        \
        lockdep_assert(cond);                                                \
        wake_up_var(var);                                                \
} while (0)

/**
 * wake_up_var_locked - wake up waiters for a variable while holding a spinlock or mutex
 * @var: the address of the variable being waited on
 * @lock: The spinlock or mutex what protects the variable
 *
 * Send a wake up for the given variable which should be waited for with
 * wait_var_event_spinlock() or wait_var_event_mutex().  Unlike wake_up_var(),
 * no extra barriers are needed as the locking provides sufficient sequencing.
 */
#define wake_up_var_locked(var, lock)                                        \
        wake_up_var_protected(var, lockdep_is_held(lock))

/**
 * clear_and_wake_up_bit - clear a bit and wake up anyone waiting on that bit
 * @bit: the bit of the word being waited on
 * @word: the address containing the bit being waited on
 *
 * The designated bit is cleared and any tasks waiting in wait_on_bit()
 * or similar will be woken.  This call has RELEASE semantics so that
 * any changes to memory made before this call are guaranteed to be visible
 * after the corresponding wait_on_bit() completes.
 */
static inline void clear_and_wake_up_bit(int bit, unsigned long *word)
{
        clear_bit_unlock(bit, word);
        /* See wake_up_bit() for which memory barrier you need to use. */
        smp_mb__after_atomic();
        wake_up_bit(word, bit);
}

/**
 * test_and_clear_wake_up_bit - clear a bit if it was set: wake up anyone waiting on that bit
 * @bit: the bit of the word being waited on
 * @word: the address of memory containing that bit
 *
 * If the bit is set and can be atomically cleared, any tasks waiting in
 * wait_on_bit() or similar will be woken.  This call has the same
 * complete ordering semantics as test_and_clear_bit().  Any changes to
 * memory made before this call are guaranteed to be visible after the
 * corresponding wait_on_bit() completes.
 *
 * Returns %true if the bit was successfully set and the wake up was sent.
 */
static inline bool test_and_clear_wake_up_bit(int bit, unsigned long *word)
{
        if (!test_and_clear_bit(bit, word))
                return false;
        /* no extra barrier required */
        wake_up_bit(word, bit);
        return true;
}

/**
 * atomic_dec_and_wake_up - decrement an atomic_t and if zero, wake up waiters
 * @var: the variable to dec and test
 *
 * Decrements the atomic variable and if it reaches zero, send a wake_up to any
 * processes waiting on the variable.
 *
 * This function has the same complete ordering semantics as atomic_dec_and_test.
 *
 * Returns %true is the variable reaches zero and the wake up was sent.
 */

static inline bool atomic_dec_and_wake_up(atomic_t *var)
{
        if (!atomic_dec_and_test(var))
                return false;
        /* No extra barrier required */
        wake_up_var(var);
        return true;
}

/**
 * store_release_wake_up - update a variable and send a wake_up
 * @var: the address of the variable to be updated and woken
 * @val: the value to store in the variable.
 *
 * Store the given value in the variable send a wake up to any tasks
 * waiting on the variable.  All necessary barriers are included to ensure
 * the task calling wait_var_event() sees the new value and all values
 * written to memory before this call.
 */
#define store_release_wake_up(var, val)                                        \
do {                                                                        \
        smp_store_release(var, val);                                        \
        smp_mb();                                                        \
        wake_up_var(var);                                                \
} while (0)

#endif /* _LINUX_WAIT_BIT_H */











   20 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM bpf_trace

#if !defined(_TRACE_BPF_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)

#define _TRACE_BPF_TRACE_H

#include <linux/tracepoint.h>

TRACE_EVENT(bpf_trace_printk,

        TP_PROTO(const char *bpf_string),

        TP_ARGS(bpf_string),

        TP_STRUCT__entry(
                __string(bpf_string, bpf_string)
        ),

        TP_fast_assign(
                __assign_str(bpf_string);
        ),

        TP_printk("%s", __get_str(bpf_string))
);

#endif /* _TRACE_BPF_TRACE_H */

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE bpf_trace

#include <trace/define_trace.h>

































 2746 


















 1894 





















































































 1174 






















































 1216 




























 1359 






























































































 1195 





































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM timer

#if !defined(_TRACE_TIMER_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_TIMER_H

#include <linux/tracepoint.h>
#include <linux/hrtimer.h>
#include <linux/timer.h>

DECLARE_EVENT_CLASS(timer_class,

        TP_PROTO(struct timer_list *timer),

        TP_ARGS(timer),

        TP_STRUCT__entry(
                __field( void *,        timer        )
        ),

        TP_fast_assign(
                __entry->timer        = timer;
        ),

        TP_printk("timer=%p", __entry->timer)
);

/**
 * timer_init - called when the timer is initialized
 * @timer:        pointer to struct timer_list
 */
DEFINE_EVENT(timer_class, timer_init,

        TP_PROTO(struct timer_list *timer),

        TP_ARGS(timer)
);

#define decode_timer_flags(flags)                        \
        __print_flags(flags, "|",                        \
                {  TIMER_MIGRATING,        "M" },                \
                {  TIMER_DEFERRABLE,        "D" },                \
                {  TIMER_PINNED,        "P" },                \
                {  TIMER_IRQSAFE,        "I" })

/**
 * timer_start - called when the timer is started
 * @timer:                pointer to struct timer_list
 * @bucket_expiry:        the bucket expiry time
 */
TRACE_EVENT(timer_start,

        TP_PROTO(struct timer_list *timer,
                unsigned long bucket_expiry),

        TP_ARGS(timer, bucket_expiry),

        TP_STRUCT__entry(
                __field( void *,        timer                )
                __field( void *,        function        )
                __field( unsigned long,        expires                )
                __field( unsigned long,        bucket_expiry        )
                __field( unsigned long,        now                )
                __field( unsigned int,        flags                )
        ),

        TP_fast_assign(
                __entry->timer                = timer;
                __entry->function        = timer->function;
                __entry->expires        = timer->expires;
                __entry->bucket_expiry        = bucket_expiry;
                __entry->now                = jiffies;
                __entry->flags                = timer->flags;
        ),

        TP_printk("timer=%p function=%ps expires=%lu [timeout=%ld] bucket_expiry=%lu cpu=%u idx=%u flags=%s",
                  __entry->timer, __entry->function, __entry->expires,
                  (long)__entry->expires - __entry->now,
                  __entry->bucket_expiry, __entry->flags & TIMER_CPUMASK,
                  __entry->flags >> TIMER_ARRAYSHIFT,
                  decode_timer_flags(__entry->flags & TIMER_TRACE_FLAGMASK))
);

/**
 * timer_expire_entry - called immediately before the timer callback
 * @timer:        pointer to struct timer_list
 * @baseclk:        value of timer_base::clk when timer expires
 *
 * Allows to determine the timer latency.
 */
TRACE_EVENT(timer_expire_entry,

        TP_PROTO(struct timer_list *timer, unsigned long baseclk),

        TP_ARGS(timer, baseclk),

        TP_STRUCT__entry(
                __field( void *,        timer        )
                __field( unsigned long,        now        )
                __field( void *,        function)
                __field( unsigned long,        baseclk        )
        ),

        TP_fast_assign(
                __entry->timer                = timer;
                __entry->now                = jiffies;
                __entry->function        = timer->function;
                __entry->baseclk        = baseclk;
        ),

        TP_printk("timer=%p function=%ps now=%lu baseclk=%lu",
                  __entry->timer, __entry->function, __entry->now,
                  __entry->baseclk)
);

/**
 * timer_expire_exit - called immediately after the timer callback returns
 * @timer:        pointer to struct timer_list
 *
 * When used in combination with the timer_expire_entry tracepoint we can
 * determine the runtime of the timer callback function.
 *
 * NOTE: Do NOT dereference timer in TP_fast_assign. The pointer might
 * be invalid. We solely track the pointer.
 */
DEFINE_EVENT(timer_class, timer_expire_exit,

        TP_PROTO(struct timer_list *timer),

        TP_ARGS(timer)
);

/**
 * timer_cancel - called when the timer is canceled
 * @timer:        pointer to struct timer_list
 */
DEFINE_EVENT(timer_class, timer_cancel,

        TP_PROTO(struct timer_list *timer),

        TP_ARGS(timer)
);

TRACE_EVENT(timer_base_idle,

        TP_PROTO(bool is_idle, unsigned int cpu),

        TP_ARGS(is_idle, cpu),

        TP_STRUCT__entry(
                __field( bool,                is_idle        )
                __field( unsigned int,        cpu        )
        ),

        TP_fast_assign(
                __entry->is_idle        = is_idle;
                __entry->cpu                = cpu;
        ),

        TP_printk("is_idle=%d cpu=%d",
                  __entry->is_idle, __entry->cpu)
);

#define decode_clockid(type)                                                \
        __print_symbolic(type,                                                \
                { CLOCK_REALTIME,        "CLOCK_REALTIME"        },        \
                { CLOCK_MONOTONIC,        "CLOCK_MONOTONIC"        },        \
                { CLOCK_BOOTTIME,        "CLOCK_BOOTTIME"        },        \
                { CLOCK_TAI,                "CLOCK_TAI"                })

#define decode_hrtimer_mode(mode)                                        \
        __print_symbolic(mode,                                                \
                { HRTIMER_MODE_ABS,                "ABS"                },        \
                { HRTIMER_MODE_REL,                "REL"                },        \
                { HRTIMER_MODE_ABS_PINNED,        "ABS|PINNED"        },        \
                { HRTIMER_MODE_REL_PINNED,        "REL|PINNED"        },        \
                { HRTIMER_MODE_ABS_SOFT,        "ABS|SOFT"        },        \
                { HRTIMER_MODE_REL_SOFT,        "REL|SOFT"        },        \
                { HRTIMER_MODE_ABS_PINNED_SOFT,        "ABS|PINNED|SOFT" },        \
                { HRTIMER_MODE_REL_PINNED_SOFT,        "REL|PINNED|SOFT" },        \
                { HRTIMER_MODE_ABS_HARD,        "ABS|HARD" },                \
                { HRTIMER_MODE_REL_HARD,        "REL|HARD" },                \
                { HRTIMER_MODE_ABS_PINNED_HARD, "ABS|PINNED|HARD" },        \
                { HRTIMER_MODE_REL_PINNED_HARD,        "REL|PINNED|HARD" })

/**
 * hrtimer_setup - called when the hrtimer is initialized
 * @hrtimer:        pointer to struct hrtimer
 * @clockid:        the hrtimers clock
 * @mode:        the hrtimers mode
 */
TRACE_EVENT(hrtimer_setup,

        TP_PROTO(struct hrtimer *hrtimer, clockid_t clockid,
                 enum hrtimer_mode mode),

        TP_ARGS(hrtimer, clockid, mode),

        TP_STRUCT__entry(
                __field( void *,                hrtimer                )
                __field( clockid_t,                clockid                )
                __field( enum hrtimer_mode,        mode                )
        ),

        TP_fast_assign(
                __entry->hrtimer        = hrtimer;
                __entry->clockid        = clockid;
                __entry->mode                = mode;
        ),

        TP_printk("hrtimer=%p clockid=%s mode=%s", __entry->hrtimer,
                  decode_clockid(__entry->clockid),
                  decode_hrtimer_mode(__entry->mode))
);

/**
 * hrtimer_start - called when the hrtimer is started
 * @hrtimer:        pointer to struct hrtimer
 * @mode:        the hrtimers mode
 */
TRACE_EVENT(hrtimer_start,

        TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode),

        TP_ARGS(hrtimer, mode),

        TP_STRUCT__entry(
                __field( void *,        hrtimer                )
                __field( void *,        function        )
                __field( s64,                expires                )
                __field( s64,                softexpires        )
                __field( enum hrtimer_mode,        mode        )
        ),

        TP_fast_assign(
                __entry->hrtimer        = hrtimer;
                __entry->function        = ACCESS_PRIVATE(hrtimer, function);
                __entry->expires        = hrtimer_get_expires(hrtimer);
                __entry->softexpires        = hrtimer_get_softexpires(hrtimer);
                __entry->mode                = mode;
        ),

        TP_printk("hrtimer=%p function=%ps expires=%llu softexpires=%llu "
                  "mode=%s", __entry->hrtimer, __entry->function,
                  (unsigned long long) __entry->expires,
                  (unsigned long long) __entry->softexpires,
                  decode_hrtimer_mode(__entry->mode))
);

/**
 * hrtimer_expire_entry - called immediately before the hrtimer callback
 * @hrtimer:        pointer to struct hrtimer
 * @now:        pointer to variable which contains current time of the
 *                timers base.
 *
 * Allows to determine the timer latency.
 */
TRACE_EVENT(hrtimer_expire_entry,

        TP_PROTO(struct hrtimer *hrtimer, ktime_t *now),

        TP_ARGS(hrtimer, now),

        TP_STRUCT__entry(
                __field( void *,        hrtimer        )
                __field( s64,                now        )
                __field( void *,        function)
        ),

        TP_fast_assign(
                __entry->hrtimer        = hrtimer;
                __entry->now                = *now;
                __entry->function        = ACCESS_PRIVATE(hrtimer, function);
        ),

        TP_printk("hrtimer=%p function=%ps now=%llu",
                  __entry->hrtimer, __entry->function,
                  (unsigned long long) __entry->now)
);

DECLARE_EVENT_CLASS(hrtimer_class,

        TP_PROTO(struct hrtimer *hrtimer),

        TP_ARGS(hrtimer),

        TP_STRUCT__entry(
                __field( void *,        hrtimer        )
        ),

        TP_fast_assign(
                __entry->hrtimer        = hrtimer;
        ),

        TP_printk("hrtimer=%p", __entry->hrtimer)
);

/**
 * hrtimer_expire_exit - called immediately after the hrtimer callback returns
 * @hrtimer:        pointer to struct hrtimer
 *
 * When used in combination with the hrtimer_expire_entry tracepoint we can
 * determine the runtime of the callback function.
 */
DEFINE_EVENT(hrtimer_class, hrtimer_expire_exit,

        TP_PROTO(struct hrtimer *hrtimer),

        TP_ARGS(hrtimer)
);

/**
 * hrtimer_cancel - called when the hrtimer is canceled
 * @hrtimer:        pointer to struct hrtimer
 */
DEFINE_EVENT(hrtimer_class, hrtimer_cancel,

        TP_PROTO(struct hrtimer *hrtimer),

        TP_ARGS(hrtimer)
);

/**
 * itimer_state - called when itimer is started or canceled
 * @which:        name of the interval timer
 * @value:        the itimers value, itimer is canceled if value->it_value is
 *                zero, otherwise it is started
 * @expires:        the itimers expiry time
 */
TRACE_EVENT(itimer_state,

        TP_PROTO(int which, const struct itimerspec64 *const value,
                 unsigned long long expires),

        TP_ARGS(which, value, expires),

        TP_STRUCT__entry(
                __field(        int,                        which                )
                __field(        unsigned long long,        expires                )
                __field(        long,                        value_sec        )
                __field(        long,                        value_nsec        )
                __field(        long,                        interval_sec        )
                __field(        long,                        interval_nsec        )
        ),

        TP_fast_assign(
                __entry->which                = which;
                __entry->expires        = expires;
                __entry->value_sec        = value->it_value.tv_sec;
                __entry->value_nsec        = value->it_value.tv_nsec;
                __entry->interval_sec        = value->it_interval.tv_sec;
                __entry->interval_nsec        = value->it_interval.tv_nsec;
        ),

        TP_printk("which=%d expires=%llu it_value=%ld.%06ld it_interval=%ld.%06ld",
                  __entry->which, __entry->expires,
                  __entry->value_sec, __entry->value_nsec / NSEC_PER_USEC,
                  __entry->interval_sec, __entry->interval_nsec / NSEC_PER_USEC)
);

/**
 * itimer_expire - called when itimer expires
 * @which:        type of the interval timer
 * @pid:        pid of the process which owns the timer
 * @now:        current time, used to calculate the latency of itimer
 */
TRACE_EVENT(itimer_expire,

        TP_PROTO(int which, struct pid *pid, unsigned long long now),

        TP_ARGS(which, pid, now),

        TP_STRUCT__entry(
                __field( int ,                        which        )
                __field( pid_t,                        pid        )
                __field( unsigned long long,        now        )
        ),

        TP_fast_assign(
                __entry->which        = which;
                __entry->now        = now;
                __entry->pid        = pid_nr(pid);
        ),

        TP_printk("which=%d pid=%d now=%llu", __entry->which,
                  (int) __entry->pid, __entry->now)
);

#ifdef CONFIG_NO_HZ_COMMON

#define TICK_DEP_NAMES                                        \
                tick_dep_mask_name(NONE)                \
                tick_dep_name(POSIX_TIMER)                \
                tick_dep_name(PERF_EVENTS)                \
                tick_dep_name(SCHED)                        \
                tick_dep_name(CLOCK_UNSTABLE)                \
                tick_dep_name(RCU)                        \
                tick_dep_name_end(RCU_EXP)

#undef tick_dep_name
#undef tick_dep_mask_name
#undef tick_dep_name_end

/* The MASK will convert to their bits and they need to be processed too */
#define tick_dep_name(sdep) TRACE_DEFINE_ENUM(TICK_DEP_BIT_##sdep); \
        TRACE_DEFINE_ENUM(TICK_DEP_MASK_##sdep);
#define tick_dep_name_end(sdep)  TRACE_DEFINE_ENUM(TICK_DEP_BIT_##sdep); \
        TRACE_DEFINE_ENUM(TICK_DEP_MASK_##sdep);
/* NONE only has a mask defined for it */
#define tick_dep_mask_name(sdep) TRACE_DEFINE_ENUM(TICK_DEP_MASK_##sdep);

TICK_DEP_NAMES

#undef tick_dep_name
#undef tick_dep_mask_name
#undef tick_dep_name_end

#define tick_dep_name(sdep) { TICK_DEP_MASK_##sdep, #sdep },
#define tick_dep_mask_name(sdep) { TICK_DEP_MASK_##sdep, #sdep },
#define tick_dep_name_end(sdep) { TICK_DEP_MASK_##sdep, #sdep }

#define show_tick_dep_name(val)                                \
        __print_symbolic(val, TICK_DEP_NAMES)

TRACE_EVENT(tick_stop,

        TP_PROTO(int success, int dependency),

        TP_ARGS(success, dependency),

        TP_STRUCT__entry(
                __field( int ,                success        )
                __field( int ,                dependency )
        ),

        TP_fast_assign(
                __entry->success        = success;
                __entry->dependency        = dependency;
        ),

        TP_printk("success=%d dependency=%s",  __entry->success, \
                        show_tick_dep_name(__entry->dependency))
);
#endif

#endif /*  _TRACE_TIMER_H */

/* This part must be outside protection */
#include <trace/define_trace.h>





























































































    5 




    1 






    1 




    4 























    4 
    1 



















    2 




    2 

    1 
    1 

































































    9 

















    1 



    1 






    1 

    1 



    1 



    1 










    1 


















    1 
    1 







    2 











































































































































































    2 
    1 
    1 






























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
/*
 * Routines to compress and uncompress tcp packets (for transmission
 * over low speed serial lines).
 *
 * Copyright (c) 1989 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms are permitted
 * provided that the above copyright notice and this paragraph are
 * duplicated in all such forms and that any documentation,
 * advertising materials, and other materials related to such
 * distribution and use acknowledge that the software was developed
 * by the University of California, Berkeley.  The name of the
 * University may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *        Van Jacobson (van@helios.ee.lbl.gov), Dec 31, 1989:
 *        - Initial distribution.
 *
 *
 * modified for KA9Q Internet Software Package by
 * Katie Stevens (dkstevens@ucdavis.edu)
 * University of California, Davis
 * Computing Services
 *        - 01-31-90        initial adaptation (from 1.19)
 *        PPP.05        02-15-90 [ks]
 *        PPP.08        05-02-90 [ks]        use PPP protocol field to signal compression
 *        PPP.15        09-90         [ks]        improve mbuf handling
 *        PPP.16        11-02         [karn]        substantially rewritten to use NOS facilities
 *
 *        - Feb 1991        Bill_Simpson@um.cc.umich.edu
 *                        variable number of conversation slots
 *                        allow zero or one slots
 *                        separate routines
 *                        status display
 *        - Jul 1994        Dmitry Gorodchanin
 *                        Fixes for memory leaks.
 *      - Oct 1994      Dmitry Gorodchanin
 *                      Modularization.
 *        - Jan 1995        Bjorn Ekwall
 *                        Use ip_fast_csum from ip.h
 *        - July 1995        Christos A. Polyzols
 *                        Spotted bug in tcp option checking
 *
 *
 *        This module is a difficult issue. It's clearly inet code but it's also clearly
 *        driver code belonging close to PPP and SLIP
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <net/slhc_vj.h>

#ifdef CONFIG_INET
/* Entire module is for IP only */
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/termios.h>
#include <linux/in.h>
#include <linux/fcntl.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/icmp.h>
#include <net/tcp.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
#include <net/checksum.h>
#include <linux/unaligned.h>

static unsigned char *encode(unsigned char *cp, unsigned short n);
static long decode(unsigned char **cpp);
static unsigned char * put16(unsigned char *cp, unsigned short x);
static unsigned short pull16(unsigned char **cpp);

/* Allocate compression data structure
 *        slots must be in range 0 to 255 (zero meaning no compression)
 * Returns pointer to structure or ERR_PTR() on error.
 */
struct slcompress *
slhc_init(int rslots, int tslots)
{
        short i;
        struct cstate *ts;
        struct slcompress *comp;

        if (rslots < 0 || rslots > 255 || tslots < 0 || tslots > 255)
                return ERR_PTR(-EINVAL);

        comp = kzalloc(sizeof(struct slcompress), GFP_KERNEL);
        if (! comp)
                goto out_fail;

        if (rslots > 0) {
                size_t rsize = rslots * sizeof(struct cstate);
                comp->rstate = kzalloc(rsize, GFP_KERNEL);
                if (! comp->rstate)
                        goto out_free;
                comp->rslot_limit = rslots - 1;
        }

        if (tslots > 0) {
                size_t tsize = tslots * sizeof(struct cstate);
                comp->tstate = kzalloc(t