Total coverage: 254411 (14%)of 1829120 





















   14 

   14 





   10 




    1 





















































   86 















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  UDPLITE     An implementation of the UDP-Lite protocol (RFC 3828).
 *
 *  Authors:    Gerrit Renker       <gerrit@erg.abdn.ac.uk>
 *
 *  Changes:
 *  Fixes:
 */

#define pr_fmt(fmt) "UDPLite: " fmt

#include <linux/export.h>
#include <linux/proc_fs.h>
#include "udp_impl.h"

struct udp_table         udplite_table __read_mostly;
EXPORT_SYMBOL(udplite_table);

/* Designate sk as UDP-Lite socket */
static int udplite_sk_init(struct sock *sk)
{
        udp_init_sock(sk);
        pr_warn_once("UDP-Lite is deprecated and scheduled to be removed in 2025, "
                     "please contact the netdev mailing list\n");
        return 0;
}

static int udplite_rcv(struct sk_buff *skb)
{
        return __udp4_lib_rcv(skb, &udplite_table, IPPROTO_UDPLITE);
}

static int udplite_err(struct sk_buff *skb, u32 info)
{
        return __udp4_lib_err(skb, info, &udplite_table);
}

static const struct net_protocol udplite_protocol = {
        .handler        = udplite_rcv,
        .err_handler        = udplite_err,
        .no_policy        = 1,
};

struct proto         udplite_prot = {
        .name                   = "UDP-Lite",
        .owner                   = THIS_MODULE,
        .close                   = udp_lib_close,
        .connect           = ip4_datagram_connect,
        .disconnect           = udp_disconnect,
        .ioctl                   = udp_ioctl,
        .init                   = udplite_sk_init,
        .destroy           = udp_destroy_sock,
        .setsockopt           = udp_setsockopt,
        .getsockopt           = udp_getsockopt,
        .sendmsg           = udp_sendmsg,
        .recvmsg           = udp_recvmsg,
        .hash                   = udp_lib_hash,
        .unhash                   = udp_lib_unhash,
        .rehash                   = udp_v4_rehash,
        .get_port           = udp_v4_get_port,

        .memory_allocated  = &udp_memory_allocated,
        .per_cpu_fw_alloc  = &udp_memory_per_cpu_fw_alloc,

        .sysctl_mem           = sysctl_udp_mem,
        .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
        .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
        .obj_size           = sizeof(struct udp_sock),
        .h.udp_table           = &udplite_table,
};
EXPORT_SYMBOL(udplite_prot);

static struct inet_protosw udplite4_protosw = {
        .type                =  SOCK_DGRAM,
        .protocol        =  IPPROTO_UDPLITE,
        .prot                =  &udplite_prot,
        .ops                =  &inet_dgram_ops,
        .flags                =  INET_PROTOSW_PERMANENT,
};

#ifdef CONFIG_PROC_FS
static struct udp_seq_afinfo udplite4_seq_afinfo = {
        .family                = AF_INET,
        .udp_table         = &udplite_table,
};

static int __net_init udplite4_proc_init_net(struct net *net)
{
        if (!proc_create_net_data("udplite", 0444, net->proc_net, &udp_seq_ops,
                        sizeof(struct udp_iter_state), &udplite4_seq_afinfo))
                return -ENOMEM;
        return 0;
}

static void __net_exit udplite4_proc_exit_net(struct net *net)
{
        remove_proc_entry("udplite", net->proc_net);
}

static struct pernet_operations udplite4_net_ops = {
        .init = udplite4_proc_init_net,
        .exit = udplite4_proc_exit_net,
};

static __init int udplite4_proc_init(void)
{
        return register_pernet_subsys(&udplite4_net_ops);
}
#else
static inline int udplite4_proc_init(void)
{
        return 0;
}
#endif

void __init udplite4_register(void)
{
        udp_table_init(&udplite_table, "UDP-Lite");
        if (proto_register(&udplite_prot, 1))
                goto out_register_err;

        if (inet_add_protocol(&udplite_protocol, IPPROTO_UDPLITE) < 0)
                goto out_unregister_proto;

        inet_register_protosw(&udplite4_protosw);

        if (udplite4_proc_init())
                pr_err("%s: Cannot register /proc!\n", __func__);
        return;

out_unregister_proto:
        proto_unregister(&udplite_prot);
out_register_err:
        pr_crit("%s: Cannot add UDP-Lite protocol\n", __func__);
}
















































































































    4 







  293 

    3 



    3 
  726 















  387 


















   22 
   68 
  128 
   62 
   66 


  128 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/mount.h>
#include <linux/seq_file.h>
#include <linux/poll.h>
#include <linux/ns_common.h>
#include <linux/fs_pin.h>

struct mnt_namespace {
        struct ns_common        ns;
        struct mount *        root;
        struct {
                struct rb_root        mounts;                 /* Protected by namespace_sem */
                struct rb_node        *mnt_last_node;         /* last (rightmost) mount in the rbtree */
                struct rb_node        *mnt_first_node; /* first (leftmost) mount in the rbtree */
        };
        struct user_namespace        *user_ns;
        struct ucounts                *ucounts;
        u64                        seq;        /* Sequence number to prevent loops */
        union {
                wait_queue_head_t        poll;
                struct rcu_head                mnt_ns_rcu;
        };
        u64 event;
        unsigned int                nr_mounts; /* # of mounts in the namespace */
        unsigned int                pending_mounts;
        struct rb_node                mnt_ns_tree_node; /* node in the mnt_ns_tree */
        struct list_head        mnt_ns_list; /* entry in the sequential list of mounts namespace */
        refcount_t                passive; /* number references not pinning @mounts */
} __randomize_layout;

struct mnt_pcp {
        int mnt_count;
        int mnt_writers;
};

struct mountpoint {
        struct hlist_node m_hash;
        struct dentry *m_dentry;
        struct hlist_head m_list;
        int m_count;
};

struct mount {
        struct hlist_node mnt_hash;
        struct mount *mnt_parent;
        struct dentry *mnt_mountpoint;
        struct vfsmount mnt;
        union {
                struct rb_node mnt_node; /* node in the ns->mounts rbtree */
                struct rcu_head mnt_rcu;
                struct llist_node mnt_llist;
        };
#ifdef CONFIG_SMP
        struct mnt_pcp __percpu *mnt_pcp;
#else
        int mnt_count;
        int mnt_writers;
#endif
        struct list_head mnt_mounts;        /* list of children, anchored here */
        struct list_head mnt_child;        /* and going through their mnt_child */
        struct list_head mnt_instance;        /* mount instance on sb->s_mounts */
        const char *mnt_devname;        /* Name of device e.g. /dev/dsk/hda1 */
        struct list_head mnt_list;
        struct list_head mnt_expire;        /* link in fs-specific expiry list */
        struct list_head mnt_share;        /* circular list of shared mounts */
        struct list_head mnt_slave_list;/* list of slave mounts */
        struct list_head mnt_slave;        /* slave list entry */
        struct mount *mnt_master;        /* slave is on master->mnt_slave_list */
        struct mnt_namespace *mnt_ns;        /* containing namespace */
        struct mountpoint *mnt_mp;        /* where is it mounted */
        union {
                struct hlist_node mnt_mp_list;        /* list mounts with the same mountpoint */
                struct hlist_node mnt_umount;
        };
        struct list_head mnt_umounting; /* list entry for umount propagation */
#ifdef CONFIG_FSNOTIFY
        struct fsnotify_mark_connector __rcu *mnt_fsnotify_marks;
        __u32 mnt_fsnotify_mask;
#endif
        int mnt_id;                        /* mount identifier, reused */
        u64 mnt_id_unique;                /* mount ID unique until reboot */
        int mnt_group_id;                /* peer group identifier */
        int mnt_expiry_mark;                /* true if marked for expiry */
        struct hlist_head mnt_pins;
        struct hlist_head mnt_stuck_children;
} __randomize_layout;

#define MNT_NS_INTERNAL ERR_PTR(-EINVAL) /* distinct from any mnt_namespace */

static inline struct mount *real_mount(struct vfsmount *mnt)
{
        return container_of(mnt, struct mount, mnt);
}

static inline int mnt_has_parent(struct mount *mnt)
{
        return mnt != mnt->mnt_parent;
}

static inline int is_mounted(struct vfsmount *mnt)
{
        /* neither detached nor internal? */
        return !IS_ERR_OR_NULL(real_mount(mnt)->mnt_ns);
}

extern struct mount *__lookup_mnt(struct vfsmount *, struct dentry *);

extern int __legitimize_mnt(struct vfsmount *, unsigned);

static inline bool __path_is_mountpoint(const struct path *path)
{
        struct mount *m = __lookup_mnt(path->mnt, path->dentry);
        return m && likely(!(m->mnt.mnt_flags & MNT_SYNC_UMOUNT));
}

extern void __detach_mounts(struct dentry *dentry);

static inline void detach_mounts(struct dentry *dentry)
{
        if (!d_mountpoint(dentry))
                return;
        __detach_mounts(dentry);
}

static inline void get_mnt_ns(struct mnt_namespace *ns)
{
        refcount_inc(&ns->ns.count);
}

extern seqlock_t mount_lock;

struct proc_mounts {
        struct mnt_namespace *ns;
        struct path root;
        int (*show)(struct seq_file *, struct vfsmount *);
};

extern const struct seq_operations mounts_op;

extern bool __is_local_mountpoint(struct dentry *dentry);
static inline bool is_local_mountpoint(struct dentry *dentry)
{
        if (!d_mountpoint(dentry))
                return false;

        return __is_local_mountpoint(dentry);
}

static inline bool is_anon_ns(struct mnt_namespace *ns)
{
        return ns->seq == 0;
}

static inline bool mnt_ns_attached(const struct mount *mnt)
{
        return !RB_EMPTY_NODE(&mnt->mnt_node);
}

static inline void move_from_ns(struct mount *mnt, struct list_head *dt_list)
{
        struct mnt_namespace *ns = mnt->mnt_ns;
        WARN_ON(!mnt_ns_attached(mnt));
        if (ns->mnt_last_node == &mnt->mnt_node)
                ns->mnt_last_node = rb_prev(&mnt->mnt_node);
        if (ns->mnt_first_node == &mnt->mnt_node)
                ns->mnt_first_node = rb_next(&mnt->mnt_node);
        rb_erase(&mnt->mnt_node, &ns->mounts);
        RB_CLEAR_NODE(&mnt->mnt_node);
        list_add_tail(&mnt->mnt_list, dt_list);
}

bool has_locked_children(struct mount *mnt, struct dentry *dentry);
struct mnt_namespace *get_sequential_mnt_ns(struct mnt_namespace *mnt_ns,
                                            bool previous);

static inline struct mnt_namespace *to_mnt_ns(struct ns_common *ns)
{
        return container_of(ns, struct mnt_namespace, ns);
}

















































































































































































































































































































    1 
    1 






















    1 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
// SPDX-License-Identifier: GPL-2.0
/* -*- linux-c -*-
 * Cypress USB Thermometer driver 
 * 
 * Copyright (c) 2004 Erik Rigtorp <erkki@linux.nu> <erik@rigtorp.com>
 * 
 * This driver works with Elektor magazine USB Interface as published in 
 * issue #291. It should also work with the original starter kit/demo board
 * from Cypress.
 */


#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/usb.h>

#define DRIVER_AUTHOR "Erik Rigtorp"
#define DRIVER_DESC "Cypress USB Thermometer driver"

#define USB_SKEL_VENDOR_ID        0x04b4
#define USB_SKEL_PRODUCT_ID        0x0002

static const struct usb_device_id id_table[] = {
        { USB_DEVICE(USB_SKEL_VENDOR_ID, USB_SKEL_PRODUCT_ID) },
        { }
};
MODULE_DEVICE_TABLE (usb, id_table);

/* Structure to hold all of our device specific stuff */
struct usb_cytherm {
        struct usb_device    *udev;         /* save off the usb device pointer */
        struct usb_interface *interface; /* the interface for this device */
        int brightness;
};


/* Vendor requests */
/* They all operate on one byte at a time */
#define PING       0x00
#define READ_ROM   0x01 /* Reads form ROM, value = address */
#define READ_RAM   0x02 /* Reads form RAM, value = address */
#define WRITE_RAM  0x03 /* Write to RAM, value = address, index = data */
#define READ_PORT  0x04 /* Reads from port, value = address */
#define WRITE_PORT 0x05 /* Write to port, value = address, index = data */ 


/* Send a vendor command to device */
static int vendor_command(struct usb_device *dev, unsigned char request, 
                          unsigned char value, unsigned char index,
                          void *buf, int size)
{
        return usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
                               request, 
                               USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_OTHER,
                               value, 
                               index, buf, size,
                               USB_CTRL_GET_TIMEOUT);
}



#define BRIGHTNESS 0x2c     /* RAM location for brightness value */
#define BRIGHTNESS_SEM 0x2b /* RAM location for brightness semaphore */

static ssize_t brightness_show(struct device *dev, struct device_attribute *attr, char *buf)
{
        struct usb_interface *intf = to_usb_interface(dev);    
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);     

        return sprintf(buf, "%i", cytherm->brightness);
}

static ssize_t brightness_store(struct device *dev, struct device_attribute *attr, const char *buf,
                              size_t count)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        unsigned char *buffer;
        int retval;
   
        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        cytherm->brightness = simple_strtoul(buf, NULL, 10);
   
        if (cytherm->brightness > 0xFF)
                cytherm->brightness = 0xFF;
        else if (cytherm->brightness < 0)
                cytherm->brightness = 0;
   
        /* Set brightness */
        retval = vendor_command(cytherm->udev, WRITE_RAM, BRIGHTNESS, 
                                cytherm->brightness, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
        /* Inform µC that we have changed the brightness setting */
        retval = vendor_command(cytherm->udev, WRITE_RAM, BRIGHTNESS_SEM,
                                0x01, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
   
        kfree(buffer);
   
        return count;
}
static DEVICE_ATTR_RW(brightness);


#define TEMP 0x33 /* RAM location for temperature */
#define SIGN 0x34 /* RAM location for temperature sign */

static ssize_t temp_show(struct device *dev, struct device_attribute *attr, char *buf)
{

        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        int retval;
        unsigned char *buffer;

        int temp, sign;
   
        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        /* read temperature */
        retval = vendor_command(cytherm->udev, READ_RAM, TEMP, 0, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
        temp = buffer[1];
   
        /* read sign */
        retval = vendor_command(cytherm->udev, READ_RAM, SIGN, 0, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
        sign = buffer[1];

        kfree(buffer);
   
        return sprintf(buf, "%c%i.%i", sign ? '-' : '+', temp >> 1,
                       5*(temp - ((temp >> 1) << 1)));
}
static DEVICE_ATTR_RO(temp);


#define BUTTON 0x7a

static ssize_t button_show(struct device *dev, struct device_attribute *attr, char *buf)
{

        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        int retval;
        unsigned char *buffer;

        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        /* check button */
        retval = vendor_command(cytherm->udev, READ_RAM, BUTTON, 0, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
   
        retval = buffer[1];

        kfree(buffer);

        if (retval)
                return sprintf(buf, "1");
        else
                return sprintf(buf, "0");
}
static DEVICE_ATTR_RO(button);


static ssize_t port0_show(struct device *dev, struct device_attribute *attr, char *buf)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        int retval;
        unsigned char *buffer;

        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        retval = vendor_command(cytherm->udev, READ_PORT, 0, 0, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);

        retval = buffer[1];

        kfree(buffer);

        return sprintf(buf, "%d", retval);
}


static ssize_t port0_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        unsigned char *buffer;
        int retval;
        int tmp;
   
        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        tmp = simple_strtoul(buf, NULL, 10);
   
        if (tmp > 0xFF)
                tmp = 0xFF;
        else if (tmp < 0)
                tmp = 0;
   
        retval = vendor_command(cytherm->udev, WRITE_PORT, 0,
                                tmp, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);

        kfree(buffer);

        return count;
}
static DEVICE_ATTR_RW(port0);

static ssize_t port1_show(struct device *dev, struct device_attribute *attr, char *buf)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        int retval;
        unsigned char *buffer;

        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        retval = vendor_command(cytherm->udev, READ_PORT, 1, 0, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);
   
        retval = buffer[1];

        kfree(buffer);

        return sprintf(buf, "%d", retval);
}


static ssize_t port1_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_cytherm *cytherm = usb_get_intfdata(intf);

        unsigned char *buffer;
        int retval;
        int tmp;
   
        buffer = kmalloc(8, GFP_KERNEL);
        if (!buffer)
                return 0;

        tmp = simple_strtoul(buf, NULL, 10);
   
        if (tmp > 0xFF)
                tmp = 0xFF;
        else if (tmp < 0)
                tmp = 0;
   
        retval = vendor_command(cytherm->udev, WRITE_PORT, 1,
                                tmp, buffer, 8);
        if (retval)
                dev_dbg(&cytherm->udev->dev, "retval = %d\n", retval);

        kfree(buffer);

        return count;
}
static DEVICE_ATTR_RW(port1);

static struct attribute *cytherm_attrs[] = {
        &dev_attr_brightness.attr,
        &dev_attr_temp.attr,
        &dev_attr_button.attr,
        &dev_attr_port0.attr,
        &dev_attr_port1.attr,
        NULL,
};
ATTRIBUTE_GROUPS(cytherm);

static int cytherm_probe(struct usb_interface *interface, 
                         const struct usb_device_id *id)
{
        struct usb_device *udev = interface_to_usbdev(interface);
        struct usb_cytherm *dev;
        int retval = -ENOMEM;

        dev = kzalloc(sizeof(struct usb_cytherm), GFP_KERNEL);
        if (!dev)
                goto error_mem;

        dev->udev = usb_get_dev(udev);

        usb_set_intfdata(interface, dev);

        dev->brightness = 0xFF;

        dev_info(&interface->dev,
                  "Cypress thermometer device now attached\n");
        return 0;

error_mem:
        return retval;
}

static void cytherm_disconnect(struct usb_interface *interface)
{
        struct usb_cytherm *dev;

        dev = usb_get_intfdata(interface);

        /* first remove the files, then NULL the pointer */
        usb_set_intfdata(interface, NULL);

        usb_put_dev(dev->udev);

        kfree(dev);

        dev_info(&interface->dev, "Cypress thermometer now disconnected\n");
}

/* usb specific object needed to register this driver with the usb subsystem */
static struct usb_driver cytherm_driver = {
        .name =                "cytherm",
        .probe =        cytherm_probe,
        .disconnect =        cytherm_disconnect,
        .id_table =        id_table,
        .dev_groups =        cytherm_groups,
};

module_usb_driver(cytherm_driver);

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");



































































    1 















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
// SPDX-License-Identifier: GPL-2.0+

#include <linux/iosys-map.h>

#include <drm/drm_atomic.h>
#include <drm/drm_edid.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_writeback.h>
#include <drm/drm_probe_helper.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_gem_shmem_helper.h>

#include "vkms_drv.h"
#include "vkms_formats.h"

static const u32 vkms_wb_formats[] = {
        DRM_FORMAT_ARGB8888,
        DRM_FORMAT_XRGB8888,
        DRM_FORMAT_XRGB16161616,
        DRM_FORMAT_ARGB16161616,
        DRM_FORMAT_RGB565
};

static const struct drm_connector_funcs vkms_wb_connector_funcs = {
        .fill_modes = drm_helper_probe_single_connector_modes,
        .destroy = drm_connector_cleanup,
        .reset = drm_atomic_helper_connector_reset,
        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
};

static int vkms_wb_atomic_check(struct drm_connector *connector,
                                struct drm_atomic_state *state)
{
        struct drm_connector_state *conn_state =
                drm_atomic_get_new_connector_state(state, connector);
        struct drm_crtc_state *crtc_state;
        struct drm_framebuffer *fb;
        const struct drm_display_mode *mode;
        int ret;

        if (!conn_state->writeback_job || !conn_state->writeback_job->fb)
                return 0;

        if (!conn_state->crtc)
                return 0;

        crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc);
        mode = &crtc_state->mode;

        fb = conn_state->writeback_job->fb;
        if (fb->width != mode->hdisplay || fb->height != mode->vdisplay) {
                DRM_DEBUG_KMS("Invalid framebuffer size %ux%u\n",
                              fb->width, fb->height);
                return -EINVAL;
        }

        ret = drm_atomic_helper_check_wb_connector_state(connector, state);
        if (ret < 0)
                return ret;

        return 0;
}

static int vkms_wb_connector_get_modes(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;

        return drm_add_modes_noedid(connector, dev->mode_config.max_width,
                                    dev->mode_config.max_height);
}

static int vkms_wb_prepare_job(struct drm_writeback_connector *wb_connector,
                               struct drm_writeback_job *job)
{
        struct vkms_writeback_job *vkmsjob;
        int ret;

        if (!job->fb)
                return 0;

        vkmsjob = kzalloc(sizeof(*vkmsjob), GFP_KERNEL);
        if (!vkmsjob)
                return -ENOMEM;

        ret = drm_gem_fb_vmap(job->fb, vkmsjob->wb_frame_info.map, vkmsjob->data);
        if (ret) {
                DRM_ERROR("vmap failed: %d\n", ret);
                goto err_kfree;
        }

        vkmsjob->wb_frame_info.fb = job->fb;
        drm_framebuffer_get(vkmsjob->wb_frame_info.fb);

        job->priv = vkmsjob;

        return 0;

err_kfree:
        kfree(vkmsjob);
        return ret;
}

static void vkms_wb_cleanup_job(struct drm_writeback_connector *connector,
                                struct drm_writeback_job *job)
{
        struct vkms_writeback_job *vkmsjob = job->priv;
        struct vkms_device *vkmsdev;

        if (!job->fb)
                return;

        drm_gem_fb_vunmap(job->fb, vkmsjob->wb_frame_info.map);

        drm_framebuffer_put(vkmsjob->wb_frame_info.fb);

        vkmsdev = drm_device_to_vkms_device(job->fb->dev);
        vkms_set_composer(&vkmsdev->output, false);
        kfree(vkmsjob);
}

static void vkms_wb_atomic_commit(struct drm_connector *conn,
                                  struct drm_atomic_state *state)
{
        struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
                                                                                         conn);
        struct vkms_device *vkmsdev = drm_device_to_vkms_device(conn->dev);
        struct vkms_output *output = &vkmsdev->output;
        struct drm_writeback_connector *wb_conn = &output->wb_connector;
        struct drm_connector_state *conn_state = wb_conn->base.state;
        struct vkms_crtc_state *crtc_state = output->composer_state;
        struct drm_framebuffer *fb = connector_state->writeback_job->fb;
        u16 crtc_height = crtc_state->base.mode.vdisplay;
        u16 crtc_width = crtc_state->base.mode.hdisplay;
        struct vkms_writeback_job *active_wb;
        struct vkms_frame_info *wb_frame_info;
        u32 wb_format = fb->format->format;

        if (!conn_state)
                return;

        vkms_set_composer(&vkmsdev->output, true);

        active_wb = conn_state->writeback_job->priv;
        wb_frame_info = &active_wb->wb_frame_info;

        spin_lock_irq(&output->composer_lock);
        crtc_state->active_writeback = active_wb;
        crtc_state->wb_pending = true;
        spin_unlock_irq(&output->composer_lock);
        drm_writeback_queue_job(wb_conn, connector_state);
        active_wb->pixel_write = get_pixel_write_function(wb_format);
        drm_rect_init(&wb_frame_info->src, 0, 0, crtc_width, crtc_height);
        drm_rect_init(&wb_frame_info->dst, 0, 0, crtc_width, crtc_height);
}

static const struct drm_connector_helper_funcs vkms_wb_conn_helper_funcs = {
        .get_modes = vkms_wb_connector_get_modes,
        .prepare_writeback_job = vkms_wb_prepare_job,
        .cleanup_writeback_job = vkms_wb_cleanup_job,
        .atomic_commit = vkms_wb_atomic_commit,
        .atomic_check = vkms_wb_atomic_check,
};

int vkms_enable_writeback_connector(struct vkms_device *vkmsdev)
{
        struct drm_writeback_connector *wb = &vkmsdev->output.wb_connector;

        drm_connector_helper_add(&wb->base, &vkms_wb_conn_helper_funcs);

        return drm_writeback_connector_init(&vkmsdev->drm, wb,
                                            &vkms_wb_connector_funcs,
                                            NULL,
                                            vkms_wb_formats,
                                            ARRAY_SIZE(vkms_wb_formats),
                                            1);
}


























    4 














    4 
















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * IPVS:        Least-Connection Scheduling module
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *
 * Changes:
 *     Wensong Zhang            :     added the ip_vs_lc_update_svc
 *     Wensong Zhang            :     added any dest with weight=0 is quiesced
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>

#include <net/ip_vs.h>

/*
 *        Least Connection scheduling
 */
static struct ip_vs_dest *
ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb,
                  struct ip_vs_iphdr *iph)
{
        struct ip_vs_dest *dest, *least = NULL;
        unsigned int loh = 0, doh;

        IP_VS_DBG(6, "%s(): Scheduling...\n", __func__);

        /*
         * Simply select the server with the least number of
         *        (activeconns<<5) + inactconns
         * Except whose weight is equal to zero.
         * If the weight is equal to zero, it means that the server is
         * quiesced, the existing connections to the server still get
         * served, but no new connection is assigned to the server.
         */

        list_for_each_entry_rcu(dest, &svc->destinations, n_list) {
                if ((dest->flags & IP_VS_DEST_F_OVERLOAD) ||
                    atomic_read(&dest->weight) == 0)
                        continue;
                doh = ip_vs_dest_conn_overhead(dest);
                if (!least || doh < loh) {
                        least = dest;
                        loh = doh;
                }
        }

        if (!least)
                ip_vs_scheduler_err(svc, "no destination available");
        else
                IP_VS_DBG_BUF(6, "LC: server %s:%u activeconns %d "
                              "inactconns %d\n",
                              IP_VS_DBG_ADDR(least->af, &least->addr),
                              ntohs(least->port),
                              atomic_read(&least->activeconns),
                              atomic_read(&least->inactconns));

        return least;
}


static struct ip_vs_scheduler ip_vs_lc_scheduler = {
        .name =                        "lc",
        .refcnt =                ATOMIC_INIT(0),
        .module =                THIS_MODULE,
        .n_list =                LIST_HEAD_INIT(ip_vs_lc_scheduler.n_list),
        .schedule =                ip_vs_lc_schedule,
};


static int __init ip_vs_lc_init(void)
{
        return register_ip_vs_scheduler(&ip_vs_lc_scheduler) ;
}

static void __exit ip_vs_lc_cleanup(void)
{
        unregister_ip_vs_scheduler(&ip_vs_lc_scheduler);
        synchronize_rcu();
}

module_init(ip_vs_lc_init);
module_exit(ip_vs_lc_cleanup);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("ipvs least connection scheduler");


























































   11 











   11 

















































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
// SPDX-License-Identifier: GPL-2.0-only
/*
 * scsi_logging.c
 *
 * Copyright (C) 2014 SUSE Linux Products GmbH
 * Copyright (C) 2014 Hannes Reinecke <hare@suse.de>
 */

#include <linux/kernel.h>
#include <linux/atomic.h>

#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_dbg.h>

static char *scsi_log_reserve_buffer(size_t *len)
{
        *len = 128;
        return kmalloc(*len, GFP_ATOMIC);
}

static void scsi_log_release_buffer(char *bufptr)
{
        kfree(bufptr);
}

static inline const char *scmd_name(const struct scsi_cmnd *scmd)
{
        struct request *rq = scsi_cmd_to_rq((struct scsi_cmnd *)scmd);

        if (!rq->q || !rq->q->disk)
                return NULL;
        return rq->q->disk->disk_name;
}

static size_t sdev_format_header(char *logbuf, size_t logbuf_len,
                                 const char *name, int tag)
{
        size_t off = 0;

        if (name)
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "[%s] ", name);

        if (WARN_ON(off >= logbuf_len))
                return off;

        if (tag >= 0)
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "tag#%d ", tag);
        return off;
}

void sdev_prefix_printk(const char *level, const struct scsi_device *sdev,
                        const char *name, const char *fmt, ...)
{
        va_list args;
        char *logbuf;
        size_t off = 0, logbuf_len;

        if (!sdev)
                return;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;

        if (name)
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "[%s] ", name);
        if (!WARN_ON(off >= logbuf_len)) {
                va_start(args, fmt);
                off += vscnprintf(logbuf + off, logbuf_len - off, fmt, args);
                va_end(args);
        }
        dev_printk(level, &sdev->sdev_gendev, "%s", logbuf);
        scsi_log_release_buffer(logbuf);
}
EXPORT_SYMBOL(sdev_prefix_printk);

void scmd_printk(const char *level, const struct scsi_cmnd *scmd,
                const char *fmt, ...)
{
        va_list args;
        char *logbuf;
        size_t off = 0, logbuf_len;

        if (!scmd)
                return;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;
        off = sdev_format_header(logbuf, logbuf_len, scmd_name(scmd),
                                 scsi_cmd_to_rq((struct scsi_cmnd *)scmd)->tag);
        if (off < logbuf_len) {
                va_start(args, fmt);
                off += vscnprintf(logbuf + off, logbuf_len - off, fmt, args);
                va_end(args);
        }
        dev_printk(level, &scmd->device->sdev_gendev, "%s", logbuf);
        scsi_log_release_buffer(logbuf);
}
EXPORT_SYMBOL(scmd_printk);

static size_t scsi_format_opcode_name(char *buffer, size_t buf_len,
                                      const unsigned char *cdbp)
{
        int sa, cdb0;
        const char *cdb_name = NULL, *sa_name = NULL;
        size_t off;

        cdb0 = cdbp[0];
        if (cdb0 == VARIABLE_LENGTH_CMD) {
                int len = scsi_varlen_cdb_length(cdbp);

                if (len < 10) {
                        off = scnprintf(buffer, buf_len,
                                        "short variable length command, len=%d",
                                        len);
                        return off;
                }
                sa = (cdbp[8] << 8) + cdbp[9];
        } else
                sa = cdbp[1] & 0x1f;

        if (!scsi_opcode_sa_name(cdb0, sa, &cdb_name, &sa_name)) {
                if (cdb_name)
                        off = scnprintf(buffer, buf_len, "%s", cdb_name);
                else {
                        off = scnprintf(buffer, buf_len, "opcode=0x%x", cdb0);
                        if (WARN_ON(off >= buf_len))
                                return off;
                        if (cdb0 >= VENDOR_SPECIFIC_CDB)
                                off += scnprintf(buffer + off, buf_len - off,
                                                 " (vendor)");
                        else if (cdb0 >= 0x60 && cdb0 < 0x7e)
                                off += scnprintf(buffer + off, buf_len - off,
                                                 " (reserved)");
                }
        } else {
                if (sa_name)
                        off = scnprintf(buffer, buf_len, "%s", sa_name);
                else if (cdb_name)
                        off = scnprintf(buffer, buf_len, "%s, sa=0x%x",
                                        cdb_name, sa);
                else
                        off = scnprintf(buffer, buf_len,
                                        "opcode=0x%x, sa=0x%x", cdb0, sa);
        }
        WARN_ON(off >= buf_len);
        return off;
}

size_t __scsi_format_command(char *logbuf, size_t logbuf_len,
                             const unsigned char *cdb, size_t cdb_len)
{
        int len, k;
        size_t off;

        off = scsi_format_opcode_name(logbuf, logbuf_len, cdb);
        if (off >= logbuf_len)
                return off;
        len = scsi_command_size(cdb);
        if (cdb_len < len)
                len = cdb_len;
        /* print out all bytes in cdb */
        for (k = 0; k < len; ++k) {
                if (off > logbuf_len - 3)
                        break;
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 " %02x", cdb[k]);
        }
        return off;
}
EXPORT_SYMBOL(__scsi_format_command);

void scsi_print_command(struct scsi_cmnd *cmd)
{
        int k;
        char *logbuf;
        size_t off, logbuf_len;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;

        off = sdev_format_header(logbuf, logbuf_len,
                                 scmd_name(cmd), scsi_cmd_to_rq(cmd)->tag);
        if (off >= logbuf_len)
                goto out_printk;
        off += scnprintf(logbuf + off, logbuf_len - off, "CDB: ");
        if (WARN_ON(off >= logbuf_len))
                goto out_printk;

        off += scsi_format_opcode_name(logbuf + off, logbuf_len - off,
                                       cmd->cmnd);
        if (off >= logbuf_len)
                goto out_printk;

        /* print out all bytes in cdb */
        if (cmd->cmd_len > 16) {
                /* Print opcode in one line and use separate lines for CDB */
                off += scnprintf(logbuf + off, logbuf_len - off, "\n");
                dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s", logbuf);
                for (k = 0; k < cmd->cmd_len; k += 16) {
                        size_t linelen = min(cmd->cmd_len - k, 16);

                        off = sdev_format_header(logbuf, logbuf_len,
                                                 scmd_name(cmd),
                                                 scsi_cmd_to_rq(cmd)->tag);
                        if (!WARN_ON(off > logbuf_len - 58)) {
                                off += scnprintf(logbuf + off, logbuf_len - off,
                                                 "CDB[%02x]: ", k);
                                hex_dump_to_buffer(&cmd->cmnd[k], linelen,
                                                   16, 1, logbuf + off,
                                                   logbuf_len - off, false);
                        }
                        dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s",
                                   logbuf);
                }
                goto out;
        }
        if (!WARN_ON(off > logbuf_len - 49)) {
                off += scnprintf(logbuf + off, logbuf_len - off, " ");
                hex_dump_to_buffer(cmd->cmnd, cmd->cmd_len, 16, 1,
                                   logbuf + off, logbuf_len - off,
                                   false);
        }
out_printk:
        dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s", logbuf);
out:
        scsi_log_release_buffer(logbuf);
}
EXPORT_SYMBOL(scsi_print_command);

static size_t
scsi_format_extd_sense(char *buffer, size_t buf_len,
                       unsigned char asc, unsigned char ascq)
{
        size_t off = 0;
        const char *extd_sense_fmt = NULL;
        const char *extd_sense_str = scsi_extd_sense_format(asc, ascq,
                                                            &extd_sense_fmt);

        if (extd_sense_str) {
                off = scnprintf(buffer, buf_len, "Add. Sense: %s",
                                extd_sense_str);
                if (extd_sense_fmt)
                        off += scnprintf(buffer + off, buf_len - off,
                                         "(%s%x)", extd_sense_fmt, ascq);
        } else {
                if (asc >= 0x80)
                        off = scnprintf(buffer, buf_len, "<<vendor>>");
                off += scnprintf(buffer + off, buf_len - off,
                                 "ASC=0x%x ", asc);
                if (ascq >= 0x80)
                        off += scnprintf(buffer + off, buf_len - off,
                                         "<<vendor>>");
                off += scnprintf(buffer + off, buf_len - off,
                                 "ASCQ=0x%x ", ascq);
        }
        return off;
}

static size_t
scsi_format_sense_hdr(char *buffer, size_t buf_len,
                      const struct scsi_sense_hdr *sshdr)
{
        const char *sense_txt;
        size_t off;

        off = scnprintf(buffer, buf_len, "Sense Key : ");
        sense_txt = scsi_sense_key_string(sshdr->sense_key);
        if (sense_txt)
                off += scnprintf(buffer + off, buf_len - off,
                                 "%s ", sense_txt);
        else
                off += scnprintf(buffer + off, buf_len - off,
                                 "0x%x ", sshdr->sense_key);
        off += scnprintf(buffer + off, buf_len - off,
                scsi_sense_is_deferred(sshdr) ? "[deferred] " : "[current] ");

        if (sshdr->response_code >= 0x72)
                off += scnprintf(buffer + off, buf_len - off, "[descriptor] ");
        return off;
}

static void
scsi_log_dump_sense(const struct scsi_device *sdev, const char *name, int tag,
                    const unsigned char *sense_buffer, int sense_len)
{
        char *logbuf;
        size_t logbuf_len;
        int i;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;

        for (i = 0; i < sense_len; i += 16) {
                int len = min(sense_len - i, 16);
                size_t off;

                off = sdev_format_header(logbuf, logbuf_len,
                                         name, tag);
                hex_dump_to_buffer(&sense_buffer[i], len, 16, 1,
                                   logbuf + off, logbuf_len - off,
                                   false);
                dev_printk(KERN_INFO, &sdev->sdev_gendev, "%s", logbuf);
        }
        scsi_log_release_buffer(logbuf);
}

static void
scsi_log_print_sense_hdr(const struct scsi_device *sdev, const char *name,
                         int tag, const struct scsi_sense_hdr *sshdr)
{
        char *logbuf;
        size_t off, logbuf_len;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;
        off = sdev_format_header(logbuf, logbuf_len, name, tag);
        off += scsi_format_sense_hdr(logbuf + off, logbuf_len - off, sshdr);
        dev_printk(KERN_INFO, &sdev->sdev_gendev, "%s", logbuf);
        scsi_log_release_buffer(logbuf);

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;
        off = sdev_format_header(logbuf, logbuf_len, name, tag);
        off += scsi_format_extd_sense(logbuf + off, logbuf_len - off,
                                      sshdr->asc, sshdr->ascq);
        dev_printk(KERN_INFO, &sdev->sdev_gendev, "%s", logbuf);
        scsi_log_release_buffer(logbuf);
}

static void
scsi_log_print_sense(const struct scsi_device *sdev, const char *name, int tag,
                     const unsigned char *sense_buffer, int sense_len)
{
        struct scsi_sense_hdr sshdr;

        if (scsi_normalize_sense(sense_buffer, sense_len, &sshdr))
                scsi_log_print_sense_hdr(sdev, name, tag, &sshdr);
        else
                scsi_log_dump_sense(sdev, name, tag, sense_buffer, sense_len);
}

/*
 * Print normalized SCSI sense header with a prefix.
 */
void
scsi_print_sense_hdr(const struct scsi_device *sdev, const char *name,
                     const struct scsi_sense_hdr *sshdr)
{
        scsi_log_print_sense_hdr(sdev, name, -1, sshdr);
}
EXPORT_SYMBOL(scsi_print_sense_hdr);

/* Normalize and print sense buffer with name prefix */
void __scsi_print_sense(const struct scsi_device *sdev, const char *name,
                        const unsigned char *sense_buffer, int sense_len)
{
        scsi_log_print_sense(sdev, name, -1, sense_buffer, sense_len);
}
EXPORT_SYMBOL(__scsi_print_sense);

/* Normalize and print sense buffer in SCSI command */
void scsi_print_sense(const struct scsi_cmnd *cmd)
{
        scsi_log_print_sense(cmd->device, scmd_name(cmd),
                             scsi_cmd_to_rq((struct scsi_cmnd *)cmd)->tag,
                             cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE);
}
EXPORT_SYMBOL(scsi_print_sense);

void scsi_print_result(const struct scsi_cmnd *cmd, const char *msg,
                       int disposition)
{
        char *logbuf;
        size_t off, logbuf_len;
        const char *mlret_string = scsi_mlreturn_string(disposition);
        const char *hb_string = scsi_hostbyte_string(cmd->result);
        unsigned long cmd_age = (jiffies - cmd->jiffies_at_alloc) / HZ;

        logbuf = scsi_log_reserve_buffer(&logbuf_len);
        if (!logbuf)
                return;

        off = sdev_format_header(logbuf, logbuf_len, scmd_name(cmd),
                                 scsi_cmd_to_rq((struct scsi_cmnd *)cmd)->tag);

        if (off >= logbuf_len)
                goto out_printk;

        if (msg) {
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "%s: ", msg);
                if (WARN_ON(off >= logbuf_len))
                        goto out_printk;
        }
        if (mlret_string)
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "%s ", mlret_string);
        else
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "UNKNOWN(0x%02x) ", disposition);
        if (WARN_ON(off >= logbuf_len))
                goto out_printk;

        off += scnprintf(logbuf + off, logbuf_len - off, "Result: ");
        if (WARN_ON(off >= logbuf_len))
                goto out_printk;

        if (hb_string)
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "hostbyte=%s ", hb_string);
        else
                off += scnprintf(logbuf + off, logbuf_len - off,
                                 "hostbyte=0x%02x ", host_byte(cmd->result));
        if (WARN_ON(off >= logbuf_len))
                goto out_printk;

        off += scnprintf(logbuf + off, logbuf_len - off,
                         "driverbyte=DRIVER_OK ");

        off += scnprintf(logbuf + off, logbuf_len - off,
                         "cmd_age=%lus", cmd_age);

out_printk:
        dev_printk(KERN_INFO, &cmd->device->sdev_gendev, "%s", logbuf);
        scsi_log_release_buffer(logbuf);
}
EXPORT_SYMBOL(scsi_print_result);


















































































































































































































  346 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef __SOUND_INFO_H
#define __SOUND_INFO_H

/*
 *  Header file for info interface
 *  Copyright (c) by Jaroslav Kysela <perex@perex.cz>
 */

#include <linux/poll.h>
#include <linux/seq_file.h>
#include <sound/core.h>

/* buffer for information */
struct snd_info_buffer {
        char *buffer;                /* pointer to begin of buffer */
        unsigned int curr;        /* current position in buffer */
        unsigned int size;        /* current size */
        unsigned int len;        /* total length of buffer */
        int stop;                /* stop flag */
        int error;                /* error code */
};

#define SNDRV_INFO_CONTENT_TEXT                0
#define SNDRV_INFO_CONTENT_DATA                1

struct snd_info_entry;

struct snd_info_entry_text {
        void (*read)(struct snd_info_entry *entry,
                     struct snd_info_buffer *buffer);
        void (*write)(struct snd_info_entry *entry,
                      struct snd_info_buffer *buffer);
};

struct snd_info_entry_ops {
        int (*open)(struct snd_info_entry *entry,
                    unsigned short mode, void **file_private_data);
        int (*release)(struct snd_info_entry *entry,
                       unsigned short mode, void *file_private_data);
        ssize_t (*read)(struct snd_info_entry *entry, void *file_private_data,
                        struct file *file, char __user *buf,
                        size_t count, loff_t pos);
        ssize_t (*write)(struct snd_info_entry *entry, void *file_private_data,
                         struct file *file, const char __user *buf,
                         size_t count, loff_t pos);
        loff_t (*llseek)(struct snd_info_entry *entry,
                         void *file_private_data, struct file *file,
                         loff_t offset, int orig);
        __poll_t (*poll)(struct snd_info_entry *entry,
                             void *file_private_data, struct file *file,
                             poll_table *wait);
        int (*ioctl)(struct snd_info_entry *entry, void *file_private_data,
                     struct file *file, unsigned int cmd, unsigned long arg);
        int (*mmap)(struct snd_info_entry *entry, void *file_private_data,
                    struct inode *inode, struct file *file,
                    struct vm_area_struct *vma);
};

struct snd_info_entry {
        const char *name;
        umode_t mode;
        long size;
        unsigned short content;
        union {
                struct snd_info_entry_text text;
                const struct snd_info_entry_ops *ops;
        } c;
        struct snd_info_entry *parent;
        struct module *module;
        void *private_data;
        void (*private_free)(struct snd_info_entry *entry);
        struct proc_dir_entry *p;
        struct mutex access;
        struct list_head children;
        struct list_head list;
};

#if defined(CONFIG_SND_OSSEMUL) && defined(CONFIG_SND_PROC_FS)
int snd_info_minor_register(void);
#else
#define snd_info_minor_register()        0
#endif


#ifdef CONFIG_SND_PROC_FS

extern struct snd_info_entry *snd_seq_root;
#ifdef CONFIG_SND_OSSEMUL
extern struct snd_info_entry *snd_oss_root;
void snd_card_info_read_oss(struct snd_info_buffer *buffer);
#else
#define snd_oss_root NULL
static inline void snd_card_info_read_oss(struct snd_info_buffer *buffer) {}
#endif

/**
 * snd_iprintf - printf on the procfs buffer
 * @buf: the procfs buffer
 * @fmt: the printf format
 *
 * Outputs the string on the procfs buffer just like printf().
 *
 * Return: zero for success, or a negative error code.
 */
#define snd_iprintf(buf, fmt, args...) \
        seq_printf((struct seq_file *)(buf)->buffer, fmt, ##args)

int snd_info_init(void);
int snd_info_done(void);

int snd_info_get_line(struct snd_info_buffer *buffer, char *line, int len);
const char *snd_info_get_str(char *dest, const char *src, int len);
struct snd_info_entry *snd_info_create_module_entry(struct module *module,
                                               const char *name,
                                               struct snd_info_entry *parent);
struct snd_info_entry *snd_info_create_card_entry(struct snd_card *card,
                                             const char *name,
                                             struct snd_info_entry *parent);
void snd_info_free_entry(struct snd_info_entry *entry);

int snd_info_card_create(struct snd_card *card);
int snd_info_card_register(struct snd_card *card);
int snd_info_card_free(struct snd_card *card);
void snd_info_card_disconnect(struct snd_card *card);
void snd_info_card_id_change(struct snd_card *card);
int snd_info_register(struct snd_info_entry *entry);

/* for card drivers */
static inline int snd_card_proc_new(struct snd_card *card, const char *name,
                                    struct snd_info_entry **entryp)
{
        *entryp = snd_info_create_card_entry(card, name, card->proc_root);
        return *entryp ? 0 : -ENOMEM;
}

static inline void snd_info_set_text_ops(struct snd_info_entry *entry, 
        void *private_data,
        void (*read)(struct snd_info_entry *, struct snd_info_buffer *))
{
        entry->private_data = private_data;
        entry->c.text.read = read;
}

int snd_card_rw_proc_new(struct snd_card *card, const char *name,
                         void *private_data,
                         void (*read)(struct snd_info_entry *,
                                      struct snd_info_buffer *),
                         void (*write)(struct snd_info_entry *entry,
                                       struct snd_info_buffer *buffer));

int snd_info_check_reserved_words(const char *str);

#else

#define snd_seq_root NULL
#define snd_oss_root NULL

static inline int snd_iprintf(struct snd_info_buffer *buffer, char *fmt, ...) { return 0; }
static inline int snd_info_init(void) { return 0; }
static inline int snd_info_done(void) { return 0; }

static inline int snd_info_get_line(struct snd_info_buffer *buffer, char *line, int len) { return 0; }
static inline char *snd_info_get_str(char *dest, char *src, int len) { return NULL; }
static inline struct snd_info_entry *snd_info_create_module_entry(struct module *module, const char *name, struct snd_info_entry *parent) { return NULL; }
static inline struct snd_info_entry *snd_info_create_card_entry(struct snd_card *card, const char *name, struct snd_info_entry *parent) { return NULL; }
static inline void snd_info_free_entry(struct snd_info_entry *entry) { ; }

static inline int snd_info_card_create(struct snd_card *card) { return 0; }
static inline int snd_info_card_register(struct snd_card *card) { return 0; }
static inline int snd_info_card_free(struct snd_card *card) { return 0; }
static inline void snd_info_card_disconnect(struct snd_card *card) { }
static inline void snd_info_card_id_change(struct snd_card *card) { }
static inline int snd_info_register(struct snd_info_entry *entry) { return 0; }

static inline int snd_card_proc_new(struct snd_card *card, const char *name,
                                    struct snd_info_entry **entryp) { return -EINVAL; }
static inline void snd_info_set_text_ops(struct snd_info_entry *entry __attribute__((unused)),
                                         void *private_data,
                                         void (*read)(struct snd_info_entry *, struct snd_info_buffer *)) {}
static inline int snd_card_rw_proc_new(struct snd_card *card, const char *name,
                                       void *private_data,
                                       void (*read)(struct snd_info_entry *,
                                                    struct snd_info_buffer *),
                                       void (*write)(struct snd_info_entry *entry,
                                                     struct snd_info_buffer *buffer))
{
        return 0;
}
static inline int snd_info_check_reserved_words(const char *str) { return 1; }

#endif

/**
 * snd_card_ro_proc_new - Create a read-only text proc file entry for the card
 * @card: the card instance
 * @name: the file name
 * @private_data: the arbitrary private data
 * @read: the read callback
 *
 * This proc file entry will be registered via snd_card_register() call, and
 * it will be removed automatically at the card removal, too.
 */
static inline int
snd_card_ro_proc_new(struct snd_card *card, const char *name,
                     void *private_data,
                     void (*read)(struct snd_info_entry *,
                                  struct snd_info_buffer *))
{
        return snd_card_rw_proc_new(card, name, private_data, read, NULL);
}

/*
 * OSS info part
 */

#if defined(CONFIG_SND_OSSEMUL) && defined(CONFIG_SND_PROC_FS)

#define SNDRV_OSS_INFO_DEV_AUDIO        0
#define SNDRV_OSS_INFO_DEV_SYNTH        1
#define SNDRV_OSS_INFO_DEV_MIDI                2
#define SNDRV_OSS_INFO_DEV_TIMERS        4
#define SNDRV_OSS_INFO_DEV_MIXERS        5

#define SNDRV_OSS_INFO_DEV_COUNT        6

int snd_oss_info_register(int dev, int num, char *string);
#define snd_oss_info_unregister(dev, num) snd_oss_info_register(dev, num, NULL)

#endif /* CONFIG_SND_OSSEMUL && CONFIG_SND_PROC_FS */

#endif /* __SOUND_INFO_H */



































































    7 
    3 






   29 












   29 

   16 















    3 


    3 




    2 






    8 

    3 











   52 





   52 









   45 
   45 

   43 




   45 



    6 





    2 



    7 
    2 

    2 
    3 





    3 


    1 


    2 


    2 


    3 



    2 







   19 







   29 
   16 

   45 
   45 




   12 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/syscalls.h>
#include <linux/fdtable.h>
#include <linux/string.h>
#include <linux/random.h>
#include <linux/module.h>
#include <linux/ptrace.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/cache.h>
#include <linux/bug.h>
#include <linux/err.h>
#include <linux/kcmp.h>
#include <linux/capability.h>
#include <linux/list.h>
#include <linux/eventpoll.h>
#include <linux/file.h>

#include <asm/unistd.h>

/*
 * We don't expose the real in-memory order of objects for security reasons.
 * But still the comparison results should be suitable for sorting. So we
 * obfuscate kernel pointers values and compare the production instead.
 *
 * The obfuscation is done in two steps. First we xor the kernel pointer with
 * a random value, which puts pointer into a new position in a reordered space.
 * Secondly we multiply the xor production with a large odd random number to
 * permute its bits even more (the odd multiplier guarantees that the product
 * is unique ever after the high bits are truncated, since any odd number is
 * relative prime to 2^n).
 *
 * Note also that the obfuscation itself is invisible to userspace and if needed
 * it can be changed to an alternate scheme.
 */
static unsigned long cookies[KCMP_TYPES][2] __read_mostly;

static long kptr_obfuscate(long v, int type)
{
        return (v ^ cookies[type][0]) * cookies[type][1];
}

/*
 * 0 - equal, i.e. v1 = v2
 * 1 - less than, i.e. v1 < v2
 * 2 - greater than, i.e. v1 > v2
 * 3 - not equal but ordering unavailable (reserved for future)
 */
static int kcmp_ptr(void *v1, void *v2, enum kcmp_type type)
{
        long t1, t2;

        t1 = kptr_obfuscate((long)v1, type);
        t2 = kptr_obfuscate((long)v2, type);

        return (t1 < t2) | ((t1 > t2) << 1);
}

/* The caller must have pinned the task */
static struct file *
get_file_raw_ptr(struct task_struct *task, unsigned int idx)
{
        struct file *file;

        file = fget_task(task, idx);
        if (file)
                fput(file);

        return file;
}

static void kcmp_unlock(struct rw_semaphore *l1, struct rw_semaphore *l2)
{
        if (likely(l2 != l1))
                up_read(l2);
        up_read(l1);
}

static int kcmp_lock(struct rw_semaphore *l1, struct rw_semaphore *l2)
{
        int err;

        if (l2 > l1)
                swap(l1, l2);

        err = down_read_killable(l1);
        if (!err && likely(l1 != l2)) {
                err = down_read_killable_nested(l2, SINGLE_DEPTH_NESTING);
                if (err)
                        up_read(l1);
        }

        return err;
}

#ifdef CONFIG_EPOLL
static int kcmp_epoll_target(struct task_struct *task1,
                             struct task_struct *task2,
                             unsigned long idx1,
                             struct kcmp_epoll_slot __user *uslot)
{
        struct file *filp, *filp_epoll, *filp_tgt;
        struct kcmp_epoll_slot slot;

        if (copy_from_user(&slot, uslot, sizeof(slot)))
                return -EFAULT;

        filp = get_file_raw_ptr(task1, idx1);
        if (!filp)
                return -EBADF;

        filp_epoll = fget_task(task2, slot.efd);
        if (!filp_epoll)
                return -EBADF;

        filp_tgt = get_epoll_tfile_raw_ptr(filp_epoll, slot.tfd, slot.toff);
        fput(filp_epoll);

        if (IS_ERR(filp_tgt))
                return PTR_ERR(filp_tgt);

        return kcmp_ptr(filp, filp_tgt, KCMP_FILE);
}
#else
static int kcmp_epoll_target(struct task_struct *task1,
                             struct task_struct *task2,
                             unsigned long idx1,
                             struct kcmp_epoll_slot __user *uslot)
{
        return -EOPNOTSUPP;
}
#endif

SYSCALL_DEFINE5(kcmp, pid_t, pid1, pid_t, pid2, int, type,
                unsigned long, idx1, unsigned long, idx2)
{
        struct task_struct *task1, *task2;
        int ret;

        rcu_read_lock();

        /*
         * Tasks are looked up in caller's PID namespace only.
         */
        task1 = find_task_by_vpid(pid1);
        task2 = find_task_by_vpid(pid2);
        if (!task1 || !task2)
                goto err_no_task;

        get_task_struct(task1);
        get_task_struct(task2);

        rcu_read_unlock();

        /*
         * One should have enough rights to inspect task details.
         */
        ret = kcmp_lock(&task1->signal->exec_update_lock,
                        &task2->signal->exec_update_lock);
        if (ret)
                goto err;
        if (!ptrace_may_access(task1, PTRACE_MODE_READ_REALCREDS) ||
            !ptrace_may_access(task2, PTRACE_MODE_READ_REALCREDS)) {
                ret = -EPERM;
                goto err_unlock;
        }

        switch (type) {
        case KCMP_FILE: {
                struct file *filp1, *filp2;

                filp1 = get_file_raw_ptr(task1, idx1);
                filp2 = get_file_raw_ptr(task2, idx2);

                if (filp1 && filp2)
                        ret = kcmp_ptr(filp1, filp2, KCMP_FILE);
                else
                        ret = -EBADF;
                break;
        }
        case KCMP_VM:
                ret = kcmp_ptr(task1->mm, task2->mm, KCMP_VM);
                break;
        case KCMP_FILES:
                ret = kcmp_ptr(task1->files, task2->files, KCMP_FILES);
                break;
        case KCMP_FS:
                ret = kcmp_ptr(task1->fs, task2->fs, KCMP_FS);
                break;
        case KCMP_SIGHAND:
                ret = kcmp_ptr(task1->sighand, task2->sighand, KCMP_SIGHAND);
                break;
        case KCMP_IO:
                ret = kcmp_ptr(task1->io_context, task2->io_context, KCMP_IO);
                break;
        case KCMP_SYSVSEM:
#ifdef CONFIG_SYSVIPC
                ret = kcmp_ptr(task1->sysvsem.undo_list,
                               task2->sysvsem.undo_list,
                               KCMP_SYSVSEM);
#else
                ret = -EOPNOTSUPP;
#endif
                break;
        case KCMP_EPOLL_TFD:
                ret = kcmp_epoll_target(task1, task2, idx1, (void *)idx2);
                break;
        default:
                ret = -EINVAL;
                break;
        }

err_unlock:
        kcmp_unlock(&task1->signal->exec_update_lock,
                    &task2->signal->exec_update_lock);
err:
        put_task_struct(task1);
        put_task_struct(task2);

        return ret;

err_no_task:
        rcu_read_unlock();
        return -ESRCH;
}

static __init int kcmp_cookies_init(void)
{
        int i;

        get_random_bytes(cookies, sizeof(cookies));

        for (i = 0; i < KCMP_TYPES; i++)
                cookies[i][1] |= (~(~0UL >>  1) | 1);

        return 0;
}
arch_initcall(kcmp_cookies_init);
























   36 




















   25 






   25 













    4 






    4 










   50 


   19 





























   37 



   37 

   37 










  162 

  131 



   32 











   27 

   20 

    7 











  170 











   48 
















































    3 
    3 














    5 





    5 












   37 
    1 






   41 
   41 








  157 





  104 
   23 















   37 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
 */

#include <rdma/rdma_cm.h>
#include <rdma/ib_verbs.h>
#include <rdma/restrack.h>
#include <rdma/rdma_counter.h>
#include <linux/mutex.h>
#include <linux/sched/task.h>
#include <linux/pid_namespace.h>

#include "cma_priv.h"
#include "restrack.h"

/**
 * rdma_restrack_init() - initialize and allocate resource tracking
 * @dev:  IB device
 *
 * Return: 0 on success
 */
int rdma_restrack_init(struct ib_device *dev)
{
        struct rdma_restrack_root *rt;
        int i;

        dev->res = kcalloc(RDMA_RESTRACK_MAX, sizeof(*rt), GFP_KERNEL);
        if (!dev->res)
                return -ENOMEM;

        rt = dev->res;

        for (i = 0; i < RDMA_RESTRACK_MAX; i++)
                xa_init_flags(&rt[i].xa, XA_FLAGS_ALLOC);

        return 0;
}

/**
 * rdma_restrack_clean() - clean resource tracking
 * @dev:  IB device
 */
void rdma_restrack_clean(struct ib_device *dev)
{
        struct rdma_restrack_root *rt = dev->res;
        int i;

        for (i = 0 ; i < RDMA_RESTRACK_MAX; i++) {
                struct xarray *xa = &dev->res[i].xa;

                WARN_ON(!xa_empty(xa));
                xa_destroy(xa);
        }
        kfree(rt);
}

/**
 * rdma_restrack_count() - the current usage of specific object
 * @dev:  IB device
 * @type: actual type of object to operate
 * @show_details: count driver specific objects
 */
int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type,
                        bool show_details)
{
        struct rdma_restrack_root *rt = &dev->res[type];
        struct rdma_restrack_entry *e;
        XA_STATE(xas, &rt->xa, 0);
        u32 cnt = 0;

        xa_lock(&rt->xa);
        xas_for_each(&xas, e, U32_MAX) {
                if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details)
                        continue;
                cnt++;
        }
        xa_unlock(&rt->xa);
        return cnt;
}
EXPORT_SYMBOL(rdma_restrack_count);

static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
{
        switch (res->type) {
        case RDMA_RESTRACK_PD:
                return container_of(res, struct ib_pd, res)->device;
        case RDMA_RESTRACK_CQ:
                return container_of(res, struct ib_cq, res)->device;
        case RDMA_RESTRACK_QP:
                return container_of(res, struct ib_qp, res)->device;
        case RDMA_RESTRACK_CM_ID:
                return container_of(res, struct rdma_id_private,
                                    res)->id.device;
        case RDMA_RESTRACK_MR:
                return container_of(res, struct ib_mr, res)->device;
        case RDMA_RESTRACK_CTX:
                return container_of(res, struct ib_ucontext, res)->device;
        case RDMA_RESTRACK_COUNTER:
                return container_of(res, struct rdma_counter, res)->device;
        case RDMA_RESTRACK_SRQ:
                return container_of(res, struct ib_srq, res)->device;
        default:
                WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
                return NULL;
        }
}

/**
 * rdma_restrack_attach_task() - attach the task onto this resource,
 * valid for user space restrack entries.
 * @res:  resource entry
 * @task: the task to attach
 */
static void rdma_restrack_attach_task(struct rdma_restrack_entry *res,
                                      struct task_struct *task)
{
        if (WARN_ON_ONCE(!task))
                return;

        if (res->task)
                put_task_struct(res->task);
        get_task_struct(task);
        res->task = task;
        res->user = true;
}

/**
 * rdma_restrack_set_name() - set the task for this resource
 * @res:  resource entry
 * @caller: kernel name, the current task will be used if the caller is NULL.
 */
void rdma_restrack_set_name(struct rdma_restrack_entry *res, const char *caller)
{
        if (caller) {
                res->kern_name = caller;
                return;
        }

        rdma_restrack_attach_task(res, current);
}
EXPORT_SYMBOL(rdma_restrack_set_name);

/**
 * rdma_restrack_parent_name() - set the restrack name properties based
 * on parent restrack
 * @dst: destination resource entry
 * @parent: parent resource entry
 */
void rdma_restrack_parent_name(struct rdma_restrack_entry *dst,
                               const struct rdma_restrack_entry *parent)
{
        if (rdma_is_kernel_res(parent))
                dst->kern_name = parent->kern_name;
        else
                rdma_restrack_attach_task(dst, parent->task);
}
EXPORT_SYMBOL(rdma_restrack_parent_name);

/**
 * rdma_restrack_new() - Initializes new restrack entry to allow _put() interface
 * to release memory in fully automatic way.
 * @res: Entry to initialize
 * @type: REstrack type
 */
void rdma_restrack_new(struct rdma_restrack_entry *res,
                       enum rdma_restrack_type type)
{
        kref_init(&res->kref);
        init_completion(&res->comp);
        res->type = type;
}
EXPORT_SYMBOL(rdma_restrack_new);

/**
 * rdma_restrack_add() - add object to the reource tracking database
 * @res:  resource entry
 */
void rdma_restrack_add(struct rdma_restrack_entry *res)
{
        struct ib_device *dev = res_to_dev(res);
        struct rdma_restrack_root *rt;
        int ret = 0;

        if (!dev)
                return;

        if (res->no_track)
                goto out;

        rt = &dev->res[res->type];

        if (res->type == RDMA_RESTRACK_QP) {
                /* Special case to ensure that LQPN points to right QP */
                struct ib_qp *qp = container_of(res, struct ib_qp, res);

                WARN_ONCE(qp->qp_num >> 24 || qp->port >> 8,
                          "QP number 0x%0X and port 0x%0X", qp->qp_num,
                          qp->port);
                res->id = qp->qp_num;
                if (qp->qp_type == IB_QPT_SMI || qp->qp_type == IB_QPT_GSI)
                        res->id |= qp->port << 24;
                ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
                if (ret)
                        res->id = 0;

                if (qp->qp_type >= IB_QPT_DRIVER)
                        xa_set_mark(&rt->xa, res->id, RESTRACK_DD);
        } else if (res->type == RDMA_RESTRACK_COUNTER) {
                /* Special case to ensure that cntn points to right counter */
                struct rdma_counter *counter;

                counter = container_of(res, struct rdma_counter, res);
                ret = xa_insert(&rt->xa, counter->id, res, GFP_KERNEL);
                res->id = ret ? 0 : counter->id;
        } else {
                ret = xa_alloc_cyclic(&rt->xa, &res->id, res, xa_limit_32b,
                                      &rt->next_id, GFP_KERNEL);
                ret = (ret < 0) ? ret : 0;
        }

out:
        if (!ret)
                res->valid = true;
}
EXPORT_SYMBOL(rdma_restrack_add);

int __must_check rdma_restrack_get(struct rdma_restrack_entry *res)
{
        return kref_get_unless_zero(&res->kref);
}
EXPORT_SYMBOL(rdma_restrack_get);

/**
 * rdma_restrack_get_byid() - translate from ID to restrack object
 * @dev: IB device
 * @type: resource track type
 * @id: ID to take a look
 *
 * Return: Pointer to restrack entry or -ENOENT in case of error.
 */
struct rdma_restrack_entry *
rdma_restrack_get_byid(struct ib_device *dev,
                       enum rdma_restrack_type type, u32 id)
{
        struct rdma_restrack_root *rt = &dev->res[type];
        struct rdma_restrack_entry *res;

        xa_lock(&rt->xa);
        res = xa_load(&rt->xa, id);
        if (!res || !rdma_restrack_get(res))
                res = ERR_PTR(-ENOENT);
        xa_unlock(&rt->xa);

        return res;
}
EXPORT_SYMBOL(rdma_restrack_get_byid);

static void restrack_release(struct kref *kref)
{
        struct rdma_restrack_entry *res;

        res = container_of(kref, struct rdma_restrack_entry, kref);
        if (res->task) {
                put_task_struct(res->task);
                res->task = NULL;
        }
        complete(&res->comp);
}

int rdma_restrack_put(struct rdma_restrack_entry *res)
{
        return kref_put(&res->kref, restrack_release);
}
EXPORT_SYMBOL(rdma_restrack_put);

/**
 * rdma_restrack_del() - delete object from the reource tracking database
 * @res:  resource entry
 */
void rdma_restrack_del(struct rdma_restrack_entry *res)
{
        struct rdma_restrack_entry *old;
        struct rdma_restrack_root *rt;
        struct ib_device *dev;

        if (!res->valid) {
                if (res->task) {
                        put_task_struct(res->task);
                        res->task = NULL;
                }
                return;
        }

        if (res->no_track)
                goto out;

        dev = res_to_dev(res);
        if (WARN_ON(!dev))
                return;

        rt = &dev->res[res->type];

        old = xa_erase(&rt->xa, res->id);
        WARN_ON(old != res);

out:
        res->valid = false;
        rdma_restrack_put(res);
        wait_for_completion(&res->comp);
}
EXPORT_SYMBOL(rdma_restrack_del);












































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 



    5 

































    6 


    6 

















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
// SPDX-License-Identifier: GPL-2.0-or-later
//
// core.c  --  Voltage/Current Regulator framework.
//
// Copyright 2007, 2008 Wolfson Microelectronics PLC.
// Copyright 2008 SlimLogic Ltd.
//
// Author: Liam Girdwood <lrg@slimlogic.co.uk>

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/async.h>
#include <linux/err.h>
#include <linux/mutex.h>
#include <linux/suspend.h>
#include <linux/delay.h>
#include <linux/gpio/consumer.h>
#include <linux/of.h>
#include <linux/reboot.h>
#include <linux/regmap.h>
#include <linux/regulator/of_regulator.h>
#include <linux/regulator/consumer.h>
#include <linux/regulator/coupler.h>
#include <linux/regulator/driver.h>
#include <linux/regulator/machine.h>
#include <linux/module.h>

#define CREATE_TRACE_POINTS
#include <trace/events/regulator.h>

#include "dummy.h"
#include "internal.h"
#include "regnl.h"

static DEFINE_WW_CLASS(regulator_ww_class);
static DEFINE_MUTEX(regulator_nesting_mutex);
static DEFINE_MUTEX(regulator_list_mutex);
static LIST_HEAD(regulator_map_list);
static LIST_HEAD(regulator_ena_gpio_list);
static LIST_HEAD(regulator_supply_alias_list);
static LIST_HEAD(regulator_coupler_list);
static bool has_full_constraints;

static struct dentry *debugfs_root;

/*
 * struct regulator_map
 *
 * Used to provide symbolic supply names to devices.
 */
struct regulator_map {
        struct list_head list;
        const char *dev_name;   /* The dev_name() for the consumer */
        const char *supply;
        struct regulator_dev *regulator;
};

/*
 * struct regulator_enable_gpio
 *
 * Management for shared enable GPIO pin
 */
struct regulator_enable_gpio {
        struct list_head list;
        struct gpio_desc *gpiod;
        u32 enable_count;        /* a number of enabled shared GPIO */
        u32 request_count;        /* a number of requested shared GPIO */
};

/*
 * struct regulator_supply_alias
 *
 * Used to map lookups for a supply onto an alternative device.
 */
struct regulator_supply_alias {
        struct list_head list;
        struct device *src_dev;
        const char *src_supply;
        struct device *alias_dev;
        const char *alias_supply;
};

static int _regulator_is_enabled(struct regulator_dev *rdev);
static int _regulator_disable(struct regulator *regulator);
static int _regulator_get_error_flags(struct regulator_dev *rdev, unsigned int *flags);
static int _regulator_get_current_limit(struct regulator_dev *rdev);
static unsigned int _regulator_get_mode(struct regulator_dev *rdev);
static int _notifier_call_chain(struct regulator_dev *rdev,
                                  unsigned long event, void *data);
static int _regulator_do_set_voltage(struct regulator_dev *rdev,
                                     int min_uV, int max_uV);
static int regulator_balance_voltage(struct regulator_dev *rdev,
                                     suspend_state_t state);
static struct regulator *create_regulator(struct regulator_dev *rdev,
                                          struct device *dev,
                                          const char *supply_name);
static void destroy_regulator(struct regulator *regulator);
static void _regulator_put(struct regulator *regulator);

const char *rdev_get_name(struct regulator_dev *rdev)
{
        if (rdev->constraints && rdev->constraints->name)
                return rdev->constraints->name;
        else if (rdev->desc->name)
                return rdev->desc->name;
        else
                return "";
}
EXPORT_SYMBOL_GPL(rdev_get_name);

static bool have_full_constraints(void)
{
        return has_full_constraints || of_have_populated_dt();
}

static bool regulator_ops_is_valid(struct regulator_dev *rdev, int ops)
{
        if (!rdev->constraints) {
                rdev_err(rdev, "no constraints\n");
                return false;
        }

        if (rdev->constraints->valid_ops_mask & ops)
                return true;

        return false;
}

/**
 * regulator_lock_nested - lock a single regulator
 * @rdev:                regulator source
 * @ww_ctx:                w/w mutex acquire context
 *
 * This function can be called many times by one task on
 * a single regulator and its mutex will be locked only
 * once. If a task, which is calling this function is other
 * than the one, which initially locked the mutex, it will
 * wait on mutex.
 *
 * Return: 0 on success or a negative error number on failure.
 */
static inline int regulator_lock_nested(struct regulator_dev *rdev,
                                        struct ww_acquire_ctx *ww_ctx)
{
        bool lock = false;
        int ret = 0;

        mutex_lock(&regulator_nesting_mutex);

        if (!ww_mutex_trylock(&rdev->mutex, ww_ctx)) {
                if (rdev->mutex_owner == current)
                        rdev->ref_cnt++;
                else
                        lock = true;

                if (lock) {
                        mutex_unlock(&regulator_nesting_mutex);
                        ret = ww_mutex_lock(&rdev->mutex, ww_ctx);
                        mutex_lock(&regulator_nesting_mutex);
                }
        } else {
                lock = true;
        }

        if (lock && ret != -EDEADLK) {
                rdev->ref_cnt++;
                rdev->mutex_owner = current;
        }

        mutex_unlock(&regulator_nesting_mutex);

        return ret;
}

/**
 * regulator_lock - lock a single regulator
 * @rdev:                regulator source
 *
 * This function can be called many times by one task on
 * a single regulator and its mutex will be locked only
 * once. If a task, which is calling this function is other
 * than the one, which initially locked the mutex, it will
 * wait on mutex.
 */
static void regulator_lock(struct regulator_dev *rdev)
{
        regulator_lock_nested(rdev, NULL);
}

/**
 * regulator_unlock - unlock a single regulator
 * @rdev:                regulator_source
 *
 * This function unlocks the mutex when the
 * reference counter reaches 0.
 */
static void regulator_unlock(struct regulator_dev *rdev)
{
        mutex_lock(&regulator_nesting_mutex);

        if (--rdev->ref_cnt == 0) {
                rdev->mutex_owner = NULL;
                ww_mutex_unlock(&rdev->mutex);
        }

        WARN_ON_ONCE(rdev->ref_cnt < 0);

        mutex_unlock(&regulator_nesting_mutex);
}

/**
 * regulator_lock_two - lock two regulators
 * @rdev1:                first regulator
 * @rdev2:                second regulator
 * @ww_ctx:                w/w mutex acquire context
 *
 * Locks both rdevs using the regulator_ww_class.
 */
static void regulator_lock_two(struct regulator_dev *rdev1,
                               struct regulator_dev *rdev2,
                               struct ww_acquire_ctx *ww_ctx)
{
        struct regulator_dev *held, *contended;
        int ret;

        ww_acquire_init(ww_ctx, &regulator_ww_class);

        /* Try to just grab both of them */
        ret = regulator_lock_nested(rdev1, ww_ctx);
        WARN_ON(ret);
        ret = regulator_lock_nested(rdev2, ww_ctx);
        if (ret != -EDEADLOCK) {
                WARN_ON(ret);
                goto exit;
        }

        held = rdev1;
        contended = rdev2;
        while (true) {
                regulator_unlock(held);

                ww_mutex_lock_slow(&contended->mutex, ww_ctx);
                contended->ref_cnt++;
                contended->mutex_owner = current;
                swap(held, contended);
                ret = regulator_lock_nested(contended, ww_ctx);

                if (ret != -EDEADLOCK) {
                        WARN_ON(ret);
                        break;
                }
        }

exit:
        ww_acquire_done(ww_ctx);
}

/**
 * regulator_unlock_two - unlock two regulators
 * @rdev1:                first regulator
 * @rdev2:                second regulator
 * @ww_ctx:                w/w mutex acquire context
 *
 * The inverse of regulator_lock_two().
 */

static void regulator_unlock_two(struct regulator_dev *rdev1,
                                 struct regulator_dev *rdev2,
                                 struct ww_acquire_ctx *ww_ctx)
{
        regulator_unlock(rdev2);
        regulator_unlock(rdev1);
        ww_acquire_fini(ww_ctx);
}

static bool regulator_supply_is_couple(struct regulator_dev *rdev)
{
        struct regulator_dev *c_rdev;
        int i;

        for (i = 1; i < rdev->coupling_desc.n_coupled; i++) {
                c_rdev = rdev->coupling_desc.coupled_rdevs[i];

                if (rdev->supply->rdev == c_rdev)
                        return true;
        }

        return false;
}

static void regulator_unlock_recursive(struct regulator_dev *rdev,
                                       unsigned int n_coupled)
{
        struct regulator_dev *c_rdev, *supply_rdev;
        int i, supply_n_coupled;

        for (i = n_coupled; i > 0; i--) {
                c_rdev = rdev->coupling_desc.coupled_rdevs[i - 1];

                if (!c_rdev)
                        continue;

                if (c_rdev->supply && !regulator_supply_is_couple(c_rdev)) {
                        supply_rdev = c_rdev->supply->rdev;
                        supply_n_coupled = supply_rdev->coupling_desc.n_coupled;

                        regulator_unlock_recursive(supply_rdev,
                                                   supply_n_coupled);
                }

                regulator_unlock(c_rdev);
        }
}

static int regulator_lock_recursive(struct regulator_dev *rdev,
                                    struct regulator_dev **new_contended_rdev,
                                    struct regulator_dev **old_contended_rdev,
                                    struct ww_acquire_ctx *ww_ctx)
{
        struct regulator_dev *c_rdev;
        int i, err;

        for (i = 0; i < rdev->coupling_desc.n_coupled; i++) {
                c_rdev = rdev->coupling_desc.coupled_rdevs[i];

                if (!c_rdev)
                        continue;

                if (c_rdev != *old_contended_rdev) {
                        err = regulator_lock_nested(c_rdev, ww_ctx);
                        if (err) {
                                if (err == -EDEADLK) {
                                        *new_contended_rdev = c_rdev;
                                        goto err_unlock;
                                }

                                /* shouldn't happen */
                                WARN_ON_ONCE(err != -EALREADY);
                        }
                } else {
                        *old_contended_rdev = NULL;
                }

                if (c_rdev->supply && !regulator_supply_is_couple(c_rdev)) {
                        err = regulator_lock_recursive(c_rdev->supply->rdev,
                                                       new_contended_rdev,
                                                       old_contended_rdev,
                                                       ww_ctx);
                        if (err) {
                                regulator_unlock(c_rdev);
                                goto err_unlock;
                        }
                }
        }

        return 0;

err_unlock:
        regulator_unlock_recursive(rdev, i);

        return err;
}

/**
 * regulator_unlock_dependent - unlock regulator's suppliers and coupled
 *                                regulators
 * @rdev:                        regulator source
 * @ww_ctx:                        w/w mutex acquire context
 *
 * Unlock all regulators related with rdev by coupling or supplying.
 */
static void regulator_unlock_dependent(struct regulator_dev *rdev,
                                       struct ww_acquire_ctx *ww_ctx)
{
        regulator_unlock_recursive(rdev, rdev->coupling_desc.n_coupled);
        ww_acquire_fini(ww_ctx);
}

/**
 * regulator_lock_dependent - lock regulator's suppliers and coupled regulators
 * @rdev:                        regulator source
 * @ww_ctx:                        w/w mutex acquire context
 *
 * This function as a wrapper on regulator_lock_recursive(), which locks
 * all regulators related with rdev by coupling or supplying.
 */
static void regulator_lock_dependent(struct regulator_dev *rdev,
                                     struct ww_acquire_ctx *ww_ctx)
{
        struct regulator_dev *new_contended_rdev = NULL;
        struct regulator_dev *old_contended_rdev = NULL;
        int err;

        mutex_lock(&regulator_list_mutex);

        ww_acquire_init(ww_ctx, &regulator_ww_class);

        do {
                if (new_contended_rdev) {
                        ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
                        old_contended_rdev = new_contended_rdev;
                        old_contended_rdev->ref_cnt++;
                        old_contended_rdev->mutex_owner = current;
                }

                err = regulator_lock_recursive(rdev,
                                               &new_contended_rdev,
                                               &old_contended_rdev,
                                               ww_ctx);

                if (old_contended_rdev)
                        regulator_unlock(old_contended_rdev);

        } while (err == -EDEADLK);

        ww_acquire_done(ww_ctx);

        mutex_unlock(&regulator_list_mutex);
}

/* Platform voltage constraint check */
int regulator_check_voltage(struct regulator_dev *rdev,
                            int *min_uV, int *max_uV)
{
        BUG_ON(*min_uV > *max_uV);

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
                rdev_err(rdev, "voltage operation not allowed\n");
                return -EPERM;
        }

        if (*max_uV > rdev->constraints->max_uV)
                *max_uV = rdev->constraints->max_uV;
        if (*min_uV < rdev->constraints->min_uV)
                *min_uV = rdev->constraints->min_uV;

        if (*min_uV > *max_uV) {
                rdev_err(rdev, "unsupportable voltage range: %d-%duV\n",
                         *min_uV, *max_uV);
                return -EINVAL;
        }

        return 0;
}

/* return 0 if the state is valid */
static int regulator_check_states(suspend_state_t state)
{
        return (state > PM_SUSPEND_MAX || state == PM_SUSPEND_TO_IDLE);
}

/* Make sure we select a voltage that suits the needs of all
 * regulator consumers
 */
int regulator_check_consumers(struct regulator_dev *rdev,
                              int *min_uV, int *max_uV,
                              suspend_state_t state)
{
        struct regulator *regulator;
        struct regulator_voltage *voltage;

        list_for_each_entry(regulator, &rdev->consumer_list, list) {
                voltage = &regulator->voltage[state];
                /*
                 * Assume consumers that didn't say anything are OK
                 * with anything in the constraint range.
                 */
                if (!voltage->min_uV && !voltage->max_uV)
                        continue;

                if (*max_uV > voltage->max_uV)
                        *max_uV = voltage->max_uV;
                if (*min_uV < voltage->min_uV)
                        *min_uV = voltage->min_uV;
        }

        if (*min_uV > *max_uV) {
                rdev_err(rdev, "Restricting voltage, %u-%uuV\n",
                        *min_uV, *max_uV);
                return -EINVAL;
        }

        return 0;
}

/* current constraint check */
static int regulator_check_current_limit(struct regulator_dev *rdev,
                                        int *min_uA, int *max_uA)
{
        BUG_ON(*min_uA > *max_uA);

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_CURRENT)) {
                rdev_err(rdev, "current operation not allowed\n");
                return -EPERM;
        }

        if (*max_uA > rdev->constraints->max_uA &&
            rdev->constraints->max_uA)
                *max_uA = rdev->constraints->max_uA;
        if (*min_uA < rdev->constraints->min_uA)
                *min_uA = rdev->constraints->min_uA;

        if (*min_uA > *max_uA) {
                rdev_err(rdev, "unsupportable current range: %d-%duA\n",
                         *min_uA, *max_uA);
                return -EINVAL;
        }

        return 0;
}

/* operating mode constraint check */
static int regulator_mode_constrain(struct regulator_dev *rdev,
                                    unsigned int *mode)
{
        switch (*mode) {
        case REGULATOR_MODE_FAST:
        case REGULATOR_MODE_NORMAL:
        case REGULATOR_MODE_IDLE:
        case REGULATOR_MODE_STANDBY:
                break;
        default:
                rdev_err(rdev, "invalid mode %x specified\n", *mode);
                return -EINVAL;
        }

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_MODE)) {
                rdev_err(rdev, "mode operation not allowed\n");
                return -EPERM;
        }

        /* The modes are bitmasks, the most power hungry modes having
         * the lowest values. If the requested mode isn't supported
         * try higher modes.
         */
        while (*mode) {
                if (rdev->constraints->valid_modes_mask & *mode)
                        return 0;
                *mode /= 2;
        }

        return -EINVAL;
}

static inline struct regulator_state *
regulator_get_suspend_state(struct regulator_dev *rdev, suspend_state_t state)
{
        if (rdev->constraints == NULL)
                return NULL;

        switch (state) {
        case PM_SUSPEND_STANDBY:
                return &rdev->constraints->state_standby;
        case PM_SUSPEND_MEM:
                return &rdev->constraints->state_mem;
        case PM_SUSPEND_MAX:
                return &rdev->constraints->state_disk;
        default:
                return NULL;
        }
}

static const struct regulator_state *
regulator_get_suspend_state_check(struct regulator_dev *rdev, suspend_state_t state)
{
        const struct regulator_state *rstate;

        rstate = regulator_get_suspend_state(rdev, state);
        if (rstate == NULL)
                return NULL;

        /* If we have no suspend mode configuration don't set anything;
         * only warn if the driver implements set_suspend_voltage or
         * set_suspend_mode callback.
         */
        if (rstate->enabled != ENABLE_IN_SUSPEND &&
            rstate->enabled != DISABLE_IN_SUSPEND) {
                if (rdev->desc->ops->set_suspend_voltage ||
                    rdev->desc->ops->set_suspend_mode)
                        rdev_warn(rdev, "No configuration\n");
                return NULL;
        }

        return rstate;
}

static ssize_t microvolts_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        int uV;

        regulator_lock(rdev);
        uV = regulator_get_voltage_rdev(rdev);
        regulator_unlock(rdev);

        if (uV < 0)
                return uV;
        return sprintf(buf, "%d\n", uV);
}
static DEVICE_ATTR_RO(microvolts);

static ssize_t microamps_show(struct device *dev,
                              struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", _regulator_get_current_limit(rdev));
}
static DEVICE_ATTR_RO(microamps);

static ssize_t name_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%s\n", rdev_get_name(rdev));
}
static DEVICE_ATTR_RO(name);

static const char *regulator_opmode_to_str(int mode)
{
        switch (mode) {
        case REGULATOR_MODE_FAST:
                return "fast";
        case REGULATOR_MODE_NORMAL:
                return "normal";
        case REGULATOR_MODE_IDLE:
                return "idle";
        case REGULATOR_MODE_STANDBY:
                return "standby";
        }
        return "unknown";
}

static ssize_t regulator_print_opmode(char *buf, int mode)
{
        return sprintf(buf, "%s\n", regulator_opmode_to_str(mode));
}

static ssize_t opmode_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_opmode(buf, _regulator_get_mode(rdev));
}
static DEVICE_ATTR_RO(opmode);

static ssize_t regulator_print_state(char *buf, int state)
{
        if (state > 0)
                return sprintf(buf, "enabled\n");
        else if (state == 0)
                return sprintf(buf, "disabled\n");
        else
                return sprintf(buf, "unknown\n");
}

static ssize_t state_show(struct device *dev,
                          struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        ssize_t ret;

        regulator_lock(rdev);
        ret = regulator_print_state(buf, _regulator_is_enabled(rdev));
        regulator_unlock(rdev);

        return ret;
}
static DEVICE_ATTR_RO(state);

static ssize_t status_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        int status;
        char *label;

        status = rdev->desc->ops->get_status(rdev);
        if (status < 0)
                return status;

        switch (status) {
        case REGULATOR_STATUS_OFF:
                label = "off";
                break;
        case REGULATOR_STATUS_ON:
                label = "on";
                break;
        case REGULATOR_STATUS_ERROR:
                label = "error";
                break;
        case REGULATOR_STATUS_FAST:
                label = "fast";
                break;
        case REGULATOR_STATUS_NORMAL:
                label = "normal";
                break;
        case REGULATOR_STATUS_IDLE:
                label = "idle";
                break;
        case REGULATOR_STATUS_STANDBY:
                label = "standby";
                break;
        case REGULATOR_STATUS_BYPASS:
                label = "bypass";
                break;
        case REGULATOR_STATUS_UNDEFINED:
                label = "undefined";
                break;
        default:
                return -ERANGE;
        }

        return sprintf(buf, "%s\n", label);
}
static DEVICE_ATTR_RO(status);

static ssize_t min_microamps_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        if (!rdev->constraints)
                return sprintf(buf, "constraint not defined\n");

        return sprintf(buf, "%d\n", rdev->constraints->min_uA);
}
static DEVICE_ATTR_RO(min_microamps);

static ssize_t max_microamps_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        if (!rdev->constraints)
                return sprintf(buf, "constraint not defined\n");

        return sprintf(buf, "%d\n", rdev->constraints->max_uA);
}
static DEVICE_ATTR_RO(max_microamps);

static ssize_t min_microvolts_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        if (!rdev->constraints)
                return sprintf(buf, "constraint not defined\n");

        return sprintf(buf, "%d\n", rdev->constraints->min_uV);
}
static DEVICE_ATTR_RO(min_microvolts);

static ssize_t max_microvolts_show(struct device *dev,
                                   struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        if (!rdev->constraints)
                return sprintf(buf, "constraint not defined\n");

        return sprintf(buf, "%d\n", rdev->constraints->max_uV);
}
static DEVICE_ATTR_RO(max_microvolts);

static ssize_t requested_microamps_show(struct device *dev,
                                        struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        struct regulator *regulator;
        int uA = 0;

        regulator_lock(rdev);
        list_for_each_entry(regulator, &rdev->consumer_list, list) {
                if (regulator->enable_count)
                        uA += regulator->uA_load;
        }
        regulator_unlock(rdev);
        return sprintf(buf, "%d\n", uA);
}
static DEVICE_ATTR_RO(requested_microamps);

static ssize_t num_users_show(struct device *dev, struct device_attribute *attr,
                              char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        return sprintf(buf, "%d\n", rdev->use_count);
}
static DEVICE_ATTR_RO(num_users);

static ssize_t type_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        switch (rdev->desc->type) {
        case REGULATOR_VOLTAGE:
                return sprintf(buf, "voltage\n");
        case REGULATOR_CURRENT:
                return sprintf(buf, "current\n");
        }
        return sprintf(buf, "unknown\n");
}
static DEVICE_ATTR_RO(type);

static ssize_t suspend_mem_microvolts_show(struct device *dev,
                                           struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", rdev->constraints->state_mem.uV);
}
static DEVICE_ATTR_RO(suspend_mem_microvolts);

static ssize_t suspend_disk_microvolts_show(struct device *dev,
                                            struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", rdev->constraints->state_disk.uV);
}
static DEVICE_ATTR_RO(suspend_disk_microvolts);

static ssize_t suspend_standby_microvolts_show(struct device *dev,
                                               struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", rdev->constraints->state_standby.uV);
}
static DEVICE_ATTR_RO(suspend_standby_microvolts);

static ssize_t suspend_mem_mode_show(struct device *dev,
                                     struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_opmode(buf,
                rdev->constraints->state_mem.mode);
}
static DEVICE_ATTR_RO(suspend_mem_mode);

static ssize_t suspend_disk_mode_show(struct device *dev,
                                      struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_opmode(buf,
                rdev->constraints->state_disk.mode);
}
static DEVICE_ATTR_RO(suspend_disk_mode);

static ssize_t suspend_standby_mode_show(struct device *dev,
                                         struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_opmode(buf,
                rdev->constraints->state_standby.mode);
}
static DEVICE_ATTR_RO(suspend_standby_mode);

static ssize_t suspend_mem_state_show(struct device *dev,
                                      struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_state(buf,
                        rdev->constraints->state_mem.enabled);
}
static DEVICE_ATTR_RO(suspend_mem_state);

static ssize_t suspend_disk_state_show(struct device *dev,
                                       struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_state(buf,
                        rdev->constraints->state_disk.enabled);
}
static DEVICE_ATTR_RO(suspend_disk_state);

static ssize_t suspend_standby_state_show(struct device *dev,
                                          struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return regulator_print_state(buf,
                        rdev->constraints->state_standby.enabled);
}
static DEVICE_ATTR_RO(suspend_standby_state);

static ssize_t bypass_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);
        const char *report;
        bool bypass;
        int ret;

        ret = rdev->desc->ops->get_bypass(rdev, &bypass);

        if (ret != 0)
                report = "unknown";
        else if (bypass)
                report = "enabled";
        else
                report = "disabled";

        return sprintf(buf, "%s\n", report);
}
static DEVICE_ATTR_RO(bypass);

static ssize_t power_budget_milliwatt_show(struct device *dev,
                                           struct device_attribute *attr,
                                           char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", rdev->constraints->pw_budget_mW);
}
static DEVICE_ATTR_RO(power_budget_milliwatt);

static ssize_t power_requested_milliwatt_show(struct device *dev,
                                              struct device_attribute *attr,
                                              char *buf)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        return sprintf(buf, "%d\n", rdev->pw_requested_mW);
}
static DEVICE_ATTR_RO(power_requested_milliwatt);

#define REGULATOR_ERROR_ATTR(name, bit)                                                        \
        static ssize_t name##_show(struct device *dev, struct device_attribute *attr,        \
                                   char *buf)                                                \
        {                                                                                \
                int ret;                                                                \
                unsigned int flags;                                                        \
                struct regulator_dev *rdev = dev_get_drvdata(dev);                        \
                ret = _regulator_get_error_flags(rdev, &flags);                                \
                if (ret)                                                                \
                        return ret;                                                        \
                return sysfs_emit(buf, "%d\n", !!(flags & (bit)));                        \
        }                                                                                \
        static DEVICE_ATTR_RO(name)

REGULATOR_ERROR_ATTR(under_voltage, REGULATOR_ERROR_UNDER_VOLTAGE);
REGULATOR_ERROR_ATTR(over_current, REGULATOR_ERROR_OVER_CURRENT);
REGULATOR_ERROR_ATTR(regulation_out, REGULATOR_ERROR_REGULATION_OUT);
REGULATOR_ERROR_ATTR(fail, REGULATOR_ERROR_FAIL);
REGULATOR_ERROR_ATTR(over_temp, REGULATOR_ERROR_OVER_TEMP);
REGULATOR_ERROR_ATTR(under_voltage_warn, REGULATOR_ERROR_UNDER_VOLTAGE_WARN);
REGULATOR_ERROR_ATTR(over_current_warn, REGULATOR_ERROR_OVER_CURRENT_WARN);
REGULATOR_ERROR_ATTR(over_voltage_warn, REGULATOR_ERROR_OVER_VOLTAGE_WARN);
REGULATOR_ERROR_ATTR(over_temp_warn, REGULATOR_ERROR_OVER_TEMP_WARN);

/* Calculate the new optimum regulator operating mode based on the new total
 * consumer load. All locks held by caller
 */
static int drms_uA_update(struct regulator_dev *rdev)
{
        struct regulator *sibling;
        int current_uA = 0, output_uV, input_uV, err;
        unsigned int mode;

        /*
         * first check to see if we can set modes at all, otherwise just
         * tell the consumer everything is OK.
         */
        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_DRMS)) {
                rdev_dbg(rdev, "DRMS operation not allowed\n");
                return 0;
        }

        if (!rdev->desc->ops->get_optimum_mode &&
            !rdev->desc->ops->set_load)
                return 0;

        if (!rdev->desc->ops->set_mode &&
            !rdev->desc->ops->set_load)
                return -EINVAL;

        /* calc total requested load */
        list_for_each_entry(sibling, &rdev->consumer_list, list) {
                if (sibling->enable_count)
                        current_uA += sibling->uA_load;
        }

        current_uA += rdev->constraints->system_load;

        if (rdev->desc->ops->set_load) {
                /* set the optimum mode for our new total regulator load */
                err = rdev->desc->ops->set_load(rdev, current_uA);
                if (err < 0)
                        rdev_err(rdev, "failed to set load %d: %pe\n",
                                 current_uA, ERR_PTR(err));
        } else {
                /*
                 * Unfortunately in some cases the constraints->valid_ops has
                 * REGULATOR_CHANGE_DRMS but there are no valid modes listed.
                 * That's not really legit but we won't consider it a fatal
                 * error here. We'll treat it as if REGULATOR_CHANGE_DRMS
                 * wasn't set.
                 */
                if (!rdev->constraints->valid_modes_mask) {
                        rdev_dbg(rdev, "Can change modes; but no valid mode\n");
                        return 0;
                }

                /* get output voltage */
                output_uV = regulator_get_voltage_rdev(rdev);

                /*
                 * Don't return an error; if regulator driver cares about
                 * output_uV then it's up to the driver to validate.
                 */
                if (output_uV <= 0)
                        rdev_dbg(rdev, "invalid output voltage found\n");

                /* get input voltage */
                input_uV = 0;
                if (rdev->supply)
                        input_uV = regulator_get_voltage_rdev(rdev->supply->rdev);
                if (input_uV <= 0)
                        input_uV = rdev->constraints->input_uV;

                /*
                 * Don't return an error; if regulator driver cares about
                 * input_uV then it's up to the driver to validate.
                 */
                if (input_uV <= 0)
                        rdev_dbg(rdev, "invalid input voltage found\n");

                /* now get the optimum mode for our new total regulator load */
                mode = rdev->desc->ops->get_optimum_mode(rdev, input_uV,
                                                         output_uV, current_uA);

                /* check the new mode is allowed */
                err = regulator_mode_constrain(rdev, &mode);
                if (err < 0) {
                        rdev_err(rdev, "failed to get optimum mode @ %d uA %d -> %d uV: %pe\n",
                                 current_uA, input_uV, output_uV, ERR_PTR(err));
                        return err;
                }

                err = rdev->desc->ops->set_mode(rdev, mode);
                if (err < 0)
                        rdev_err(rdev, "failed to set optimum mode %x: %pe\n",
                                 mode, ERR_PTR(err));
        }

        return err;
}

static int __suspend_set_state(struct regulator_dev *rdev,
                               const struct regulator_state *rstate)
{
        int ret = 0;

        if (rstate->enabled == ENABLE_IN_SUSPEND &&
                rdev->desc->ops->set_suspend_enable)
                ret = rdev->desc->ops->set_suspend_enable(rdev);
        else if (rstate->enabled == DISABLE_IN_SUSPEND &&
                rdev->desc->ops->set_suspend_disable)
                ret = rdev->desc->ops->set_suspend_disable(rdev);
        else /* OK if set_suspend_enable or set_suspend_disable is NULL */
                ret = 0;

        if (ret < 0) {
                rdev_err(rdev, "failed to enabled/disable: %pe\n", ERR_PTR(ret));
                return ret;
        }

        if (rdev->desc->ops->set_suspend_voltage && rstate->uV > 0) {
                ret = rdev->desc->ops->set_suspend_voltage(rdev, rstate->uV);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set voltage: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        if (rdev->desc->ops->set_suspend_mode && rstate->mode > 0) {
                ret = rdev->desc->ops->set_suspend_mode(rdev, rstate->mode);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set mode: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        return ret;
}

static int suspend_set_initial_state(struct regulator_dev *rdev)
{
        const struct regulator_state *rstate;

        rstate = regulator_get_suspend_state_check(rdev,
                        rdev->constraints->initial_state);
        if (!rstate)
                return 0;

        return __suspend_set_state(rdev, rstate);
}

#if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG)
static void print_constraints_debug(struct regulator_dev *rdev)
{
        struct regulation_constraints *constraints = rdev->constraints;
        char buf[160] = "";
        size_t len = sizeof(buf) - 1;
        int count = 0;
        int ret;

        if (constraints->min_uV && constraints->max_uV) {
                if (constraints->min_uV == constraints->max_uV)
                        count += scnprintf(buf + count, len - count, "%d mV ",
                                           constraints->min_uV / 1000);
                else
                        count += scnprintf(buf + count, len - count,
                                           "%d <--> %d mV ",
                                           constraints->min_uV / 1000,
                                           constraints->max_uV / 1000);
        }

        if (!constraints->min_uV ||
            constraints->min_uV != constraints->max_uV) {
                ret = regulator_get_voltage_rdev(rdev);
                if (ret > 0)
                        count += scnprintf(buf + count, len - count,
                                           "at %d mV ", ret / 1000);
        }

        if (constraints->uV_offset)
                count += scnprintf(buf + count, len - count, "%dmV offset ",
                                   constraints->uV_offset / 1000);

        if (constraints->min_uA && constraints->max_uA) {
                if (constraints->min_uA == constraints->max_uA)
                        count += scnprintf(buf + count, len - count, "%d mA ",
                                           constraints->min_uA / 1000);
                else
                        count += scnprintf(buf + count, len - count,
                                           "%d <--> %d mA ",
                                           constraints->min_uA / 1000,
                                           constraints->max_uA / 1000);
        }

        if (!constraints->min_uA ||
            constraints->min_uA != constraints->max_uA) {
                ret = _regulator_get_current_limit(rdev);
                if (ret > 0)
                        count += scnprintf(buf + count, len - count,
                                           "at %d mA ", ret / 1000);
        }

        if (constraints->valid_modes_mask & REGULATOR_MODE_FAST)
                count += scnprintf(buf + count, len - count, "fast ");
        if (constraints->valid_modes_mask & REGULATOR_MODE_NORMAL)
                count += scnprintf(buf + count, len - count, "normal ");
        if (constraints->valid_modes_mask & REGULATOR_MODE_IDLE)
                count += scnprintf(buf + count, len - count, "idle ");
        if (constraints->valid_modes_mask & REGULATOR_MODE_STANDBY)
                count += scnprintf(buf + count, len - count, "standby ");

        if (constraints->pw_budget_mW)
                count += scnprintf(buf + count, len - count, "%d mW budget",
                                   constraints->pw_budget_mW);

        if (!count)
                count = scnprintf(buf, len, "no parameters");
        else
                --count;

        count += scnprintf(buf + count, len - count, ", %s",
                _regulator_is_enabled(rdev) ? "enabled" : "disabled");

        rdev_dbg(rdev, "%s\n", buf);
}
#else /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */
static inline void print_constraints_debug(struct regulator_dev *rdev) {}
#endif /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */

static void print_constraints(struct regulator_dev *rdev)
{
        struct regulation_constraints *constraints = rdev->constraints;

        print_constraints_debug(rdev);

        if ((constraints->min_uV != constraints->max_uV) &&
            !regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE))
                rdev_warn(rdev,
                          "Voltage range but no REGULATOR_CHANGE_VOLTAGE\n");
}

static int machine_constraints_voltage(struct regulator_dev *rdev,
        struct regulation_constraints *constraints)
{
        const struct regulator_ops *ops = rdev->desc->ops;
        int ret;

        /* do we need to apply the constraint voltage */
        if (rdev->constraints->apply_uV &&
            rdev->constraints->min_uV && rdev->constraints->max_uV) {
                int target_min, target_max;
                int current_uV = regulator_get_voltage_rdev(rdev);

                if (current_uV == -ENOTRECOVERABLE) {
                        /* This regulator can't be read and must be initialized */
                        rdev_info(rdev, "Setting %d-%duV\n",
                                  rdev->constraints->min_uV,
                                  rdev->constraints->max_uV);
                        _regulator_do_set_voltage(rdev,
                                                  rdev->constraints->min_uV,
                                                  rdev->constraints->max_uV);
                        current_uV = regulator_get_voltage_rdev(rdev);
                }

                if (current_uV < 0) {
                        if (current_uV != -EPROBE_DEFER)
                                rdev_err(rdev,
                                         "failed to get the current voltage: %pe\n",
                                         ERR_PTR(current_uV));
                        return current_uV;
                }

                /*
                 * If we're below the minimum voltage move up to the
                 * minimum voltage, if we're above the maximum voltage
                 * then move down to the maximum.
                 */
                target_min = current_uV;
                target_max = current_uV;

                if (current_uV < rdev->constraints->min_uV) {
                        target_min = rdev->constraints->min_uV;
                        target_max = rdev->constraints->min_uV;
                }

                if (current_uV > rdev->constraints->max_uV) {
                        target_min = rdev->constraints->max_uV;
                        target_max = rdev->constraints->max_uV;
                }

                if (target_min != current_uV || target_max != current_uV) {
                        rdev_info(rdev, "Bringing %duV into %d-%duV\n",
                                  current_uV, target_min, target_max);
                        ret = _regulator_do_set_voltage(
                                rdev, target_min, target_max);
                        if (ret < 0) {
                                rdev_err(rdev,
                                        "failed to apply %d-%duV constraint: %pe\n",
                                        target_min, target_max, ERR_PTR(ret));
                                return ret;
                        }
                }
        }

        /* constrain machine-level voltage specs to fit
         * the actual range supported by this regulator.
         */
        if (ops->list_voltage && rdev->desc->n_voltages) {
                int        count = rdev->desc->n_voltages;
                int        i;
                int        min_uV = INT_MAX;
                int        max_uV = INT_MIN;
                int        cmin = constraints->min_uV;
                int        cmax = constraints->max_uV;

                /* it's safe to autoconfigure fixed-voltage supplies
                 * and the constraints are used by list_voltage.
                 */
                if (count == 1 && !cmin) {
                        cmin = 1;
                        cmax = INT_MAX;
                        constraints->min_uV = cmin;
                        constraints->max_uV = cmax;
                }

                /* voltage constraints are optional */
                if ((cmin == 0) && (cmax == 0))
                        return 0;

                /* else require explicit machine-level constraints */
                if (cmin <= 0 || cmax <= 0 || cmax < cmin) {
                        rdev_err(rdev, "invalid voltage constraints\n");
                        return -EINVAL;
                }

                /* no need to loop voltages if range is continuous */
                if (rdev->desc->continuous_voltage_range)
                        return 0;

                /* initial: [cmin..cmax] valid, [min_uV..max_uV] not */
                for (i = 0; i < count; i++) {
                        int        value;

                        value = ops->list_voltage(rdev, i);
                        if (value <= 0)
                                continue;

                        /* maybe adjust [min_uV..max_uV] */
                        if (value >= cmin && value < min_uV)
                                min_uV = value;
                        if (value <= cmax && value > max_uV)
                                max_uV = value;
                }

                /* final: [min_uV..max_uV] valid iff constraints valid */
                if (max_uV < min_uV) {
                        rdev_err(rdev,
                                 "unsupportable voltage constraints %u-%uuV\n",
                                 min_uV, max_uV);
                        return -EINVAL;
                }

                /* use regulator's subset of machine constraints */
                if (constraints->min_uV < min_uV) {
                        rdev_dbg(rdev, "override min_uV, %d -> %d\n",
                                 constraints->min_uV, min_uV);
                        constraints->min_uV = min_uV;
                }
                if (constraints->max_uV > max_uV) {
                        rdev_dbg(rdev, "override max_uV, %d -> %d\n",
                                 constraints->max_uV, max_uV);
                        constraints->max_uV = max_uV;
                }
        }

        return 0;
}

static int machine_constraints_current(struct regulator_dev *rdev,
        struct regulation_constraints *constraints)
{
        const struct regulator_ops *ops = rdev->desc->ops;
        int ret;

        if (!constraints->min_uA && !constraints->max_uA)
                return 0;

        if (constraints->min_uA > constraints->max_uA) {
                rdev_err(rdev, "Invalid current constraints\n");
                return -EINVAL;
        }

        if (!ops->set_current_limit || !ops->get_current_limit) {
                rdev_warn(rdev, "Operation of current configuration missing\n");
                return 0;
        }

        /* Set regulator current in constraints range */
        ret = ops->set_current_limit(rdev, constraints->min_uA,
                        constraints->max_uA);
        if (ret < 0) {
                rdev_err(rdev, "Failed to set current constraint, %d\n", ret);
                return ret;
        }

        return 0;
}

static int _regulator_do_enable(struct regulator_dev *rdev);

static int notif_set_limit(struct regulator_dev *rdev,
                           int (*set)(struct regulator_dev *, int, int, bool),
                           int limit, int severity)
{
        bool enable;

        if (limit == REGULATOR_NOTIF_LIMIT_DISABLE) {
                enable = false;
                limit = 0;
        } else {
                enable = true;
        }

        if (limit == REGULATOR_NOTIF_LIMIT_ENABLE)
                limit = 0;

        return set(rdev, limit, severity, enable);
}

static int handle_notify_limits(struct regulator_dev *rdev,
                        int (*set)(struct regulator_dev *, int, int, bool),
                        struct notification_limit *limits)
{
        int ret = 0;

        if (!set)
                return -EOPNOTSUPP;

        if (limits->prot)
                ret = notif_set_limit(rdev, set, limits->prot,
                                      REGULATOR_SEVERITY_PROT);
        if (ret)
                return ret;

        if (limits->err)
                ret = notif_set_limit(rdev, set, limits->err,
                                      REGULATOR_SEVERITY_ERR);
        if (ret)
                return ret;

        if (limits->warn)
                ret = notif_set_limit(rdev, set, limits->warn,
                                      REGULATOR_SEVERITY_WARN);

        return ret;
}
/**
 * set_machine_constraints - sets regulator constraints
 * @rdev: regulator source
 *
 * Allows platform initialisation code to define and constrain
 * regulator circuits e.g. valid voltage/current ranges, etc.  NOTE:
 * Constraints *must* be set by platform code in order for some
 * regulator operations to proceed i.e. set_voltage, set_current_limit,
 * set_mode.
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int set_machine_constraints(struct regulator_dev *rdev)
{
        int ret = 0;
        const struct regulator_ops *ops = rdev->desc->ops;

        ret = machine_constraints_voltage(rdev, rdev->constraints);
        if (ret != 0)
                return ret;

        ret = machine_constraints_current(rdev, rdev->constraints);
        if (ret != 0)
                return ret;

        if (rdev->constraints->ilim_uA && ops->set_input_current_limit) {
                ret = ops->set_input_current_limit(rdev,
                                                   rdev->constraints->ilim_uA);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set input limit: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        /* do we need to setup our suspend state */
        if (rdev->constraints->initial_state) {
                ret = suspend_set_initial_state(rdev);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set suspend state: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        if (rdev->constraints->initial_mode) {
                if (!ops->set_mode) {
                        rdev_err(rdev, "no set_mode operation\n");
                        return -EINVAL;
                }

                ret = ops->set_mode(rdev, rdev->constraints->initial_mode);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set initial mode: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        } else if (rdev->constraints->system_load) {
                /*
                 * We'll only apply the initial system load if an
                 * initial mode wasn't specified.
                 */
                drms_uA_update(rdev);
        }

        if ((rdev->constraints->ramp_delay || rdev->constraints->ramp_disable)
                && ops->set_ramp_delay) {
                ret = ops->set_ramp_delay(rdev, rdev->constraints->ramp_delay);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set ramp_delay: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        if (rdev->constraints->pull_down && ops->set_pull_down) {
                ret = ops->set_pull_down(rdev);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set pull down: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        if (rdev->constraints->soft_start && ops->set_soft_start) {
                ret = ops->set_soft_start(rdev);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set soft start: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        /*
         * Existing logic does not warn if over_current_protection is given as
         * a constraint but driver does not support that. I think we should
         * warn about this type of issues as it is possible someone changes
         * PMIC on board to another type - and the another PMIC's driver does
         * not support setting protection. Board composer may happily believe
         * the DT limits are respected - especially if the new PMIC HW also
         * supports protection but the driver does not. I won't change the logic
         * without hearing more experienced opinion on this though.
         *
         * If warning is seen as a good idea then we can merge handling the
         * over-curret protection and detection and get rid of this special
         * handling.
         */
        if (rdev->constraints->over_current_protection
                && ops->set_over_current_protection) {
                int lim = rdev->constraints->over_curr_limits.prot;

                ret = ops->set_over_current_protection(rdev, lim,
                                                       REGULATOR_SEVERITY_PROT,
                                                       true);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set over current protection: %pe\n",
                                 ERR_PTR(ret));
                        return ret;
                }
        }

        if (rdev->constraints->over_current_detection)
                ret = handle_notify_limits(rdev,
                                           ops->set_over_current_protection,
                                           &rdev->constraints->over_curr_limits);
        if (ret) {
                if (ret != -EOPNOTSUPP) {
                        rdev_err(rdev, "failed to set over current limits: %pe\n",
                                 ERR_PTR(ret));
                        return ret;
                }
                rdev_warn(rdev,
                          "IC does not support requested over-current limits\n");
        }

        if (rdev->constraints->over_voltage_detection)
                ret = handle_notify_limits(rdev,
                                           ops->set_over_voltage_protection,
                                           &rdev->constraints->over_voltage_limits);
        if (ret) {
                if (ret != -EOPNOTSUPP) {
                        rdev_err(rdev, "failed to set over voltage limits %pe\n",
                                 ERR_PTR(ret));
                        return ret;
                }
                rdev_warn(rdev,
                          "IC does not support requested over voltage limits\n");
        }

        if (rdev->constraints->under_voltage_detection)
                ret = handle_notify_limits(rdev,
                                           ops->set_under_voltage_protection,
                                           &rdev->constraints->under_voltage_limits);
        if (ret) {
                if (ret != -EOPNOTSUPP) {
                        rdev_err(rdev, "failed to set under voltage limits %pe\n",
                                 ERR_PTR(ret));
                        return ret;
                }
                rdev_warn(rdev,
                          "IC does not support requested under voltage limits\n");
        }

        if (rdev->constraints->over_temp_detection)
                ret = handle_notify_limits(rdev,
                                           ops->set_thermal_protection,
                                           &rdev->constraints->temp_limits);
        if (ret) {
                if (ret != -EOPNOTSUPP) {
                        rdev_err(rdev, "failed to set temperature limits %pe\n",
                                 ERR_PTR(ret));
                        return ret;
                }
                rdev_warn(rdev,
                          "IC does not support requested temperature limits\n");
        }

        if (rdev->constraints->active_discharge && ops->set_active_discharge) {
                bool ad_state = (rdev->constraints->active_discharge ==
                              REGULATOR_ACTIVE_DISCHARGE_ENABLE) ? true : false;

                ret = ops->set_active_discharge(rdev, ad_state);
                if (ret < 0) {
                        rdev_err(rdev, "failed to set active discharge: %pe\n", ERR_PTR(ret));
                        return ret;
                }
        }

        /*
         * If there is no mechanism for controlling the regulator then
         * flag it as always_on so we don't end up duplicating checks
         * for this so much.  Note that we could control the state of
         * a supply to control the output on a regulator that has no
         * direct control.
         */
        if (!rdev->ena_pin && !ops->enable) {
                if (rdev->supply_name && !rdev->supply)
                        return -EPROBE_DEFER;

                if (rdev->supply)
                        rdev->constraints->always_on =
                                rdev->supply->rdev->constraints->always_on;
                else
                        rdev->constraints->always_on = true;
        }

        /* If the constraints say the regulator should be on at this point
         * and we have control then make sure it is enabled.
         */
        if (rdev->constraints->always_on || rdev->constraints->boot_on) {
                /* If we want to enable this regulator, make sure that we know
                 * the supplying regulator.
                 */
                if (rdev->supply_name && !rdev->supply)
                        return -EPROBE_DEFER;

                /* If supplying regulator has already been enabled,
                 * it's not intended to have use_count increment
                 * when rdev is only boot-on.
                 */
                if (rdev->supply &&
                    (rdev->constraints->always_on ||
                     !regulator_is_enabled(rdev->supply))) {
                        ret = regulator_enable(rdev->supply);
                        if (ret < 0) {
                                _regulator_put(rdev->supply);
                                rdev->supply = NULL;
                                return ret;
                        }
                }

                ret = _regulator_do_enable(rdev);
                if (ret < 0 && ret != -EINVAL) {
                        rdev_err(rdev, "failed to enable: %pe\n", ERR_PTR(ret));
                        return ret;
                }

                if (rdev->constraints->always_on)
                        rdev->use_count++;
        } else if (rdev->desc->off_on_delay) {
                rdev->last_off = ktime_get();
        }

        if (!rdev->constraints->pw_budget_mW)
                rdev->constraints->pw_budget_mW = INT_MAX;

        print_constraints(rdev);
        return 0;
}

/**
 * set_supply - set regulator supply regulator
 * @rdev: regulator (locked)
 * @supply_rdev: supply regulator (locked))
 *
 * Called by platform initialisation code to set the supply regulator for this
 * regulator. This ensures that a regulators supply will also be enabled by the
 * core if it's child is enabled.
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int set_supply(struct regulator_dev *rdev,
                      struct regulator_dev *supply_rdev)
{
        int err;

        rdev_dbg(rdev, "supplied by %s\n", rdev_get_name(supply_rdev));

        if (!try_module_get(supply_rdev->owner))
                return -ENODEV;

        rdev->supply = create_regulator(supply_rdev, &rdev->dev, "SUPPLY");
        if (rdev->supply == NULL) {
                module_put(supply_rdev->owner);
                err = -ENOMEM;
                return err;
        }
        supply_rdev->open_count++;

        return 0;
}

/**
 * set_consumer_device_supply - Bind a regulator to a symbolic supply
 * @rdev:         regulator source
 * @consumer_dev_name: dev_name() string for device supply applies to
 * @supply:       symbolic name for supply
 *
 * Allows platform initialisation code to map physical regulator
 * sources to symbolic names for supplies for use by devices.  Devices
 * should use these symbolic names to request regulators, avoiding the
 * need to provide board-specific regulator names as platform data.
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int set_consumer_device_supply(struct regulator_dev *rdev,
                                      const char *consumer_dev_name,
                                      const char *supply)
{
        struct regulator_map *node, *new_node;
        int has_dev;

        if (supply == NULL)
                return -EINVAL;

        if (consumer_dev_name != NULL)
                has_dev = 1;
        else
                has_dev = 0;

        new_node = kzalloc(sizeof(struct regulator_map), GFP_KERNEL);
        if (new_node == NULL)
                return -ENOMEM;

        new_node->regulator = rdev;
        new_node->supply = supply;

        if (has_dev) {
                new_node->dev_name = kstrdup(consumer_dev_name, GFP_KERNEL);
                if (new_node->dev_name == NULL) {
                        kfree(new_node);
                        return -ENOMEM;
                }
        }

        mutex_lock(&regulator_list_mutex);
        list_for_each_entry(node, &regulator_map_list, list) {
                if (node->dev_name && consumer_dev_name) {
                        if (strcmp(node->dev_name, consumer_dev_name) != 0)
                                continue;
                } else if (node->dev_name || consumer_dev_name) {
                        continue;
                }

                if (strcmp(node->supply, supply) != 0)
                        continue;

                pr_debug("%s: %s/%s is '%s' supply; fail %s/%s\n",
                         consumer_dev_name,
                         dev_name(&node->regulator->dev),
                         node->regulator->desc->name,
                         supply,
                         dev_name(&rdev->dev), rdev_get_name(rdev));
                goto fail;
        }

        list_add(&new_node->list, &regulator_map_list);
        mutex_unlock(&regulator_list_mutex);

        return 0;

fail:
        mutex_unlock(&regulator_list_mutex);
        kfree(new_node->dev_name);
        kfree(new_node);
        return -EBUSY;
}

static void unset_regulator_supplies(struct regulator_dev *rdev)
{
        struct regulator_map *node, *n;

        list_for_each_entry_safe(node, n, &regulator_map_list, list) {
                if (rdev == node->regulator) {
                        list_del(&node->list);
                        kfree(node->dev_name);
                        kfree(node);
                }
        }
}

#ifdef CONFIG_DEBUG_FS
static ssize_t constraint_flags_read_file(struct file *file,
                                          char __user *user_buf,
                                          size_t count, loff_t *ppos)
{
        const struct regulator *regulator = file->private_data;
        const struct regulation_constraints *c = regulator->rdev->constraints;
        char *buf;
        ssize_t ret;

        if (!c)
                return 0;

        buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        ret = snprintf(buf, PAGE_SIZE,
                        "always_on: %u\n"
                        "boot_on: %u\n"
                        "apply_uV: %u\n"
                        "ramp_disable: %u\n"
                        "soft_start: %u\n"
                        "pull_down: %u\n"
                        "over_current_protection: %u\n",
                        c->always_on,
                        c->boot_on,
                        c->apply_uV,
                        c->ramp_disable,
                        c->soft_start,
                        c->pull_down,
                        c->over_current_protection);

        ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
        kfree(buf);

        return ret;
}

#endif

static const struct file_operations constraint_flags_fops = {
#ifdef CONFIG_DEBUG_FS
        .open = simple_open,
        .read = constraint_flags_read_file,
        .llseek = default_llseek,
#endif
};

#define REG_STR_SIZE        64

static struct regulator *create_regulator(struct regulator_dev *rdev,
                                          struct device *dev,
                                          const char *supply_name)
{
        struct regulator *regulator;
        int err = 0;

        lockdep_assert_held_once(&rdev->mutex.base);

        if (dev) {
                char buf[REG_STR_SIZE];
                int size;

                size = snprintf(buf, REG_STR_SIZE, "%s-%s",
                                dev->kobj.name, supply_name);
                if (size >= REG_STR_SIZE)
                        return NULL;

                supply_name = kstrdup(buf, GFP_KERNEL);
                if (supply_name == NULL)
                        return NULL;
        } else {
                supply_name = kstrdup_const(supply_name, GFP_KERNEL);
                if (supply_name == NULL)
                        return NULL;
        }

        regulator = kzalloc(sizeof(*regulator), GFP_KERNEL);
        if (regulator == NULL) {
                kfree_const(supply_name);
                return NULL;
        }

        regulator->rdev = rdev;
        regulator->supply_name = supply_name;

        list_add(&regulator->list, &rdev->consumer_list);

        if (dev) {
                regulator->dev = dev;

                /* Add a link to the device sysfs entry */
                err = sysfs_create_link_nowarn(&rdev->dev.kobj, &dev->kobj,
                                               supply_name);
                if (err) {
                        rdev_dbg(rdev, "could not add device link %s: %pe\n",
                                  dev->kobj.name, ERR_PTR(err));
                        /* non-fatal */
                }
        }

        if (err != -EEXIST) {
                regulator->debugfs = debugfs_create_dir(supply_name, rdev->debugfs);
                if (IS_ERR(regulator->debugfs)) {
                        rdev_dbg(rdev, "Failed to create debugfs directory\n");
                        regulator->debugfs = NULL;
                }
        }

        if (regulator->debugfs) {
                debugfs_create_u32("uA_load", 0444, regulator->debugfs,
                                   &regulator->uA_load);
                debugfs_create_u32("min_uV", 0444, regulator->debugfs,
                                   &regulator->voltage[PM_SUSPEND_ON].min_uV);
                debugfs_create_u32("max_uV", 0444, regulator->debugfs,
                                   &regulator->voltage[PM_SUSPEND_ON].max_uV);
                debugfs_create_file("constraint_flags", 0444, regulator->debugfs,
                                    regulator, &constraint_flags_fops);
        }

        /*
         * Check now if the regulator is an always on regulator - if
         * it is then we don't need to do nearly so much work for
         * enable/disable calls.
         */
        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS) &&
            _regulator_is_enabled(rdev))
                regulator->always_on = true;

        return regulator;
}

static int _regulator_get_enable_time(struct regulator_dev *rdev)
{
        if (rdev->constraints && rdev->constraints->enable_time)
                return rdev->constraints->enable_time;
        if (rdev->desc->ops->enable_time)
                return rdev->desc->ops->enable_time(rdev);
        return rdev->desc->enable_time;
}

static struct regulator_supply_alias *regulator_find_supply_alias(
                struct device *dev, const char *supply)
{
        struct regulator_supply_alias *map;

        list_for_each_entry(map, &regulator_supply_alias_list, list)
                if (map->src_dev == dev && strcmp(map->src_supply, supply) == 0)
                        return map;

        return NULL;
}

static void regulator_supply_alias(struct device **dev, const char **supply)
{
        struct regulator_supply_alias *map;

        map = regulator_find_supply_alias(*dev, *supply);
        if (map) {
                dev_dbg(*dev, "Mapping supply %s to %s,%s\n",
                                *supply, map->alias_supply,
                                dev_name(map->alias_dev));
                *dev = map->alias_dev;
                *supply = map->alias_supply;
        }
}

static int regulator_match(struct device *dev, const void *data)
{
        struct regulator_dev *r = dev_to_rdev(dev);

        return strcmp(rdev_get_name(r), data) == 0;
}

static struct regulator_dev *regulator_lookup_by_name(const char *name)
{
        struct device *dev;

        dev = class_find_device(&regulator_class, NULL, name, regulator_match);

        return dev ? dev_to_rdev(dev) : NULL;
}

static struct regulator_dev *regulator_dt_lookup(struct device *dev,
                                                 const char *supply)
{
        struct regulator_dev *r = NULL;

        if (dev_of_node(dev)) {
                r = of_regulator_dev_lookup(dev, dev_of_node(dev), supply);
                if (PTR_ERR(r) == -ENODEV)
                        r = NULL;
        }

        return r;
}

/**
 * regulator_dev_lookup - lookup a regulator device.
 * @dev: device for regulator "consumer".
 * @supply: Supply name or regulator ID.
 *
 * Return: pointer to &struct regulator_dev or ERR_PTR() encoded negative error number.
 *
 * If successful, returns a struct regulator_dev that corresponds to the name
 * @supply and with the embedded struct device refcount incremented by one.
 * The refcount must be dropped by calling put_device().
 * On failure one of the following ERR_PTR() encoded values is returned:
 * -%ENODEV if lookup fails permanently, -%EPROBE_DEFER if lookup could succeed
 * in the future.
 */
static struct regulator_dev *regulator_dev_lookup(struct device *dev,
                                                  const char *supply)
{
        struct regulator_dev *r = NULL;
        struct regulator_map *map;
        const char *devname = NULL;

        regulator_supply_alias(&dev, &supply);

        /* first do a dt based lookup */
        r = regulator_dt_lookup(dev, supply);
        if (r)
                return r;

        /* if not found, try doing it non-dt way */
        if (dev)
                devname = dev_name(dev);

        mutex_lock(&regulator_list_mutex);
        list_for_each_entry(map, &regulator_map_list, list) {
                /* If the mapping has a device set up it must match */
                if (map->dev_name &&
                    (!devname || strcmp(map->dev_name, devname)))
                        continue;

                if (strcmp(map->supply, supply) == 0 &&
                    get_device(&map->regulator->dev)) {
                        r = map->regulator;
                        break;
                }
        }
        mutex_unlock(&regulator_list_mutex);

        if (r)
                return r;

        r = regulator_lookup_by_name(supply);
        if (r)
                return r;

        return ERR_PTR(-ENODEV);
}

static int regulator_resolve_supply(struct regulator_dev *rdev)
{
        struct regulator_dev *r;
        struct device *dev = rdev->dev.parent;
        struct ww_acquire_ctx ww_ctx;
        int ret = 0;

        /* No supply to resolve? */
        if (!rdev->supply_name)
                return 0;

        /* Supply already resolved? (fast-path without locking contention) */
        if (rdev->supply)
                return 0;

        /* first do a dt based lookup on the node described in the virtual
         * device.
         */
        r = regulator_dt_lookup(&rdev->dev, rdev->supply_name);

        /* If regulator not found use usual search path in the parent
         * device.
         */
        if (!r)
                r = regulator_dev_lookup(dev, rdev->supply_name);

        if (IS_ERR(r)) {
                ret = PTR_ERR(r);

                /* Did the lookup explicitly defer for us? */
                if (ret == -EPROBE_DEFER)
                        goto out;

                if (have_full_constraints()) {
                        r = dummy_regulator_rdev;
                        get_device(&r->dev);
                } else {
                        dev_err(dev, "Failed to resolve %s-supply for %s\n",
                                rdev->supply_name, rdev->desc->name);
                        ret = -EPROBE_DEFER;
                        goto out;
                }
        }

        if (r == rdev) {
                dev_err(dev, "Supply for %s (%s) resolved to itself\n",
                        rdev->desc->name, rdev->supply_name);
                if (!have_full_constraints()) {
                        ret = -EINVAL;
                        goto out;
                }
                r = dummy_regulator_rdev;
                get_device(&r->dev);
        }

        /*
         * If the supply's parent device is not the same as the
         * regulator's parent device, then ensure the parent device
         * is bound before we resolve the supply, in case the parent
         * device get probe deferred and unregisters the supply.
         */
        if (r->dev.parent && r->dev.parent != rdev->dev.parent) {
                if (!device_is_bound(r->dev.parent)) {
                        put_device(&r->dev);
                        ret = -EPROBE_DEFER;
                        goto out;
                }
        }

        /* Recursively resolve the supply of the supply */
        ret = regulator_resolve_supply(r);
        if (ret < 0) {
                put_device(&r->dev);
                goto out;
        }

        /*
         * Recheck rdev->supply with rdev->mutex lock held to avoid a race
         * between rdev->supply null check and setting rdev->supply in
         * set_supply() from concurrent tasks.
         */
        regulator_lock_two(rdev, r, &ww_ctx);

        /* Supply just resolved by a concurrent task? */
        if (rdev->supply) {
                regulator_unlock_two(rdev, r, &ww_ctx);
                put_device(&r->dev);
                goto out;
        }

        ret = set_supply(rdev, r);
        if (ret < 0) {
                regulator_unlock_two(rdev, r, &ww_ctx);
                put_device(&r->dev);
                goto out;
        }

        regulator_unlock_two(rdev, r, &ww_ctx);

        /*
         * In set_machine_constraints() we may have turned this regulator on
         * but we couldn't propagate to the supply if it hadn't been resolved
         * yet.  Do it now.
         */
        if (rdev->use_count) {
                ret = regulator_enable(rdev->supply);
                if (ret < 0) {
                        _regulator_put(rdev->supply);
                        rdev->supply = NULL;
                        goto out;
                }
        }

out:
        return ret;
}

/* common pre-checks for regulator requests */
int _regulator_get_common_check(struct device *dev, const char *id,
                                enum regulator_get_type get_type)
{
        if (get_type >= MAX_GET_TYPE) {
                dev_err(dev, "invalid type %d in %s\n", get_type, __func__);
                return -EINVAL;
        }

        if (id == NULL) {
                dev_err(dev, "regulator request with no identifier\n");
                return -EINVAL;
        }

        return 0;
}

/**
 * _regulator_get_common - Common code for regulator requests
 * @rdev: regulator device pointer as returned by *regulator_dev_lookup()
 *       Its reference count is expected to have been incremented.
 * @dev: device used for dev_printk messages
 * @id: Supply name or regulator ID
 * @get_type: enum regulator_get_type value corresponding to type of request
 *
 * Returns: pointer to struct regulator corresponding to @rdev, or ERR_PTR()
 *            encoded error.
 *
 * This function should be chained with *regulator_dev_lookup() functions.
 */
struct regulator *_regulator_get_common(struct regulator_dev *rdev, struct device *dev,
                                        const char *id, enum regulator_get_type get_type)
{
        struct regulator *regulator;
        struct device_link *link;
        int ret;

        if (IS_ERR(rdev)) {
                ret = PTR_ERR(rdev);

                /*
                 * If regulator_dev_lookup() fails with error other
                 * than -ENODEV our job here is done, we simply return it.
                 */
                if (ret != -ENODEV)
                        return ERR_PTR(ret);

                if (!have_full_constraints()) {
                        dev_warn(dev,
                                 "incomplete constraints, dummy supplies not allowed (id=%s)\n", id);
                        return ERR_PTR(-ENODEV);
                }

                switch (get_type) {
                case NORMAL_GET:
                        /*
                         * Assume that a regulator is physically present and
                         * enabled, even if it isn't hooked up, and just
                         * provide a dummy.
                         */
                        dev_warn(dev, "supply %s not found, using dummy regulator\n", id);
                        rdev = dummy_regulator_rdev;
                        get_device(&rdev->dev);
                        break;

                case EXCLUSIVE_GET:
                        dev_warn(dev,
                                 "dummy supplies not allowed for exclusive requests (id=%s)\n", id);
                        fallthrough;

                default:
                        return ERR_PTR(-ENODEV);
                }
        }

        if (rdev->exclusive) {
                regulator = ERR_PTR(-EPERM);
                put_device(&rdev->dev);
                return regulator;
        }

        if (get_type == EXCLUSIVE_GET && rdev->open_count) {
                regulator = ERR_PTR(-EBUSY);
                put_device(&rdev->dev);
                return regulator;
        }

        mutex_lock(&regulator_list_mutex);
        ret = (rdev->coupling_desc.n_resolved != rdev->coupling_desc.n_coupled);
        mutex_unlock(&regulator_list_mutex);

        if (ret != 0) {
                regulator = ERR_PTR(-EPROBE_DEFER);
                put_device(&rdev->dev);
                return regulator;
        }

        ret = regulator_resolve_supply(rdev);
        if (ret < 0) {
                regulator = ERR_PTR(ret);
                put_device(&rdev->dev);
                return regulator;
        }

        if (!try_module_get(rdev->owner)) {
                regulator = ERR_PTR(-EPROBE_DEFER);
                put_device(&rdev->dev);
                return regulator;
        }

        regulator_lock(rdev);
        regulator = create_regulator(rdev, dev, id);
        regulator_unlock(rdev);
        if (regulator == NULL) {
                regulator = ERR_PTR(-ENOMEM);
                module_put(rdev->owner);
                put_device(&rdev->dev);
                return regulator;
        }

        rdev->open_count++;
        if (get_type == EXCLUSIVE_GET) {
                rdev->exclusive = 1;

                ret = _regulator_is_enabled(rdev);
                if (ret > 0) {
                        rdev->use_count = 1;
                        regulator->enable_count = 1;

                        /* Propagate the regulator state to its supply */
                        if (rdev->supply) {
                                ret = regulator_enable(rdev->supply);
                                if (ret < 0) {
                                        destroy_regulator(regulator);
                                        module_put(rdev->owner);
                                        put_device(&rdev->dev);
                                        return ERR_PTR(ret);
                                }
                        }
                } else {
                        rdev->use_count = 0;
                        regulator->enable_count = 0;
                }
        }

        link = device_link_add(dev, &rdev->dev, DL_FLAG_STATELESS);
        if (!IS_ERR_OR_NULL(link))
                regulator->device_link = true;

        return regulator;
}

/* Internal regulator request function */
struct regulator *_regulator_get(struct device *dev, const char *id,
                                 enum regulator_get_type get_type)
{
        struct regulator_dev *rdev;
        int ret;

        ret = _regulator_get_common_check(dev, id, get_type);
        if (ret)
                return ERR_PTR(ret);

        rdev = regulator_dev_lookup(dev, id);
        return _regulator_get_common(rdev, dev, id, get_type);
}

/**
 * regulator_get - lookup and obtain a reference to a regulator.
 * @dev: device for regulator "consumer"
 * @id: Supply name or regulator ID.
 *
 * Use of supply names configured via set_consumer_device_supply() is
 * strongly encouraged.  It is recommended that the supply name used
 * should match the name used for the supply and/or the relevant
 * device pins in the datasheet.
 *
 * Return: Pointer to a &struct regulator corresponding to the regulator
 *           producer, or an ERR_PTR() encoded negative error number.
 */
struct regulator *regulator_get(struct device *dev, const char *id)
{
        return _regulator_get(dev, id, NORMAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_get);

/**
 * regulator_get_exclusive - obtain exclusive access to a regulator.
 * @dev: device for regulator "consumer"
 * @id: Supply name or regulator ID.
 *
 * Other consumers will be unable to obtain this regulator while this
 * reference is held and the use count for the regulator will be
 * initialised to reflect the current state of the regulator.
 *
 * This is intended for use by consumers which cannot tolerate shared
 * use of the regulator such as those which need to force the
 * regulator off for correct operation of the hardware they are
 * controlling.
 *
 * Use of supply names configured via set_consumer_device_supply() is
 * strongly encouraged.  It is recommended that the supply name used
 * should match the name used for the supply and/or the relevant
 * device pins in the datasheet.
 *
 * Return: Pointer to a &struct regulator corresponding to the regulator
 *           producer, or an ERR_PTR() encoded negative error number.
 */
struct regulator *regulator_get_exclusive(struct device *dev, const char *id)
{
        return _regulator_get(dev, id, EXCLUSIVE_GET);
}
EXPORT_SYMBOL_GPL(regulator_get_exclusive);

/**
 * regulator_get_optional - obtain optional access to a regulator.
 * @dev: device for regulator "consumer"
 * @id: Supply name or regulator ID.
 *
 * This is intended for use by consumers for devices which can have
 * some supplies unconnected in normal use, such as some MMC devices.
 * It can allow the regulator core to provide stub supplies for other
 * supplies requested using normal regulator_get() calls without
 * disrupting the operation of drivers that can handle absent
 * supplies.
 *
 * Use of supply names configured via set_consumer_device_supply() is
 * strongly encouraged.  It is recommended that the supply name used
 * should match the name used for the supply and/or the relevant
 * device pins in the datasheet.
 *
 * Return: Pointer to a &struct regulator corresponding to the regulator
 *           producer, or an ERR_PTR() encoded negative error number.
 */
struct regulator *regulator_get_optional(struct device *dev, const char *id)
{
        return _regulator_get(dev, id, OPTIONAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_get_optional);

static void destroy_regulator(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;

        debugfs_remove_recursive(regulator->debugfs);

        if (regulator->dev) {
                if (regulator->device_link)
                        device_link_remove(regulator->dev, &rdev->dev);

                /* remove any sysfs entries */
                sysfs_remove_link(&rdev->dev.kobj, regulator->supply_name);
        }

        regulator_lock(rdev);
        list_del(&regulator->list);

        rdev->open_count--;
        rdev->exclusive = 0;
        regulator_unlock(rdev);

        kfree_const(regulator->supply_name);
        kfree(regulator);
}

/* regulator_list_mutex lock held by regulator_put() */
static void _regulator_put(struct regulator *regulator)
{
        struct regulator_dev *rdev;

        if (IS_ERR_OR_NULL(regulator))
                return;

        lockdep_assert_held_once(&regulator_list_mutex);

        /* Docs say you must disable before calling regulator_put() */
        WARN_ON(regulator->enable_count);

        rdev = regulator->rdev;

        destroy_regulator(regulator);

        module_put(rdev->owner);
        put_device(&rdev->dev);
}

/**
 * regulator_put - "free" the regulator source
 * @regulator: regulator source
 *
 * Note: drivers must ensure that all regulator_enable calls made on this
 * regulator source are balanced by regulator_disable calls prior to calling
 * this function.
 */
void regulator_put(struct regulator *regulator)
{
        mutex_lock(&regulator_list_mutex);
        _regulator_put(regulator);
        mutex_unlock(&regulator_list_mutex);
}
EXPORT_SYMBOL_GPL(regulator_put);

/**
 * regulator_register_supply_alias - Provide device alias for supply lookup
 *
 * @dev: device that will be given as the regulator "consumer"
 * @id: Supply name or regulator ID
 * @alias_dev: device that should be used to lookup the supply
 * @alias_id: Supply name or regulator ID that should be used to lookup the
 * supply
 *
 * All lookups for id on dev will instead be conducted for alias_id on
 * alias_dev.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_register_supply_alias(struct device *dev, const char *id,
                                    struct device *alias_dev,
                                    const char *alias_id)
{
        struct regulator_supply_alias *map;

        map = regulator_find_supply_alias(dev, id);
        if (map)
                return -EEXIST;

        map = kzalloc(sizeof(struct regulator_supply_alias), GFP_KERNEL);
        if (!map)
                return -ENOMEM;

        map->src_dev = dev;
        map->src_supply = id;
        map->alias_dev = alias_dev;
        map->alias_supply = alias_id;

        list_add(&map->list, &regulator_supply_alias_list);

        pr_info("Adding alias for supply %s,%s -> %s,%s\n",
                id, dev_name(dev), alias_id, dev_name(alias_dev));

        return 0;
}
EXPORT_SYMBOL_GPL(regulator_register_supply_alias);

/**
 * regulator_unregister_supply_alias - Remove device alias
 *
 * @dev: device that will be given as the regulator "consumer"
 * @id: Supply name or regulator ID
 *
 * Remove a lookup alias if one exists for id on dev.
 */
void regulator_unregister_supply_alias(struct device *dev, const char *id)
{
        struct regulator_supply_alias *map;

        map = regulator_find_supply_alias(dev, id);
        if (map) {
                list_del(&map->list);
                kfree(map);
        }
}
EXPORT_SYMBOL_GPL(regulator_unregister_supply_alias);

/**
 * regulator_bulk_register_supply_alias - register multiple aliases
 *
 * @dev: device that will be given as the regulator "consumer"
 * @id: List of supply names or regulator IDs
 * @alias_dev: device that should be used to lookup the supply
 * @alias_id: List of supply names or regulator IDs that should be used to
 * lookup the supply
 * @num_id: Number of aliases to register
 *
 * This helper function allows drivers to register several supply
 * aliases in one operation.  If any of the aliases cannot be
 * registered any aliases that were registered will be removed
 * before returning to the caller.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_bulk_register_supply_alias(struct device *dev,
                                         const char *const *id,
                                         struct device *alias_dev,
                                         const char *const *alias_id,
                                         int num_id)
{
        int i;
        int ret;

        for (i = 0; i < num_id; ++i) {
                ret = regulator_register_supply_alias(dev, id[i], alias_dev,
                                                      alias_id[i]);
                if (ret < 0)
                        goto err;
        }

        return 0;

err:
        dev_err(dev,
                "Failed to create supply alias %s,%s -> %s,%s\n",
                id[i], dev_name(dev), alias_id[i], dev_name(alias_dev));

        while (--i >= 0)
                regulator_unregister_supply_alias(dev, id[i]);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_register_supply_alias);

/**
 * regulator_bulk_unregister_supply_alias - unregister multiple aliases
 *
 * @dev: device that will be given as the regulator "consumer"
 * @id: List of supply names or regulator IDs
 * @num_id: Number of aliases to unregister
 *
 * This helper function allows drivers to unregister several supply
 * aliases in one operation.
 */
void regulator_bulk_unregister_supply_alias(struct device *dev,
                                            const char *const *id,
                                            int num_id)
{
        int i;

        for (i = 0; i < num_id; ++i)
                regulator_unregister_supply_alias(dev, id[i]);
}
EXPORT_SYMBOL_GPL(regulator_bulk_unregister_supply_alias);


/* Manage enable GPIO list. Same GPIO pin can be shared among regulators */
static int regulator_ena_gpio_request(struct regulator_dev *rdev,
                                const struct regulator_config *config)
{
        struct regulator_enable_gpio *pin, *new_pin;
        struct gpio_desc *gpiod;

        gpiod = config->ena_gpiod;
        new_pin = kzalloc(sizeof(*new_pin), GFP_KERNEL);

        mutex_lock(&regulator_list_mutex);

        list_for_each_entry(pin, &regulator_ena_gpio_list, list) {
                if (pin->gpiod == gpiod) {
                        rdev_dbg(rdev, "GPIO is already used\n");
                        goto update_ena_gpio_to_rdev;
                }
        }

        if (new_pin == NULL) {
                mutex_unlock(&regulator_list_mutex);
                return -ENOMEM;
        }

        pin = new_pin;
        new_pin = NULL;

        pin->gpiod = gpiod;
        list_add(&pin->list, &regulator_ena_gpio_list);

update_ena_gpio_to_rdev:
        pin->request_count++;
        rdev->ena_pin = pin;

        mutex_unlock(&regulator_list_mutex);
        kfree(new_pin);

        return 0;
}

static void regulator_ena_gpio_free(struct regulator_dev *rdev)
{
        struct regulator_enable_gpio *pin, *n;

        if (!rdev->ena_pin)
                return;

        /* Free the GPIO only in case of no use */
        list_for_each_entry_safe(pin, n, &regulator_ena_gpio_list, list) {
                if (pin != rdev->ena_pin)
                        continue;

                if (--pin->request_count)
                        break;

                gpiod_put(pin->gpiod);
                list_del(&pin->list);
                kfree(pin);
                break;
        }

        rdev->ena_pin = NULL;
}

/**
 * regulator_ena_gpio_ctrl - balance enable_count of each GPIO and actual GPIO pin control
 * @rdev: regulator_dev structure
 * @enable: enable GPIO at initial use?
 *
 * GPIO is enabled in case of initial use. (enable_count is 0)
 * GPIO is disabled when it is not shared any more. (enable_count <= 1)
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int regulator_ena_gpio_ctrl(struct regulator_dev *rdev, bool enable)
{
        struct regulator_enable_gpio *pin = rdev->ena_pin;

        if (!pin)
                return -EINVAL;

        if (enable) {
                /* Enable GPIO at initial use */
                if (pin->enable_count == 0)
                        gpiod_set_value_cansleep(pin->gpiod, 1);

                pin->enable_count++;
        } else {
                if (pin->enable_count > 1) {
                        pin->enable_count--;
                        return 0;
                }

                /* Disable GPIO if not used */
                if (pin->enable_count <= 1) {
                        gpiod_set_value_cansleep(pin->gpiod, 0);
                        pin->enable_count = 0;
                }
        }

        return 0;
}

/**
 * _regulator_check_status_enabled - check if regulator status can be
 *                                     interpreted as "regulator is enabled"
 * @rdev: the regulator device to check
 *
 * Return:
 * * 1                        - if status shows regulator is in enabled state
 * * 0                        - if not enabled state
 * * Error Value        - as received from ops->get_status()
 */
static inline int _regulator_check_status_enabled(struct regulator_dev *rdev)
{
        int ret = rdev->desc->ops->get_status(rdev);

        if (ret < 0) {
                rdev_info(rdev, "get_status returned error: %d\n", ret);
                return ret;
        }

        switch (ret) {
        case REGULATOR_STATUS_OFF:
        case REGULATOR_STATUS_ERROR:
        case REGULATOR_STATUS_UNDEFINED:
                return 0;
        default:
                return 1;
        }
}

static int _regulator_do_enable(struct regulator_dev *rdev)
{
        int ret, delay;

        /* Query before enabling in case configuration dependent.  */
        ret = _regulator_get_enable_time(rdev);
        if (ret >= 0) {
                delay = ret;
        } else {
                rdev_warn(rdev, "enable_time() failed: %pe\n", ERR_PTR(ret));
                delay = 0;
        }

        trace_regulator_enable(rdev_get_name(rdev));

        if (rdev->desc->off_on_delay) {
                /* if needed, keep a distance of off_on_delay from last time
                 * this regulator was disabled.
                 */
                ktime_t end = ktime_add_us(rdev->last_off, rdev->desc->off_on_delay);
                s64 remaining = ktime_us_delta(end, ktime_get_boottime());

                if (remaining > 0)
                        fsleep(remaining);
        }

        if (rdev->ena_pin) {
                if (!rdev->ena_gpio_state) {
                        ret = regulator_ena_gpio_ctrl(rdev, true);
                        if (ret < 0)
                                return ret;
                        rdev->ena_gpio_state = 1;
                }
        } else if (rdev->desc->ops->enable) {
                ret = rdev->desc->ops->enable(rdev);
                if (ret < 0)
                        return ret;
        } else {
                return -EINVAL;
        }

        /* Allow the regulator to ramp; it would be useful to extend
         * this for bulk operations so that the regulators can ramp
         * together.
         */
        trace_regulator_enable_delay(rdev_get_name(rdev));

        /* If poll_enabled_time is set, poll upto the delay calculated
         * above, delaying poll_enabled_time uS to check if the regulator
         * actually got enabled.
         * If the regulator isn't enabled after our delay helper has expired,
         * return -ETIMEDOUT.
         */
        if (rdev->desc->poll_enabled_time) {
                int time_remaining = delay;

                while (time_remaining > 0) {
                        fsleep(rdev->desc->poll_enabled_time);

                        if (rdev->desc->ops->get_status) {
                                ret = _regulator_check_status_enabled(rdev);
                                if (ret < 0)
                                        return ret;
                                else if (ret)
                                        break;
                        } else if (rdev->desc->ops->is_enabled(rdev))
                                break;

                        time_remaining -= rdev->desc->poll_enabled_time;
                }

                if (time_remaining <= 0) {
                        rdev_err(rdev, "Enabled check timed out\n");
                        return -ETIMEDOUT;
                }
        } else {
                fsleep(delay);
        }

        trace_regulator_enable_complete(rdev_get_name(rdev));

        return 0;
}

/**
 * _regulator_handle_consumer_enable - handle that a consumer enabled
 * @regulator: regulator source
 *
 * Some things on a regulator consumer (like the contribution towards total
 * load on the regulator) only have an effect when the consumer wants the
 * regulator enabled.  Explained in example with two consumers of the same
 * regulator:
 *   consumer A: set_load(100);       => total load = 0
 *   consumer A: regulator_enable();  => total load = 100
 *   consumer B: set_load(1000);      => total load = 100
 *   consumer B: regulator_enable();  => total load = 1100
 *   consumer A: regulator_disable(); => total_load = 1000
 *
 * This function (together with _regulator_handle_consumer_disable) is
 * responsible for keeping track of the refcount for a given regulator consumer
 * and applying / unapplying these things.
 *
 * Return: 0 on success or negative error number on failure.
 */
static int _regulator_handle_consumer_enable(struct regulator *regulator)
{
        int ret;
        struct regulator_dev *rdev = regulator->rdev;

        lockdep_assert_held_once(&rdev->mutex.base);

        regulator->enable_count++;
        if (regulator->uA_load && regulator->enable_count == 1) {
                ret = drms_uA_update(rdev);
                if (ret)
                        regulator->enable_count--;
                return ret;
        }

        return 0;
}

/**
 * _regulator_handle_consumer_disable - handle that a consumer disabled
 * @regulator: regulator source
 *
 * The opposite of _regulator_handle_consumer_enable().
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int _regulator_handle_consumer_disable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;

        lockdep_assert_held_once(&rdev->mutex.base);

        if (!regulator->enable_count) {
                rdev_err(rdev, "Underflow of regulator enable count\n");
                return -EINVAL;
        }

        regulator->enable_count--;
        if (regulator->uA_load && regulator->enable_count == 0)
                return drms_uA_update(rdev);

        return 0;
}

/* locks held by regulator_enable() */
static int _regulator_enable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        int ret;

        lockdep_assert_held_once(&rdev->mutex.base);

        if (rdev->use_count == 0 && rdev->supply) {
                ret = _regulator_enable(rdev->supply);
                if (ret < 0)
                        return ret;
        }

        /* balance only if there are regulators coupled */
        if (rdev->coupling_desc.n_coupled > 1) {
                ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
                if (ret < 0)
                        goto err_disable_supply;
        }

        ret = _regulator_handle_consumer_enable(regulator);
        if (ret < 0)
                goto err_disable_supply;

        if (rdev->use_count == 0) {
                /*
                 * The regulator may already be enabled if it's not switchable
                 * or was left on
                 */
                ret = _regulator_is_enabled(rdev);
                if (ret == -EINVAL || ret == 0) {
                        if (!regulator_ops_is_valid(rdev,
                                        REGULATOR_CHANGE_STATUS)) {
                                ret = -EPERM;
                                goto err_consumer_disable;
                        }

                        ret = _regulator_do_enable(rdev);
                        if (ret < 0)
                                goto err_consumer_disable;

                        _notifier_call_chain(rdev, REGULATOR_EVENT_ENABLE,
                                             NULL);
                } else if (ret < 0) {
                        rdev_err(rdev, "is_enabled() failed: %pe\n", ERR_PTR(ret));
                        goto err_consumer_disable;
                }
                /* Fallthrough on positive return values - already enabled */
        }

        if (regulator->enable_count == 1)
                rdev->use_count++;

        return 0;

err_consumer_disable:
        _regulator_handle_consumer_disable(regulator);

err_disable_supply:
        if (rdev->use_count == 0 && rdev->supply)
                _regulator_disable(rdev->supply);

        return ret;
}

/**
 * regulator_enable - enable regulator output
 * @regulator: regulator source
 *
 * Request that the regulator be enabled with the regulator output at
 * the predefined voltage or current value.  Calls to regulator_enable()
 * must be balanced with calls to regulator_disable().
 *
 * NOTE: the output value can be set by other drivers, boot loader or may be
 * hardwired in the regulator.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_enable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct ww_acquire_ctx ww_ctx;
        int ret;

        regulator_lock_dependent(rdev, &ww_ctx);
        ret = _regulator_enable(regulator);
        regulator_unlock_dependent(rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_enable);

static int _regulator_do_disable(struct regulator_dev *rdev)
{
        int ret;

        trace_regulator_disable(rdev_get_name(rdev));

        if (rdev->ena_pin) {
                if (rdev->ena_gpio_state) {
                        ret = regulator_ena_gpio_ctrl(rdev, false);
                        if (ret < 0)
                                return ret;
                        rdev->ena_gpio_state = 0;
                }

        } else if (rdev->desc->ops->disable) {
                ret = rdev->desc->ops->disable(rdev);
                if (ret != 0)
                        return ret;
        }

        if (rdev->desc->off_on_delay)
                rdev->last_off = ktime_get_boottime();

        trace_regulator_disable_complete(rdev_get_name(rdev));

        return 0;
}

/* locks held by regulator_disable() */
static int _regulator_disable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        int ret = 0;

        lockdep_assert_held_once(&rdev->mutex.base);

        if (WARN(regulator->enable_count == 0,
                 "unbalanced disables for %s\n", rdev_get_name(rdev)))
                return -EIO;

        if (regulator->enable_count == 1) {
        /* disabling last enable_count from this regulator */
                /* are we the last user and permitted to disable ? */
                if (rdev->use_count == 1 &&
                    (rdev->constraints && !rdev->constraints->always_on)) {

                        /* we are last user */
                        if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) {
                                ret = _notifier_call_chain(rdev,
                                                           REGULATOR_EVENT_PRE_DISABLE,
                                                           NULL);
                                if (ret & NOTIFY_STOP_MASK)
                                        return -EINVAL;

                                ret = _regulator_do_disable(rdev);
                                if (ret < 0) {
                                        rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret));
                                        _notifier_call_chain(rdev,
                                                        REGULATOR_EVENT_ABORT_DISABLE,
                                                        NULL);
                                        return ret;
                                }
                                _notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE,
                                                NULL);
                        }

                        rdev->use_count = 0;
                } else if (rdev->use_count > 1) {
                        rdev->use_count--;
                }
        }

        if (ret == 0)
                ret = _regulator_handle_consumer_disable(regulator);

        if (ret == 0 && rdev->coupling_desc.n_coupled > 1)
                ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);

        if (ret == 0 && rdev->use_count == 0 && rdev->supply)
                ret = _regulator_disable(rdev->supply);

        return ret;
}

/**
 * regulator_disable - disable regulator output
 * @regulator: regulator source
 *
 * Disable the regulator output voltage or current.  Calls to
 * regulator_enable() must be balanced with calls to
 * regulator_disable().
 *
 * NOTE: this will only disable the regulator output if no other consumer
 * devices have it enabled, the regulator device supports disabling and
 * machine constraints permit this operation.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_disable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct ww_acquire_ctx ww_ctx;
        int ret;

        regulator_lock_dependent(rdev, &ww_ctx);
        ret = _regulator_disable(regulator);
        regulator_unlock_dependent(rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_disable);

/* locks held by regulator_force_disable() */
static int _regulator_force_disable(struct regulator_dev *rdev)
{
        int ret = 0;

        lockdep_assert_held_once(&rdev->mutex.base);

        ret = _notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
                        REGULATOR_EVENT_PRE_DISABLE, NULL);
        if (ret & NOTIFY_STOP_MASK)
                return -EINVAL;

        ret = _regulator_do_disable(rdev);
        if (ret < 0) {
                rdev_err(rdev, "failed to force disable: %pe\n", ERR_PTR(ret));
                _notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
                                REGULATOR_EVENT_ABORT_DISABLE, NULL);
                return ret;
        }

        _notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
                        REGULATOR_EVENT_DISABLE, NULL);

        return 0;
}

/**
 * regulator_force_disable - force disable regulator output
 * @regulator: regulator source
 *
 * Forcibly disable the regulator output voltage or current.
 * NOTE: this *will* disable the regulator output even if other consumer
 * devices have it enabled. This should be used for situations when device
 * damage will likely occur if the regulator is not disabled (e.g. over temp).
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_force_disable(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct ww_acquire_ctx ww_ctx;
        int ret;

        regulator_lock_dependent(rdev, &ww_ctx);

        ret = _regulator_force_disable(regulator->rdev);

        if (rdev->coupling_desc.n_coupled > 1)
                regulator_balance_voltage(rdev, PM_SUSPEND_ON);

        if (regulator->uA_load) {
                regulator->uA_load = 0;
                ret = drms_uA_update(rdev);
        }

        if (rdev->use_count != 0 && rdev->supply)
                _regulator_disable(rdev->supply);

        regulator_unlock_dependent(rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_force_disable);

static void regulator_disable_work(struct work_struct *work)
{
        struct regulator_dev *rdev = container_of(work, struct regulator_dev,
                                                  disable_work.work);
        struct ww_acquire_ctx ww_ctx;
        int count, i, ret;
        struct regulator *regulator;
        int total_count = 0;

        regulator_lock_dependent(rdev, &ww_ctx);

        /*
         * Workqueue functions queue the new work instance while the previous
         * work instance is being processed. Cancel the queued work instance
         * as the work instance under processing does the job of the queued
         * work instance.
         */
        cancel_delayed_work(&rdev->disable_work);

        list_for_each_entry(regulator, &rdev->consumer_list, list) {
                count = regulator->deferred_disables;

                if (!count)
                        continue;

                total_count += count;
                regulator->deferred_disables = 0;

                for (i = 0; i < count; i++) {
                        ret = _regulator_disable(regulator);
                        if (ret != 0)
                                rdev_err(rdev, "Deferred disable failed: %pe\n",
                                         ERR_PTR(ret));
                }
        }
        WARN_ON(!total_count);

        if (rdev->coupling_desc.n_coupled > 1)
                regulator_balance_voltage(rdev, PM_SUSPEND_ON);

        regulator_unlock_dependent(rdev, &ww_ctx);
}

/**
 * regulator_disable_deferred - disable regulator output with delay
 * @regulator: regulator source
 * @ms: milliseconds until the regulator is disabled
 *
 * Execute regulator_disable() on the regulator after a delay.  This
 * is intended for use with devices that require some time to quiesce.
 *
 * NOTE: this will only disable the regulator output if no other consumer
 * devices have it enabled, the regulator device supports disabling and
 * machine constraints permit this operation.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_disable_deferred(struct regulator *regulator, int ms)
{
        struct regulator_dev *rdev = regulator->rdev;

        if (!ms)
                return regulator_disable(regulator);

        regulator_lock(rdev);
        regulator->deferred_disables++;
        mod_delayed_work(system_power_efficient_wq, &rdev->disable_work,
                         msecs_to_jiffies(ms));
        regulator_unlock(rdev);

        return 0;
}
EXPORT_SYMBOL_GPL(regulator_disable_deferred);

static int _regulator_is_enabled(struct regulator_dev *rdev)
{
        /* A GPIO control always takes precedence */
        if (rdev->ena_pin)
                return rdev->ena_gpio_state;

        /* If we don't know then assume that the regulator is always on */
        if (!rdev->desc->ops->is_enabled)
                return 1;

        return rdev->desc->ops->is_enabled(rdev);
}

static int _regulator_list_voltage(struct regulator_dev *rdev,
                                   unsigned selector, int lock)
{
        const struct regulator_ops *ops = rdev->desc->ops;
        int ret;

        if (rdev->desc->fixed_uV && rdev->desc->n_voltages == 1 && !selector)
                return rdev->desc->fixed_uV;

        if (ops->list_voltage) {
                if (selector >= rdev->desc->n_voltages)
                        return -EINVAL;
                if (selector < rdev->desc->linear_min_sel)
                        return 0;
                if (lock)
                        regulator_lock(rdev);
                ret = ops->list_voltage(rdev, selector);
                if (lock)
                        regulator_unlock(rdev);
        } else if (rdev->is_switch && rdev->supply) {
                ret = _regulator_list_voltage(rdev->supply->rdev,
                                              selector, lock);
        } else {
                return -EINVAL;
        }

        if (ret > 0) {
                if (ret < rdev->constraints->min_uV)
                        ret = 0;
                else if (ret > rdev->constraints->max_uV)
                        ret = 0;
        }

        return ret;
}

/**
 * regulator_is_enabled - is the regulator output enabled
 * @regulator: regulator source
 *
 * Note that the device backing this regulator handle can have multiple
 * users, so it might be enabled even if regulator_enable() was never
 * called for this particular source.
 *
 * Return: Positive if the regulator driver backing the source/client
 *           has requested that the device be enabled, zero if it hasn't,
 *           else a negative error number.
 */
int regulator_is_enabled(struct regulator *regulator)
{
        int ret;

        if (regulator->always_on)
                return 1;

        regulator_lock(regulator->rdev);
        ret = _regulator_is_enabled(regulator->rdev);
        regulator_unlock(regulator->rdev);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_is_enabled);

/**
 * regulator_count_voltages - count regulator_list_voltage() selectors
 * @regulator: regulator source
 *
 * Return: Number of selectors for @regulator, or negative error number.
 *
 * Selectors are numbered starting at zero, and typically correspond to
 * bitfields in hardware registers.
 */
int regulator_count_voltages(struct regulator *regulator)
{
        struct regulator_dev        *rdev = regulator->rdev;

        if (rdev->desc->n_voltages)
                return rdev->desc->n_voltages;

        if (!rdev->is_switch || !rdev->supply)
                return -EINVAL;

        return regulator_count_voltages(rdev->supply);
}
EXPORT_SYMBOL_GPL(regulator_count_voltages);

/**
 * regulator_list_voltage - enumerate supported voltages
 * @regulator: regulator source
 * @selector: identify voltage to list
 * Context: can sleep
 *
 * Return: Voltage for @selector that can be passed to regulator_set_voltage(),
 *           0 if @selector can't be used on this system, or a negative error
 *           number on failure.
 */
int regulator_list_voltage(struct regulator *regulator, unsigned selector)
{
        return _regulator_list_voltage(regulator->rdev, selector, 1);
}
EXPORT_SYMBOL_GPL(regulator_list_voltage);

/**
 * regulator_get_regmap - get the regulator's register map
 * @regulator: regulator source
 *
 * Return: Pointer to the &struct regmap for @regulator, or ERR_PTR()
 *           encoded -%EOPNOTSUPP if @regulator doesn't use regmap.
 */
struct regmap *regulator_get_regmap(struct regulator *regulator)
{
        struct regmap *map = regulator->rdev->regmap;

        return map ? map : ERR_PTR(-EOPNOTSUPP);
}
EXPORT_SYMBOL_GPL(regulator_get_regmap);

/**
 * regulator_get_hardware_vsel_register - get the HW voltage selector register
 * @regulator: regulator source
 * @vsel_reg: voltage selector register, output parameter
 * @vsel_mask: mask for voltage selector bitfield, output parameter
 *
 * Returns the hardware register offset and bitmask used for setting the
 * regulator voltage. This might be useful when configuring voltage-scaling
 * hardware or firmware that can make I2C requests behind the kernel's back,
 * for example.
 *
 * Return: 0 on success, or -%EOPNOTSUPP if the regulator does not support
 *         voltage selectors.
 *
 * On success, the output parameters @vsel_reg and @vsel_mask are filled in
 * and 0 is returned, otherwise a negative error number is returned.
 */
int regulator_get_hardware_vsel_register(struct regulator *regulator,
                                         unsigned *vsel_reg,
                                         unsigned *vsel_mask)
{
        struct regulator_dev *rdev = regulator->rdev;
        const struct regulator_ops *ops = rdev->desc->ops;

        if (ops->set_voltage_sel != regulator_set_voltage_sel_regmap)
                return -EOPNOTSUPP;

        *vsel_reg = rdev->desc->vsel_reg;
        *vsel_mask = rdev->desc->vsel_mask;

        return 0;
}
EXPORT_SYMBOL_GPL(regulator_get_hardware_vsel_register);

/**
 * regulator_list_hardware_vsel - get the HW-specific register value for a selector
 * @regulator: regulator source
 * @selector: identify voltage to list
 *
 * Converts the selector to a hardware-specific voltage selector that can be
 * directly written to the regulator registers. The address of the voltage
 * register can be determined by calling @regulator_get_hardware_vsel_register.
 *
 * Return: 0 on success, -%EINVAL if the selector is outside the supported
 *           range, or -%EOPNOTSUPP if the regulator does not support voltage
 *           selectors.
 */
int regulator_list_hardware_vsel(struct regulator *regulator,
                                 unsigned selector)
{
        struct regulator_dev *rdev = regulator->rdev;
        const struct regulator_ops *ops = rdev->desc->ops;

        if (selector >= rdev->desc->n_voltages)
                return -EINVAL;
        if (selector < rdev->desc->linear_min_sel)
                return 0;
        if (ops->set_voltage_sel != regulator_set_voltage_sel_regmap)
                return -EOPNOTSUPP;

        return selector;
}
EXPORT_SYMBOL_GPL(regulator_list_hardware_vsel);

/**
 * regulator_hardware_enable - access the HW for enable/disable regulator
 * @regulator: regulator source
 * @enable: true for enable, false for disable
 *
 * Request that the regulator be enabled/disabled with the regulator output at
 * the predefined voltage or current value.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_hardware_enable(struct regulator *regulator, bool enable)
{
        struct regulator_dev *rdev = regulator->rdev;
        const struct regulator_ops *ops = rdev->desc->ops;
        int ret = -EOPNOTSUPP;

        if (!rdev->exclusive || !ops || !ops->enable || !ops->disable)
                return ret;

        if (enable)
                ret = ops->enable(rdev);
        else
                ret = ops->disable(rdev);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_hardware_enable);

/**
 * regulator_get_linear_step - return the voltage step size between VSEL values
 * @regulator: regulator source
 *
 * Return: The voltage step size between VSEL values for linear regulators,
 *           or 0 if the regulator isn't a linear regulator.
 */
unsigned int regulator_get_linear_step(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;

        return rdev->desc->uV_step;
}
EXPORT_SYMBOL_GPL(regulator_get_linear_step);

/**
 * regulator_is_supported_voltage - check if a voltage range can be supported
 *
 * @regulator: Regulator to check.
 * @min_uV: Minimum required voltage in uV.
 * @max_uV: Maximum required voltage in uV.
 *
 * Return: 1 if the voltage range is supported, 0 if not, or a negative error
 *           number if @regulator's voltage can't be changed and voltage readback
 *           failed.
 */
int regulator_is_supported_voltage(struct regulator *regulator,
                                   int min_uV, int max_uV)
{
        struct regulator_dev *rdev = regulator->rdev;
        int i, voltages, ret;

        /* If we can't change voltage check the current voltage */
        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
                ret = regulator_get_voltage(regulator);
                if (ret >= 0)
                        return min_uV <= ret && ret <= max_uV;
                else
                        return ret;
        }

        /* Any voltage within constrains range is fine? */
        if (rdev->desc->continuous_voltage_range)
                return min_uV >= rdev->constraints->min_uV &&
                                max_uV <= rdev->constraints->max_uV;

        ret = regulator_count_voltages(regulator);
        if (ret < 0)
                return 0;
        voltages = ret;

        for (i = 0; i < voltages; i++) {
                ret = regulator_list_voltage(regulator, i);

                if (ret >= min_uV && ret <= max_uV)
                        return 1;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(regulator_is_supported_voltage);

static int regulator_map_voltage(struct regulator_dev *rdev, int min_uV,
                                 int max_uV)
{
        const struct regulator_desc *desc = rdev->desc;

        if (desc->ops->map_voltage)
                return desc->ops->map_voltage(rdev, min_uV, max_uV);

        if (desc->ops->list_voltage == regulator_list_voltage_linear)
                return regulator_map_voltage_linear(rdev, min_uV, max_uV);

        if (desc->ops->list_voltage == regulator_list_voltage_linear_range)
                return regulator_map_voltage_linear_range(rdev, min_uV, max_uV);

        if (desc->ops->list_voltage ==
                regulator_list_voltage_pickable_linear_range)
                return regulator_map_voltage_pickable_linear_range(rdev,
                                                        min_uV, max_uV);

        return regulator_map_voltage_iterate(rdev, min_uV, max_uV);
}

static int _regulator_call_set_voltage(struct regulator_dev *rdev,
                                       int min_uV, int max_uV,
                                       unsigned *selector)
{
        struct pre_voltage_change_data data;
        int ret;

        data.old_uV = regulator_get_voltage_rdev(rdev);
        data.min_uV = min_uV;
        data.max_uV = max_uV;
        ret = _notifier_call_chain(rdev, REGULATOR_EVENT_PRE_VOLTAGE_CHANGE,
                                   &data);
        if (ret & NOTIFY_STOP_MASK)
                return -EINVAL;

        ret = rdev->desc->ops->set_voltage(rdev, min_uV, max_uV, selector);
        if (ret >= 0)
                return ret;

        _notifier_call_chain(rdev, REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE,
                             (void *)data.old_uV);

        return ret;
}

static int _regulator_call_set_voltage_sel(struct regulator_dev *rdev,
                                           int uV, unsigned selector)
{
        struct pre_voltage_change_data data;
        int ret;

        data.old_uV = regulator_get_voltage_rdev(rdev);
        data.min_uV = uV;
        data.max_uV = uV;
        ret = _notifier_call_chain(rdev, REGULATOR_EVENT_PRE_VOLTAGE_CHANGE,
                                   &data);
        if (ret & NOTIFY_STOP_MASK)
                return -EINVAL;

        ret = rdev->desc->ops->set_voltage_sel(rdev, selector);
        if (ret >= 0)
                return ret;

        _notifier_call_chain(rdev, REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE,
                             (void *)data.old_uV);

        return ret;
}

static int _regulator_set_voltage_sel_step(struct regulator_dev *rdev,
                                           int uV, int new_selector)
{
        const struct regulator_ops *ops = rdev->desc->ops;
        int diff, old_sel, curr_sel, ret;

        /* Stepping is only needed if the regulator is enabled. */
        if (!_regulator_is_enabled(rdev))
                goto final_set;

        if (!ops->get_voltage_sel)
                return -EINVAL;

        old_sel = ops->get_voltage_sel(rdev);
        if (old_sel < 0)
                return old_sel;

        diff = new_selector - old_sel;
        if (diff == 0)
                return 0; /* No change needed. */

        if (diff > 0) {
                /* Stepping up. */
                for (curr_sel = old_sel + rdev->desc->vsel_step;
                     curr_sel < new_selector;
                     curr_sel += rdev->desc->vsel_step) {
                        /*
                         * Call the callback directly instead of using
                         * _regulator_call_set_voltage_sel() as we don't
                         * want to notify anyone yet. Same in the branch
                         * below.
                         */
                        ret = ops->set_voltage_sel(rdev, curr_sel);
                        if (ret)
                                goto try_revert;
                }
        } else {
                /* Stepping down. */
                for (curr_sel = old_sel - rdev->desc->vsel_step;
                     curr_sel > new_selector;
                     curr_sel -= rdev->desc->vsel_step) {
                        ret = ops->set_voltage_sel(rdev, curr_sel);
                        if (ret)
                                goto try_revert;
                }
        }

final_set:
        /* The final selector will trigger the notifiers. */
        return _regulator_call_set_voltage_sel(rdev, uV, new_selector);

try_revert:
        /*
         * At least try to return to the previous voltage if setting a new
         * one failed.
         */
        (void)ops->set_voltage_sel(rdev, old_sel);
        return ret;
}

static int _regulator_set_voltage_time(struct regulator_dev *rdev,
                                       int old_uV, int new_uV)
{
        unsigned int ramp_delay = 0;

        if (rdev->constraints->ramp_delay)
                ramp_delay = rdev->constraints->ramp_delay;
        else if (rdev->desc->ramp_delay)
                ramp_delay = rdev->desc->ramp_delay;
        else if (rdev->constraints->settling_time)
                return rdev->constraints->settling_time;
        else if (rdev->constraints->settling_time_up &&
                 (new_uV > old_uV))
                return rdev->constraints->settling_time_up;
        else if (rdev->constraints->settling_time_down &&
                 (new_uV < old_uV))
                return rdev->constraints->settling_time_down;

        if (ramp_delay == 0)
                return 0;

        return DIV_ROUND_UP(abs(new_uV - old_uV), ramp_delay);
}

static int _regulator_do_set_voltage(struct regulator_dev *rdev,
                                     int min_uV, int max_uV)
{
        int ret;
        int delay = 0;
        int best_val = 0;
        unsigned int selector;
        int old_selector = -1;
        const struct regulator_ops *ops = rdev->desc->ops;
        int old_uV = regulator_get_voltage_rdev(rdev);

        trace_regulator_set_voltage(rdev_get_name(rdev), min_uV, max_uV);

        min_uV += rdev->constraints->uV_offset;
        max_uV += rdev->constraints->uV_offset;

        /*
         * If we can't obtain the old selector there is not enough
         * info to call set_voltage_time_sel().
         */
        if (_regulator_is_enabled(rdev) &&
            ops->set_voltage_time_sel && ops->get_voltage_sel) {
                old_selector = ops->get_voltage_sel(rdev);
                if (old_selector < 0)
                        return old_selector;
        }

        if (ops->set_voltage) {
                ret = _regulator_call_set_voltage(rdev, min_uV, max_uV,
                                                  &selector);

                if (ret >= 0) {
                        if (ops->list_voltage)
                                best_val = ops->list_voltage(rdev,
                                                             selector);
                        else
                                best_val = regulator_get_voltage_rdev(rdev);
                }

        } else if (ops->set_voltage_sel) {
                ret = regulator_map_voltage(rdev, min_uV, max_uV);
                if (ret >= 0) {
                        best_val = ops->list_voltage(rdev, ret);
                        if (min_uV <= best_val && max_uV >= best_val) {
                                selector = ret;
                                if (old_selector == selector)
                                        ret = 0;
                                else if (rdev->desc->vsel_step)
                                        ret = _regulator_set_voltage_sel_step(
                                                rdev, best_val, selector);
                                else
                                        ret = _regulator_call_set_voltage_sel(
                                                rdev, best_val, selector);
                        } else {
                                ret = -EINVAL;
                        }
                }
        } else {
                ret = -EINVAL;
        }

        if (ret)
                goto out;

        if (ops->set_voltage_time_sel) {
                /*
                 * Call set_voltage_time_sel if successfully obtained
                 * old_selector
                 */
                if (old_selector >= 0 && old_selector != selector)
                        delay = ops->set_voltage_time_sel(rdev, old_selector,
                                                          selector);
        } else {
                if (old_uV != best_val) {
                        if (ops->set_voltage_time)
                                delay = ops->set_voltage_time(rdev, old_uV,
                                                              best_val);
                        else
                                delay = _regulator_set_voltage_time(rdev,
                                                                    old_uV,
                                                                    best_val);
                }
        }

        if (delay < 0) {
                rdev_warn(rdev, "failed to get delay: %pe\n", ERR_PTR(delay));
                delay = 0;
        }

        /* Insert any necessary delays */
        fsleep(delay);

        if (best_val >= 0) {
                unsigned long data = best_val;

                _notifier_call_chain(rdev, REGULATOR_EVENT_VOLTAGE_CHANGE,
                                     (void *)data);
        }

out:
        trace_regulator_set_voltage_complete(rdev_get_name(rdev), best_val);

        return ret;
}

static int _regulator_do_set_suspend_voltage(struct regulator_dev *rdev,
                                  int min_uV, int max_uV, suspend_state_t state)
{
        struct regulator_state *rstate;
        int uV, sel;

        rstate = regulator_get_suspend_state(rdev, state);
        if (rstate == NULL)
                return -EINVAL;

        if (min_uV < rstate->min_uV)
                min_uV = rstate->min_uV;
        if (max_uV > rstate->max_uV)
                max_uV = rstate->max_uV;

        sel = regulator_map_voltage(rdev, min_uV, max_uV);
        if (sel < 0)
                return sel;

        uV = rdev->desc->ops->list_voltage(rdev, sel);
        if (uV >= min_uV && uV <= max_uV)
                rstate->uV = uV;

        return 0;
}

static int regulator_set_voltage_unlocked(struct regulator *regulator,
                                          int min_uV, int max_uV,
                                          suspend_state_t state)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct regulator_voltage *voltage = &regulator->voltage[state];
        int ret = 0;
        int old_min_uV, old_max_uV;
        int current_uV;

        /* If we're setting the same range as last time the change
         * should be a noop (some cpufreq implementations use the same
         * voltage for multiple frequencies, for example).
         */
        if (voltage->min_uV == min_uV && voltage->max_uV == max_uV)
                goto out;

        /* If we're trying to set a range that overlaps the current voltage,
         * return successfully even though the regulator does not support
         * changing the voltage.
         */
        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE)) {
                current_uV = regulator_get_voltage_rdev(rdev);
                if (min_uV <= current_uV && current_uV <= max_uV) {
                        voltage->min_uV = min_uV;
                        voltage->max_uV = max_uV;
                        goto out;
                }
        }

        /* sanity check */
        if (!rdev->desc->ops->set_voltage &&
            !rdev->desc->ops->set_voltage_sel) {
                ret = -EINVAL;
                goto out;
        }

        /* constraints check */
        ret = regulator_check_voltage(rdev, &min_uV, &max_uV);
        if (ret < 0)
                goto out;

        /* restore original values in case of error */
        old_min_uV = voltage->min_uV;
        old_max_uV = voltage->max_uV;
        voltage->min_uV = min_uV;
        voltage->max_uV = max_uV;

        /* for not coupled regulators this will just set the voltage */
        ret = regulator_balance_voltage(rdev, state);
        if (ret < 0) {
                voltage->min_uV = old_min_uV;
                voltage->max_uV = old_max_uV;
        }

out:
        return ret;
}

int regulator_set_voltage_rdev(struct regulator_dev *rdev, int min_uV,
                               int max_uV, suspend_state_t state)
{
        int best_supply_uV = 0;
        int supply_change_uV = 0;
        int ret;

        if (rdev->supply &&
            regulator_ops_is_valid(rdev->supply->rdev,
                                   REGULATOR_CHANGE_VOLTAGE) &&
            (rdev->desc->min_dropout_uV || !(rdev->desc->ops->get_voltage ||
                                           rdev->desc->ops->get_voltage_sel))) {
                int current_supply_uV;
                int selector;

                selector = regulator_map_voltage(rdev, min_uV, max_uV);
                if (selector < 0) {
                        ret = selector;
                        goto out;
                }

                best_supply_uV = _regulator_list_voltage(rdev, selector, 0);
                if (best_supply_uV < 0) {
                        ret = best_supply_uV;
                        goto out;
                }

                best_supply_uV += rdev->desc->min_dropout_uV;

                current_supply_uV = regulator_get_voltage_rdev(rdev->supply->rdev);
                if (current_supply_uV < 0) {
                        ret = current_supply_uV;
                        goto out;
                }

                supply_change_uV = best_supply_uV - current_supply_uV;
        }

        if (supply_change_uV > 0) {
                ret = regulator_set_voltage_unlocked(rdev->supply,
                                best_supply_uV, INT_MAX, state);
                if (ret) {
                        dev_err(&rdev->dev, "Failed to increase supply voltage: %pe\n",
                                ERR_PTR(ret));
                        goto out;
                }
        }

        if (state == PM_SUSPEND_ON)
                ret = _regulator_do_set_voltage(rdev, min_uV, max_uV);
        else
                ret = _regulator_do_set_suspend_voltage(rdev, min_uV,
                                                        max_uV, state);
        if (ret < 0)
                goto out;

        if (supply_change_uV < 0) {
                ret = regulator_set_voltage_unlocked(rdev->supply,
                                best_supply_uV, INT_MAX, state);
                if (ret)
                        dev_warn(&rdev->dev, "Failed to decrease supply voltage: %pe\n",
                                 ERR_PTR(ret));
                /* No need to fail here */
                ret = 0;
        }

out:
        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_rdev);

static int regulator_limit_voltage_step(struct regulator_dev *rdev,
                                        int *current_uV, int *min_uV)
{
        struct regulation_constraints *constraints = rdev->constraints;

        /* Limit voltage change only if necessary */
        if (!constraints->max_uV_step || !_regulator_is_enabled(rdev))
                return 1;

        if (*current_uV < 0) {
                *current_uV = regulator_get_voltage_rdev(rdev);

                if (*current_uV < 0)
                        return *current_uV;
        }

        if (abs(*current_uV - *min_uV) <= constraints->max_uV_step)
                return 1;

        /* Clamp target voltage within the given step */
        if (*current_uV < *min_uV)
                *min_uV = min(*current_uV + constraints->max_uV_step,
                              *min_uV);
        else
                *min_uV = max(*current_uV - constraints->max_uV_step,
                              *min_uV);

        return 0;
}

static int regulator_get_optimal_voltage(struct regulator_dev *rdev,
                                         int *current_uV,
                                         int *min_uV, int *max_uV,
                                         suspend_state_t state,
                                         int n_coupled)
{
        struct coupling_desc *c_desc = &rdev->coupling_desc;
        struct regulator_dev **c_rdevs = c_desc->coupled_rdevs;
        struct regulation_constraints *constraints = rdev->constraints;
        int desired_min_uV = 0, desired_max_uV = INT_MAX;
        int max_current_uV = 0, min_current_uV = INT_MAX;
        int highest_min_uV = 0, target_uV, possible_uV;
        int i, ret, max_spread;
        bool done;

        *current_uV = -1;

        /*
         * If there are no coupled regulators, simply set the voltage
         * demanded by consumers.
         */
        if (n_coupled == 1) {
                /*
                 * If consumers don't provide any demands, set voltage
                 * to min_uV
                 */
                desired_min_uV = constraints->min_uV;
                desired_max_uV = constraints->max_uV;

                ret = regulator_check_consumers(rdev,
                                                &desired_min_uV,
                                                &desired_max_uV, state);
                if (ret < 0)
                        return ret;

                done = true;

                goto finish;
        }

        /* Find highest min desired voltage */
        for (i = 0; i < n_coupled; i++) {
                int tmp_min = 0;
                int tmp_max = INT_MAX;

                lockdep_assert_held_once(&c_rdevs[i]->mutex.base);

                ret = regulator_check_consumers(c_rdevs[i],
                                                &tmp_min,
                                                &tmp_max, state);
                if (ret < 0)
                        return ret;

                ret = regulator_check_voltage(c_rdevs[i], &tmp_min, &tmp_max);
                if (ret < 0)
                        return ret;

                highest_min_uV = max(highest_min_uV, tmp_min);

                if (i == 0) {
                        desired_min_uV = tmp_min;
                        desired_max_uV = tmp_max;
                }
        }

        max_spread = constraints->max_spread[0];

        /*
         * Let target_uV be equal to the desired one if possible.
         * If not, set it to minimum voltage, allowed by other coupled
         * regulators.
         */
        target_uV = max(desired_min_uV, highest_min_uV - max_spread);

        /*
         * Find min and max voltages, which currently aren't violating
         * max_spread.
         */
        for (i = 1; i < n_coupled; i++) {
                int tmp_act;

                if (!_regulator_is_enabled(c_rdevs[i]))
                        continue;

                tmp_act = regulator_get_voltage_rdev(c_rdevs[i]);
                if (tmp_act < 0)
                        return tmp_act;

                min_current_uV = min(tmp_act, min_current_uV);
                max_current_uV = max(tmp_act, max_current_uV);
        }

        /* There aren't any other regulators enabled */
        if (max_current_uV == 0) {
                possible_uV = target_uV;
        } else {
                /*
                 * Correct target voltage, so as it currently isn't
                 * violating max_spread
                 */
                possible_uV = max(target_uV, max_current_uV - max_spread);
                possible_uV = min(possible_uV, min_current_uV + max_spread);
        }

        if (possible_uV > desired_max_uV)
                return -EINVAL;

        done = (possible_uV == target_uV);
        desired_min_uV = possible_uV;

finish:
        /* Apply max_uV_step constraint if necessary */
        if (state == PM_SUSPEND_ON) {
                ret = regulator_limit_voltage_step(rdev, current_uV,
                                                   &desired_min_uV);
                if (ret < 0)
                        return ret;

                if (ret == 0)
                        done = false;
        }

        /* Set current_uV if wasn't done earlier in the code and if necessary */
        if (n_coupled > 1 && *current_uV == -1) {

                if (_regulator_is_enabled(rdev)) {
                        ret = regulator_get_voltage_rdev(rdev);
                        if (ret < 0)
                                return ret;

                        *current_uV = ret;
                } else {
                        *current_uV = desired_min_uV;
                }
        }

        *min_uV = desired_min_uV;
        *max_uV = desired_max_uV;

        return done;
}

int regulator_do_balance_voltage(struct regulator_dev *rdev,
                                 suspend_state_t state, bool skip_coupled)
{
        struct regulator_dev **c_rdevs;
        struct regulator_dev *best_rdev;
        struct coupling_desc *c_desc = &rdev->coupling_desc;
        int i, ret, n_coupled, best_min_uV, best_max_uV, best_c_rdev;
        unsigned int delta, best_delta;
        unsigned long c_rdev_done = 0;
        bool best_c_rdev_done;

        c_rdevs = c_desc->coupled_rdevs;
        n_coupled = skip_coupled ? 1 : c_desc->n_coupled;

        /*
         * Find the best possible voltage change on each loop. Leave the loop
         * if there isn't any possible change.
         */
        do {
                best_c_rdev_done = false;
                best_delta = 0;
                best_min_uV = 0;
                best_max_uV = 0;
                best_c_rdev = 0;
                best_rdev = NULL;

                /*
                 * Find highest difference between optimal voltage
                 * and current voltage.
                 */
                for (i = 0; i < n_coupled; i++) {
                        /*
                         * optimal_uV is the best voltage that can be set for
                         * i-th regulator at the moment without violating
                         * max_spread constraint in order to balance
                         * the coupled voltages.
                         */
                        int optimal_uV = 0, optimal_max_uV = 0, current_uV = 0;

                        if (test_bit(i, &c_rdev_done))
                                continue;

                        ret = regulator_get_optimal_voltage(c_rdevs[i],
                                                            &current_uV,
                                                            &optimal_uV,
                                                            &optimal_max_uV,
                                                            state, n_coupled);
                        if (ret < 0)
                                goto out;

                        delta = abs(optimal_uV - current_uV);

                        if (delta && best_delta <= delta) {
                                best_c_rdev_done = ret;
                                best_delta = delta;
                                best_rdev = c_rdevs[i];
                                best_min_uV = optimal_uV;
                                best_max_uV = optimal_max_uV;
                                best_c_rdev = i;
                        }
                }

                /* Nothing to change, return successfully */
                if (!best_rdev) {
                        ret = 0;
                        goto out;
                }

                ret = regulator_set_voltage_rdev(best_rdev, best_min_uV,
                                                 best_max_uV, state);

                if (ret < 0)
                        goto out;

                if (best_c_rdev_done)
                        set_bit(best_c_rdev, &c_rdev_done);

        } while (n_coupled > 1);

out:
        return ret;
}

static int regulator_balance_voltage(struct regulator_dev *rdev,
                                     suspend_state_t state)
{
        struct coupling_desc *c_desc = &rdev->coupling_desc;
        struct regulator_coupler *coupler = c_desc->coupler;
        bool skip_coupled = false;

        /*
         * If system is in a state other than PM_SUSPEND_ON, don't check
         * other coupled regulators.
         */
        if (state != PM_SUSPEND_ON)
                skip_coupled = true;

        if (c_desc->n_resolved < c_desc->n_coupled) {
                rdev_err(rdev, "Not all coupled regulators registered\n");
                return -EPERM;
        }

        /* Invoke custom balancer for customized couplers */
        if (coupler && coupler->balance_voltage)
                return coupler->balance_voltage(coupler, rdev, state);

        return regulator_do_balance_voltage(rdev, state, skip_coupled);
}

/**
 * regulator_set_voltage - set regulator output voltage
 * @regulator: regulator source
 * @min_uV: Minimum required voltage in uV
 * @max_uV: Maximum acceptable voltage in uV
 *
 * Sets a voltage regulator to the desired output voltage. This can be set
 * during any regulator state. IOW, regulator can be disabled or enabled.
 *
 * If the regulator is enabled then the voltage will change to the new value
 * immediately otherwise if the regulator is disabled the regulator will
 * output at the new voltage when enabled.
 *
 * NOTE: If the regulator is shared between several devices then the lowest
 * request voltage that meets the system constraints will be used.
 * Regulator system constraints must be set for this regulator before
 * calling this function otherwise this call will fail.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_set_voltage(struct regulator *regulator, int min_uV, int max_uV)
{
        struct ww_acquire_ctx ww_ctx;
        int ret;

        regulator_lock_dependent(regulator->rdev, &ww_ctx);

        ret = regulator_set_voltage_unlocked(regulator, min_uV, max_uV,
                                             PM_SUSPEND_ON);

        regulator_unlock_dependent(regulator->rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_voltage);

static inline int regulator_suspend_toggle(struct regulator_dev *rdev,
                                           suspend_state_t state, bool en)
{
        struct regulator_state *rstate;

        rstate = regulator_get_suspend_state(rdev, state);
        if (rstate == NULL)
                return -EINVAL;

        if (!rstate->changeable)
                return -EPERM;

        rstate->enabled = (en) ? ENABLE_IN_SUSPEND : DISABLE_IN_SUSPEND;

        return 0;
}

int regulator_suspend_enable(struct regulator_dev *rdev,
                                    suspend_state_t state)
{
        return regulator_suspend_toggle(rdev, state, true);
}
EXPORT_SYMBOL_GPL(regulator_suspend_enable);

int regulator_suspend_disable(struct regulator_dev *rdev,
                                     suspend_state_t state)
{
        struct regulator *regulator;
        struct regulator_voltage *voltage;

        /*
         * if any consumer wants this regulator device keeping on in
         * suspend states, don't set it as disabled.
         */
        list_for_each_entry(regulator, &rdev->consumer_list, list) {
                voltage = &regulator->voltage[state];
                if (voltage->min_uV || voltage->max_uV)
                        return 0;
        }

        return regulator_suspend_toggle(rdev, state, false);
}
EXPORT_SYMBOL_GPL(regulator_suspend_disable);

static int _regulator_set_suspend_voltage(struct regulator *regulator,
                                          int min_uV, int max_uV,
                                          suspend_state_t state)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct regulator_state *rstate;

        rstate = regulator_get_suspend_state(rdev, state);
        if (rstate == NULL)
                return -EINVAL;

        if (rstate->min_uV == rstate->max_uV) {
                rdev_err(rdev, "The suspend voltage can't be changed!\n");
                return -EPERM;
        }

        return regulator_set_voltage_unlocked(regulator, min_uV, max_uV, state);
}

int regulator_set_suspend_voltage(struct regulator *regulator, int min_uV,
                                  int max_uV, suspend_state_t state)
{
        struct ww_acquire_ctx ww_ctx;
        int ret;

        /* PM_SUSPEND_ON is handled by regulator_set_voltage() */
        if (regulator_check_states(state) || state == PM_SUSPEND_ON)
                return -EINVAL;

        regulator_lock_dependent(regulator->rdev, &ww_ctx);

        ret = _regulator_set_suspend_voltage(regulator, min_uV,
                                             max_uV, state);

        regulator_unlock_dependent(regulator->rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_suspend_voltage);

/**
 * regulator_set_voltage_time - get raise/fall time
 * @regulator: regulator source
 * @old_uV: starting voltage in microvolts
 * @new_uV: target voltage in microvolts
 *
 * Provided with the starting and ending voltage, this function attempts to
 * calculate the time in microseconds required to rise or fall to this new
 * voltage.
 *
 * Return: ramp time in microseconds, or a negative error number if calculation failed.
 */
int regulator_set_voltage_time(struct regulator *regulator,
                               int old_uV, int new_uV)
{
        struct regulator_dev *rdev = regulator->rdev;
        const struct regulator_ops *ops = rdev->desc->ops;
        int old_sel = -1;
        int new_sel = -1;
        int voltage;
        int i;

        if (ops->set_voltage_time)
                return ops->set_voltage_time(rdev, old_uV, new_uV);
        else if (!ops->set_voltage_time_sel)
                return _regulator_set_voltage_time(rdev, old_uV, new_uV);

        /* Currently requires operations to do this */
        if (!ops->list_voltage || !rdev->desc->n_voltages)
                return -EINVAL;

        for (i = 0; i < rdev->desc->n_voltages; i++) {
                /* We only look for exact voltage matches here */
                if (i < rdev->desc->linear_min_sel)
                        continue;

                if (old_sel >= 0 && new_sel >= 0)
                        break;

                voltage = regulator_list_voltage(regulator, i);
                if (voltage < 0)
                        return -EINVAL;
                if (voltage == 0)
                        continue;
                if (voltage == old_uV)
                        old_sel = i;
                if (voltage == new_uV)
                        new_sel = i;
        }

        if (old_sel < 0 || new_sel < 0)
                return -EINVAL;

        return ops->set_voltage_time_sel(rdev, old_sel, new_sel);
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_time);

/**
 * regulator_set_voltage_time_sel - get raise/fall time
 * @rdev: regulator source device
 * @old_selector: selector for starting voltage
 * @new_selector: selector for target voltage
 *
 * Provided with the starting and target voltage selectors, this function
 * returns time in microseconds required to rise or fall to this new voltage
 *
 * Drivers providing ramp_delay in regulation_constraints can use this as their
 * set_voltage_time_sel() operation.
 *
 * Return: ramp time in microseconds, or a negative error number if calculation failed.
 */
int regulator_set_voltage_time_sel(struct regulator_dev *rdev,
                                   unsigned int old_selector,
                                   unsigned int new_selector)
{
        int old_volt, new_volt;

        /* sanity check */
        if (!rdev->desc->ops->list_voltage)
                return -EINVAL;

        old_volt = rdev->desc->ops->list_voltage(rdev, old_selector);
        new_volt = rdev->desc->ops->list_voltage(rdev, new_selector);

        if (rdev->desc->ops->set_voltage_time)
                return rdev->desc->ops->set_voltage_time(rdev, old_volt,
                                                         new_volt);
        else
                return _regulator_set_voltage_time(rdev, old_volt, new_volt);
}
EXPORT_SYMBOL_GPL(regulator_set_voltage_time_sel);

int regulator_sync_voltage_rdev(struct regulator_dev *rdev)
{
        int ret;

        regulator_lock(rdev);

        if (!rdev->desc->ops->set_voltage &&
            !rdev->desc->ops->set_voltage_sel) {
                ret = -EINVAL;
                goto out;
        }

        /* balance only, if regulator is coupled */
        if (rdev->coupling_desc.n_coupled > 1)
                ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
        else
                ret = -EOPNOTSUPP;

out:
        regulator_unlock(rdev);
        return ret;
}

/**
 * regulator_sync_voltage - re-apply last regulator output voltage
 * @regulator: regulator source
 *
 * Re-apply the last configured voltage.  This is intended to be used
 * where some external control source the consumer is cooperating with
 * has caused the configured voltage to change.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_sync_voltage(struct regulator *regulator)
{
        struct regulator_dev *rdev = regulator->rdev;
        struct regulator_voltage *voltage = &regulator->voltage[PM_SUSPEND_ON];
        int ret, min_uV, max_uV;

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE))
                return 0;

        regulator_lock(rdev);

        if (!rdev->desc->ops->set_voltage &&
            !rdev->desc->ops->set_voltage_sel) {
                ret = -EINVAL;
                goto out;
        }

        /* This is only going to work if we've had a voltage configured. */
        if (!voltage->min_uV && !voltage->max_uV) {
                ret = -EINVAL;
                goto out;
        }

        min_uV = voltage->min_uV;
        max_uV = voltage->max_uV;

        /* This should be a paranoia check... */
        ret = regulator_check_voltage(rdev, &min_uV, &max_uV);
        if (ret < 0)
                goto out;

        ret = regulator_check_consumers(rdev, &min_uV, &max_uV, 0);
        if (ret < 0)
                goto out;

        /* balance only, if regulator is coupled */
        if (rdev->coupling_desc.n_coupled > 1)
                ret = regulator_balance_voltage(rdev, PM_SUSPEND_ON);
        else
                ret = _regulator_do_set_voltage(rdev, min_uV, max_uV);

out:
        regulator_unlock(rdev);
        return ret;
}
EXPORT_SYMBOL_GPL(regulator_sync_voltage);

int regulator_get_voltage_rdev(struct regulator_dev *rdev)
{
        int sel, ret;
        bool bypassed;

        if (rdev->desc->ops->get_bypass) {
                ret = rdev->desc->ops->get_bypass(rdev, &bypassed);
                if (ret < 0)
                        return ret;
                if (bypassed) {
                        /* if bypassed the regulator must have a supply */
                        if (!rdev->supply) {
                                rdev_err(rdev,
                                         "bypassed regulator has no supply!\n");
                                return -EPROBE_DEFER;
                        }

                        return regulator_get_voltage_rdev(rdev->supply->rdev);
                }
        }

        if (rdev->desc->ops->get_voltage_sel) {
                sel = rdev->desc->ops->get_voltage_sel(rdev);
                if (sel < 0)
                        return sel;
                ret = rdev->desc->ops->list_voltage(rdev, sel);
        } else if (rdev->desc->ops->get_voltage) {
                ret = rdev->desc->ops->get_voltage(rdev);
        } else if (rdev->desc->ops->list_voltage) {
                ret = rdev->desc->ops->list_voltage(rdev, 0);
        } else if (rdev->desc->fixed_uV && (rdev->desc->n_voltages == 1)) {
                ret = rdev->desc->fixed_uV;
        } else if (rdev->supply) {
                ret = regulator_get_voltage_rdev(rdev->supply->rdev);
        } else if (rdev->supply_name) {
                return -EPROBE_DEFER;
        } else {
                return -EINVAL;
        }

        if (ret < 0)
                return ret;
        return ret - rdev->constraints->uV_offset;
}
EXPORT_SYMBOL_GPL(regulator_get_voltage_rdev);

/**
 * regulator_get_voltage - get regulator output voltage
 * @regulator: regulator source
 *
 * Return: Current regulator voltage in uV, or a negative error number on failure.
 *
 * NOTE: If the regulator is disabled it will return the voltage value. This
 * function should not be used to determine regulator state.
 */
int regulator_get_voltage(struct regulator *regulator)
{
        struct ww_acquire_ctx ww_ctx;
        int ret;

        regulator_lock_dependent(regulator->rdev, &ww_ctx);
        ret = regulator_get_voltage_rdev(regulator->rdev);
        regulator_unlock_dependent(regulator->rdev, &ww_ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_get_voltage);

/**
 * regulator_set_current_limit - set regulator output current limit
 * @regulator: regulator source
 * @min_uA: Minimum supported current in uA
 * @max_uA: Maximum supported current in uA
 *
 * Sets current sink to the desired output current. This can be set during
 * any regulator state. IOW, regulator can be disabled or enabled.
 *
 * If the regulator is enabled then the current will change to the new value
 * immediately otherwise if the regulator is disabled the regulator will
 * output at the new current when enabled.
 *
 * NOTE: Regulator system constraints must be set for this regulator before
 * calling this function otherwise this call will fail.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_set_current_limit(struct regulator *regulator,
                               int min_uA, int max_uA)
{
        struct regulator_dev *rdev = regulator->rdev;
        int ret;

        regulator_lock(rdev);

        /* sanity check */
        if (!rdev->desc->ops->set_current_limit) {
                ret = -EINVAL;
                goto out;
        }

        /* constraints check */
        ret = regulator_check_current_limit(rdev, &min_uA, &max_uA);
        if (ret < 0)
                goto out;

        ret = rdev->desc->ops->set_current_limit(rdev, min_uA, max_uA);
out:
        regulator_unlock(rdev);
        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_current_limit);

static int _regulator_get_current_limit_unlocked(struct regulator_dev *rdev)
{
        /* sanity check */
        if (!rdev->desc->ops->get_current_limit)
                return -EINVAL;

        return rdev->desc->ops->get_current_limit(rdev);
}

static int _regulator_get_current_limit(struct regulator_dev *rdev)
{
        int ret;

        regulator_lock(rdev);
        ret = _regulator_get_current_limit_unlocked(rdev);
        regulator_unlock(rdev);

        return ret;
}

/**
 * regulator_get_current_limit - get regulator output current
 * @regulator: regulator source
 *
 * Return: Current supplied by the specified current sink in uA,
 *           or a negative error number on failure.
 *
 * NOTE: If the regulator is disabled it will return the current value. This
 * function should not be used to determine regulator state.
 */
int regulator_get_current_limit(struct regulator *regulator)
{
        return _regulator_get_current_limit(regulator->rdev);
}
EXPORT_SYMBOL_GPL(regulator_get_current_limit);

/**
 * regulator_get_unclaimed_power_budget - get regulator unclaimed power budget
 * @regulator: regulator source
 *
 * Return: Unclaimed power budget of the regulator in mW.
 */
int regulator_get_unclaimed_power_budget(struct regulator *regulator)
{
        return regulator->rdev->constraints->pw_budget_mW -
               regulator->rdev->pw_requested_mW;
}
EXPORT_SYMBOL_GPL(regulator_get_unclaimed_power_budget);

/**
 * regulator_request_power_budget - request power budget on a regulator
 * @regulator: regulator source
 * @pw_req: Power requested
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_request_power_budget(struct regulator *regulator,
                                   unsigned int pw_req)
{
        struct regulator_dev *rdev = regulator->rdev;
        int ret = 0, pw_tot_req;

        regulator_lock(rdev);
        if (rdev->supply) {
                ret = regulator_request_power_budget(rdev->supply, pw_req);
                if (ret < 0)
                        goto out;
        }

        pw_tot_req = rdev->pw_requested_mW + pw_req;
        if (pw_tot_req > rdev->constraints->pw_budget_mW) {
                rdev_warn(rdev, "power requested %d mW out of budget %d mW",
                          pw_req,
                          rdev->constraints->pw_budget_mW - rdev->pw_requested_mW);
                regulator_notifier_call_chain(rdev,
                                              REGULATOR_EVENT_OVER_CURRENT_WARN,
                                              NULL);
                ret = -ERANGE;
                goto out;
        }

        rdev->pw_requested_mW = pw_tot_req;
out:
        regulator_unlock(rdev);
        return ret;
}
EXPORT_SYMBOL_GPL(regulator_request_power_budget);

/**
 * regulator_free_power_budget - free power budget on a regulator
 * @regulator: regulator source
 * @pw: Power to be released.
 *
 * Return: Power budget of the regulator in mW.
 */
void regulator_free_power_budget(struct regulator *regulator,
                                 unsigned int pw)
{
        struct regulator_dev *rdev = regulator->rdev;
        int pw_tot_req;

        regulator_lock(rdev);
        if (rdev->supply)
                regulator_free_power_budget(rdev->supply, pw);

        pw_tot_req = rdev->pw_requested_mW - pw;
        if (pw_tot_req >= 0)
                rdev->pw_requested_mW = pw_tot_req;
        else
                rdev_warn(rdev,
                          "too much power freed %d mW (already requested %d mW)",
                          pw, rdev->pw_requested_mW);

        regulator_unlock(rdev);
}
EXPORT_SYMBOL_GPL(regulator_free_power_budget);

/**
 * regulator_set_mode - set regulator operating mode
 * @regulator: regulator source
 * @mode: operating mode - one of the REGULATOR_MODE constants
 *
 * Set regulator operating mode to increase regulator efficiency or improve
 * regulation performance.
 *
 * NOTE: Regulator system constraints must be set for this regulator before
 * calling this function otherwise this call will fail.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_set_mode(struct regulator *regulator, unsigned int mode)
{
        struct regulator_dev *rdev = regulator->rdev;
        int ret;
        int regulator_curr_mode;

        regulator_lock(rdev);

        /* sanity check */
        if (!rdev->desc->ops->set_mode) {
                ret = -EINVAL;
                goto out;
        }

        /* return if the same mode is requested */
        if (rdev->desc->ops->get_mode) {
                regulator_curr_mode = rdev->desc->ops->get_mode(rdev);
                if (regulator_curr_mode == mode) {
                        ret = 0;
                        goto out;
                }
        }

        /* constraints check */
        ret = regulator_mode_constrain(rdev, &mode);
        if (ret < 0)
                goto out;

        ret = rdev->desc->ops->set_mode(rdev, mode);
out:
        regulator_unlock(rdev);
        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_mode);

static unsigned int _regulator_get_mode_unlocked(struct regulator_dev *rdev)
{
        /* sanity check */
        if (!rdev->desc->ops->get_mode)
                return -EINVAL;

        return rdev->desc->ops->get_mode(rdev);
}

static unsigned int _regulator_get_mode(struct regulator_dev *rdev)
{
        int ret;

        regulator_lock(rdev);
        ret = _regulator_get_mode_unlocked(rdev);
        regulator_unlock(rdev);

        return ret;
}

/**
 * regulator_get_mode - get regulator operating mode
 * @regulator: regulator source
 *
 * Get the current regulator operating mode.
 *
 * Return: Current operating mode as %REGULATOR_MODE_* values,
 *           or a negative error number on failure.
 */
unsigned int regulator_get_mode(struct regulator *regulator)
{
        return _regulator_get_mode(regulator->rdev);
}
EXPORT_SYMBOL_GPL(regulator_get_mode);

static int rdev_get_cached_err_flags(struct regulator_dev *rdev)
{
        int ret = 0;

        if (rdev->use_cached_err) {
                spin_lock(&rdev->err_lock);
                ret = rdev->cached_err;
                spin_unlock(&rdev->err_lock);
        }
        return ret;
}

static int _regulator_get_error_flags(struct regulator_dev *rdev,
                                        unsigned int *flags)
{
        int cached_flags, ret = 0;

        regulator_lock(rdev);

        cached_flags = rdev_get_cached_err_flags(rdev);

        if (rdev->desc->ops->get_error_flags)
                ret = rdev->desc->ops->get_error_flags(rdev, flags);
        else if (!rdev->use_cached_err)
                ret = -EINVAL;

        *flags |= cached_flags;

        regulator_unlock(rdev);

        return ret;
}

/**
 * regulator_get_error_flags - get regulator error information
 * @regulator: regulator source
 * @flags: pointer to store error flags
 *
 * Get the current regulator error information.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_get_error_flags(struct regulator *regulator,
                                unsigned int *flags)
{
        return _regulator_get_error_flags(regulator->rdev, flags);
}
EXPORT_SYMBOL_GPL(regulator_get_error_flags);

/**
 * regulator_set_load - set regulator load
 * @regulator: regulator source
 * @uA_load: load current
 *
 * Notifies the regulator core of a new device load. This is then used by
 * DRMS (if enabled by constraints) to set the most efficient regulator
 * operating mode for the new regulator loading.
 *
 * Consumer devices notify their supply regulator of the maximum power
 * they will require (can be taken from device datasheet in the power
 * consumption tables) when they change operational status and hence power
 * state. Examples of operational state changes that can affect power
 * consumption are :-
 *
 *    o Device is opened / closed.
 *    o Device I/O is about to begin or has just finished.
 *    o Device is idling in between work.
 *
 * This information is also exported via sysfs to userspace.
 *
 * DRMS will sum the total requested load on the regulator and change
 * to the most efficient operating mode if platform constraints allow.
 *
 * NOTE: when a regulator consumer requests to have a regulator
 * disabled then any load that consumer requested no longer counts
 * toward the total requested load.  If the regulator is re-enabled
 * then the previously requested load will start counting again.
 *
 * If a regulator is an always-on regulator then an individual consumer's
 * load will still be removed if that consumer is fully disabled.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_set_load(struct regulator *regulator, int uA_load)
{
        struct regulator_dev *rdev = regulator->rdev;
        int old_uA_load;
        int ret = 0;

        regulator_lock(rdev);
        old_uA_load = regulator->uA_load;
        regulator->uA_load = uA_load;
        if (regulator->enable_count && old_uA_load != uA_load) {
                ret = drms_uA_update(rdev);
                if (ret < 0)
                        regulator->uA_load = old_uA_load;
        }
        regulator_unlock(rdev);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_set_load);

/**
 * regulator_allow_bypass - allow the regulator to go into bypass mode
 *
 * @regulator: Regulator to configure
 * @enable: enable or disable bypass mode
 *
 * Allow the regulator to go into bypass mode if all other consumers
 * for the regulator also enable bypass mode and the machine
 * constraints allow this.  Bypass mode means that the regulator is
 * simply passing the input directly to the output with no regulation.
 *
 * Return: 0 on success or if changing bypass is not possible, or
 *           a negative error number on failure.
 */
int regulator_allow_bypass(struct regulator *regulator, bool enable)
{
        struct regulator_dev *rdev = regulator->rdev;
        const char *name = rdev_get_name(rdev);
        int ret = 0;

        if (!rdev->desc->ops->set_bypass)
                return 0;

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_BYPASS))
                return 0;

        regulator_lock(rdev);

        if (enable && !regulator->bypass) {
                rdev->bypass_count++;

                if (rdev->bypass_count == rdev->open_count) {
                        trace_regulator_bypass_enable(name);

                        ret = rdev->desc->ops->set_bypass(rdev, enable);
                        if (ret != 0)
                                rdev->bypass_count--;
                        else
                                trace_regulator_bypass_enable_complete(name);
                }

        } else if (!enable && regulator->bypass) {
                rdev->bypass_count--;

                if (rdev->bypass_count != rdev->open_count) {
                        trace_regulator_bypass_disable(name);

                        ret = rdev->desc->ops->set_bypass(rdev, enable);
                        if (ret != 0)
                                rdev->bypass_count++;
                        else
                                trace_regulator_bypass_disable_complete(name);
                }
        }

        if (ret == 0)
                regulator->bypass = enable;

        regulator_unlock(rdev);

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_allow_bypass);

/**
 * regulator_register_notifier - register regulator event notifier
 * @regulator: regulator source
 * @nb: notifier block
 *
 * Register notifier block to receive regulator events.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_register_notifier(struct regulator *regulator,
                              struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&regulator->rdev->notifier,
                                                nb);
}
EXPORT_SYMBOL_GPL(regulator_register_notifier);

/**
 * regulator_unregister_notifier - unregister regulator event notifier
 * @regulator: regulator source
 * @nb: notifier block
 *
 * Unregister regulator event notifier block.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_unregister_notifier(struct regulator *regulator,
                                struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&regulator->rdev->notifier,
                                                  nb);
}
EXPORT_SYMBOL_GPL(regulator_unregister_notifier);

/* notify regulator consumers and downstream regulator consumers.
 * Note mutex must be held by caller.
 */
static int _notifier_call_chain(struct regulator_dev *rdev,
                                  unsigned long event, void *data)
{
        /* call rdev chain first */
        int ret =  blocking_notifier_call_chain(&rdev->notifier, event, data);

        if (IS_REACHABLE(CONFIG_REGULATOR_NETLINK_EVENTS)) {
                struct device *parent = rdev->dev.parent;
                const char *rname = rdev_get_name(rdev);
                char name[32];

                /* Avoid duplicate debugfs directory names */
                if (parent && rname == rdev->desc->name) {
                        snprintf(name, sizeof(name), "%s-%s", dev_name(parent),
                                 rname);
                        rname = name;
                }
                reg_generate_netlink_event(rname, event);
        }

        return ret;
}

int _regulator_bulk_get(struct device *dev, int num_consumers,
                        struct regulator_bulk_data *consumers, enum regulator_get_type get_type)
{
        int i;
        int ret;

        for (i = 0; i < num_consumers; i++)
                consumers[i].consumer = NULL;

        for (i = 0; i < num_consumers; i++) {
                consumers[i].consumer = _regulator_get(dev,
                                                       consumers[i].supply, get_type);
                if (IS_ERR(consumers[i].consumer)) {
                        ret = dev_err_probe(dev, PTR_ERR(consumers[i].consumer),
                                            "Failed to get supply '%s'\n",
                                            consumers[i].supply);
                        consumers[i].consumer = NULL;
                        goto err;
                }

                if (consumers[i].init_load_uA > 0) {
                        ret = regulator_set_load(consumers[i].consumer,
                                                 consumers[i].init_load_uA);
                        if (ret) {
                                i++;
                                goto err;
                        }
                }
        }

        return 0;

err:
        while (--i >= 0)
                regulator_put(consumers[i].consumer);

        return ret;
}

/**
 * regulator_bulk_get - get multiple regulator consumers
 *
 * @dev:           Device to supply
 * @num_consumers: Number of consumers to register
 * @consumers:     Configuration of consumers; clients are stored here.
 *
 * This helper function allows drivers to get several regulator
 * consumers in one operation.  If any of the regulators cannot be
 * acquired then any regulators that were allocated will be freed
 * before returning to the caller.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_bulk_get(struct device *dev, int num_consumers,
                       struct regulator_bulk_data *consumers)
{
        return _regulator_bulk_get(dev, num_consumers, consumers, NORMAL_GET);
}
EXPORT_SYMBOL_GPL(regulator_bulk_get);

static void regulator_bulk_enable_async(void *data, async_cookie_t cookie)
{
        struct regulator_bulk_data *bulk = data;

        bulk->ret = regulator_enable(bulk->consumer);
}

/**
 * regulator_bulk_enable - enable multiple regulator consumers
 *
 * @num_consumers: Number of consumers
 * @consumers:     Consumer data; clients are stored here.
 *
 * This convenience API allows consumers to enable multiple regulator
 * clients in a single API call.  If any consumers cannot be enabled
 * then any others that were enabled will be disabled again prior to
 * return.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_bulk_enable(int num_consumers,
                          struct regulator_bulk_data *consumers)
{
        ASYNC_DOMAIN_EXCLUSIVE(async_domain);
        int i;
        int ret = 0;

        for (i = 0; i < num_consumers; i++) {
                async_schedule_domain(regulator_bulk_enable_async,
                                      &consumers[i], &async_domain);
        }

        async_synchronize_full_domain(&async_domain);

        /* If any consumer failed we need to unwind any that succeeded */
        for (i = 0; i < num_consumers; i++) {
                if (consumers[i].ret != 0) {
                        ret = consumers[i].ret;
                        goto err;
                }
        }

        return 0;

err:
        for (i = 0; i < num_consumers; i++) {
                if (consumers[i].ret < 0)
                        pr_err("Failed to enable %s: %pe\n", consumers[i].supply,
                               ERR_PTR(consumers[i].ret));
                else
                        regulator_disable(consumers[i].consumer);
        }

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_enable);

/**
 * regulator_bulk_disable - disable multiple regulator consumers
 *
 * @num_consumers: Number of consumers
 * @consumers:     Consumer data; clients are stored here.
 *
 * This convenience API allows consumers to disable multiple regulator
 * clients in a single API call.  If any consumers cannot be disabled
 * then any others that were disabled will be enabled again prior to
 * return.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_bulk_disable(int num_consumers,
                           struct regulator_bulk_data *consumers)
{
        int i;
        int ret, r;

        for (i = num_consumers - 1; i >= 0; --i) {
                ret = regulator_disable(consumers[i].consumer);
                if (ret != 0)
                        goto err;
        }

        return 0;

err:
        pr_err("Failed to disable %s: %pe\n", consumers[i].supply, ERR_PTR(ret));
        for (++i; i < num_consumers; ++i) {
                r = regulator_enable(consumers[i].consumer);
                if (r != 0)
                        pr_err("Failed to re-enable %s: %pe\n",
                               consumers[i].supply, ERR_PTR(r));
        }

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_disable);

/**
 * regulator_bulk_force_disable - force disable multiple regulator consumers
 *
 * @num_consumers: Number of consumers
 * @consumers:     Consumer data; clients are stored here.
 *
 * This convenience API allows consumers to forcibly disable multiple regulator
 * clients in a single API call.
 * NOTE: This should be used for situations when device damage will
 * likely occur if the regulators are not disabled (e.g. over temp).
 * Although regulator_force_disable function call for some consumers can
 * return error numbers, the function is called for all consumers.
 *
 * Return: 0 on success or a negative error number on failure.
 */
int regulator_bulk_force_disable(int num_consumers,
                           struct regulator_bulk_data *consumers)
{
        int i;
        int ret = 0;

        for (i = 0; i < num_consumers; i++) {
                consumers[i].ret =
                            regulator_force_disable(consumers[i].consumer);

                /* Store first error for reporting */
                if (consumers[i].ret && !ret)
                        ret = consumers[i].ret;
        }

        return ret;
}
EXPORT_SYMBOL_GPL(regulator_bulk_force_disable);

/**
 * regulator_bulk_free - free multiple regulator consumers
 *
 * @num_consumers: Number of consumers
 * @consumers:     Consumer data; clients are stored here.
 *
 * This convenience API allows consumers to free multiple regulator
 * clients in a single API call.
 */
void regulator_bulk_free(int num_consumers,
                         struct regulator_bulk_data *consumers)
{
        int i;

        for (i = 0; i < num_consumers; i++) {
                regulator_put(consumers[i].consumer);
                consumers[i].consumer = NULL;
        }
}
EXPORT_SYMBOL_GPL(regulator_bulk_free);

/**
 * regulator_handle_critical - Handle events for system-critical regulators.
 * @rdev: The regulator device.
 * @event: The event being handled.
 *
 * This function handles critical events such as under-voltage, over-current,
 * and unknown errors for regulators deemed system-critical. On detecting such
 * events, it triggers a hardware protection shutdown with a defined timeout.
 */
static void regulator_handle_critical(struct regulator_dev *rdev,
                                      unsigned long event)
{
        const char *reason = NULL;

        if (!rdev->constraints->system_critical)
                return;

        switch (event) {
        case REGULATOR_EVENT_UNDER_VOLTAGE:
                reason = "System critical regulator: voltage drop detected";
                break;
        case REGULATOR_EVENT_OVER_CURRENT:
                reason = "System critical regulator: over-current detected";
                break;
        case REGULATOR_EVENT_FAIL:
                reason = "System critical regulator: unknown error";
        }

        if (!reason)
                return;

        hw_protection_shutdown(reason,
                               rdev->constraints->uv_less_critical_window_ms);
}

/**
 * regulator_notifier_call_chain - call regulator event notifier
 * @rdev: regulator source
 * @event: notifier block
 * @data: callback-specific data.
 *
 * Called by regulator drivers to notify clients a regulator event has
 * occurred.
 *
 * Return: %NOTIFY_DONE.
 */
int regulator_notifier_call_chain(struct regulator_dev *rdev,
                                  unsigned long event, void *data)
{
        regulator_handle_critical(rdev, event);

        _notifier_call_chain(rdev, event, data);
        return NOTIFY_DONE;

}
EXPORT_SYMBOL_GPL(regulator_notifier_call_chain);

/**
 * regulator_mode_to_status - convert a regulator mode into a status
 *
 * @mode: Mode to convert
 *
 * Convert a regulator mode into a status.
 *
 * Return: %REGULATOR_STATUS_* value corresponding to given mode.
 */
int regulator_mode_to_status(unsigned int mode)
{
        switch (mode) {
        case REGULATOR_MODE_FAST:
                return REGULATOR_STATUS_FAST;
        case REGULATOR_MODE_NORMAL:
                return REGULATOR_STATUS_NORMAL;
        case REGULATOR_MODE_IDLE:
                return REGULATOR_STATUS_IDLE;
        case REGULATOR_MODE_STANDBY:
                return REGULATOR_STATUS_STANDBY;
        default:
                return REGULATOR_STATUS_UNDEFINED;
        }
}
EXPORT_SYMBOL_GPL(regulator_mode_to_status);

static struct attribute *regulator_dev_attrs[] = {
        &dev_attr_name.attr,
        &dev_attr_num_users.attr,
        &dev_attr_type.attr,
        &dev_attr_microvolts.attr,
        &dev_attr_microamps.attr,
        &dev_attr_opmode.attr,
        &dev_attr_state.attr,
        &dev_attr_status.attr,
        &dev_attr_bypass.attr,
        &dev_attr_requested_microamps.attr,
        &dev_attr_min_microvolts.attr,
        &dev_attr_max_microvolts.attr,
        &dev_attr_min_microamps.attr,
        &dev_attr_max_microamps.attr,
        &dev_attr_under_voltage.attr,
        &dev_attr_over_current.attr,
        &dev_attr_regulation_out.attr,
        &dev_attr_fail.attr,
        &dev_attr_over_temp.attr,
        &dev_attr_under_voltage_warn.attr,
        &dev_attr_over_current_warn.attr,
        &dev_attr_over_voltage_warn.attr,
        &dev_attr_over_temp_warn.attr,
        &dev_attr_suspend_standby_state.attr,
        &dev_attr_suspend_mem_state.attr,
        &dev_attr_suspend_disk_state.attr,
        &dev_attr_suspend_standby_microvolts.attr,
        &dev_attr_suspend_mem_microvolts.attr,
        &dev_attr_suspend_disk_microvolts.attr,
        &dev_attr_suspend_standby_mode.attr,
        &dev_attr_suspend_mem_mode.attr,
        &dev_attr_suspend_disk_mode.attr,
        &dev_attr_power_budget_milliwatt.attr,
        &dev_attr_power_requested_milliwatt.attr,
        NULL
};

/*
 * To avoid cluttering sysfs (and memory) with useless state, only
 * create attributes that can be meaningfully displayed.
 */
static umode_t regulator_attr_is_visible(struct kobject *kobj,
                                         struct attribute *attr, int idx)
{
        struct device *dev = kobj_to_dev(kobj);
        struct regulator_dev *rdev = dev_to_rdev(dev);
        const struct regulator_ops *ops = rdev->desc->ops;
        umode_t mode = attr->mode;

        /* these three are always present */
        if (attr == &dev_attr_name.attr ||
            attr == &dev_attr_num_users.attr ||
            attr == &dev_attr_type.attr)
                return mode;

        /* some attributes need specific methods to be displayed */
        if (attr == &dev_attr_microvolts.attr) {
                if ((ops->get_voltage && ops->get_voltage(rdev) >= 0) ||
                    (ops->get_voltage_sel && ops->get_voltage_sel(rdev) >= 0) ||
                    (ops->list_voltage && ops->list_voltage(rdev, 0) >= 0) ||
                    (rdev->desc->fixed_uV && rdev->desc->n_voltages == 1))
                        return mode;
                return 0;
        }

        if (attr == &dev_attr_microamps.attr)
                return ops->get_current_limit ? mode : 0;

        if (attr == &dev_attr_opmode.attr)
                return ops->get_mode ? mode : 0;

        if (attr == &dev_attr_state.attr)
                return (rdev->ena_pin || ops->is_enabled) ? mode : 0;

        if (attr == &dev_attr_status.attr)
                return ops->get_status ? mode : 0;

        if (attr == &dev_attr_bypass.attr)
                return ops->get_bypass ? mode : 0;

        if (attr == &dev_attr_under_voltage.attr ||
            attr == &dev_attr_over_current.attr ||
            attr == &dev_attr_regulation_out.attr ||
            attr == &dev_attr_fail.attr ||
            attr == &dev_attr_over_temp.attr ||
            attr == &dev_attr_under_voltage_warn.attr ||
            attr == &dev_attr_over_current_warn.attr ||
            attr == &dev_attr_over_voltage_warn.attr ||
            attr == &dev_attr_over_temp_warn.attr)
                return ops->get_error_flags ? mode : 0;

        /* constraints need specific supporting methods */
        if (attr == &dev_attr_min_microvolts.attr ||
            attr == &dev_attr_max_microvolts.attr)
                return (ops->set_voltage || ops->set_voltage_sel) ? mode : 0;

        if (attr == &dev_attr_min_microamps.attr ||
            attr == &dev_attr_max_microamps.attr)
                return ops->set_current_limit ? mode : 0;

        if (attr == &dev_attr_suspend_standby_state.attr ||
            attr == &dev_attr_suspend_mem_state.attr ||
            attr == &dev_attr_suspend_disk_state.attr)
                return mode;

        if (attr == &dev_attr_suspend_standby_microvolts.attr ||
            attr == &dev_attr_suspend_mem_microvolts.attr ||
            attr == &dev_attr_suspend_disk_microvolts.attr)
                return ops->set_suspend_voltage ? mode : 0;

        if (attr == &dev_attr_suspend_standby_mode.attr ||
            attr == &dev_attr_suspend_mem_mode.attr ||
            attr == &dev_attr_suspend_disk_mode.attr)
                return ops->set_suspend_mode ? mode : 0;

        if (attr == &dev_attr_power_budget_milliwatt.attr ||
            attr == &dev_attr_power_requested_milliwatt.attr)
                return rdev->constraints->pw_budget_mW != INT_MAX ? mode : 0;

        return mode;
}

static const struct attribute_group regulator_dev_group = {
        .attrs = regulator_dev_attrs,
        .is_visible = regulator_attr_is_visible,
};

static const struct attribute_group *regulator_dev_groups[] = {
        &regulator_dev_group,
        NULL
};

static void regulator_dev_release(struct device *dev)
{
        struct regulator_dev *rdev = dev_get_drvdata(dev);

        debugfs_remove_recursive(rdev->debugfs);
        kfree(rdev->constraints);
        of_node_put(rdev->dev.of_node);
        kfree(rdev);
}

static void rdev_init_debugfs(struct regulator_dev *rdev)
{
        struct device *parent = rdev->dev.parent;
        const char *rname = rdev_get_name(rdev);
        char name[NAME_MAX];

        /* Avoid duplicate debugfs directory names */
        if (parent && rname == rdev->desc->name) {
                snprintf(name, sizeof(name), "%s-%s", dev_name(parent),
                         rname);
                rname = name;
        }

        rdev->debugfs = debugfs_create_dir(rname, debugfs_root);
        if (IS_ERR(rdev->debugfs))
                rdev_dbg(rdev, "Failed to create debugfs directory\n");

        debugfs_create_u32("use_count", 0444, rdev->debugfs,
                           &rdev->use_count);
        debugfs_create_u32("open_count", 0444, rdev->debugfs,
                           &rdev->open_count);
        debugfs_create_u32("bypass_count", 0444, rdev->debugfs,
                           &rdev->bypass_count);
}

static int regulator_register_resolve_supply(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);

        if (regulator_resolve_supply(rdev))
                rdev_dbg(rdev, "unable to resolve supply\n");

        return 0;
}

int regulator_coupler_register(struct regulator_coupler *coupler)
{
        mutex_lock(&regulator_list_mutex);
        list_add_tail(&coupler->list, &regulator_coupler_list);
        mutex_unlock(&regulator_list_mutex);

        return 0;
}

static struct regulator_coupler *
regulator_find_coupler(struct regulator_dev *rdev)
{
        struct regulator_coupler *coupler;
        int err;

        /*
         * Note that regulators are appended to the list and the generic
         * coupler is registered first, hence it will be attached at last
         * if nobody cared.
         */
        list_for_each_entry_reverse(coupler, &regulator_coupler_list, list) {
                err = coupler->attach_regulator(coupler, rdev);
                if (!err) {
                        if (!coupler->balance_voltage &&
                            rdev->coupling_desc.n_coupled > 2)
                                goto err_unsupported;

                        return coupler;
                }

                if (err < 0)
                        return ERR_PTR(err);

                if (err == 1)
                        continue;

                break;
        }

        return ERR_PTR(-EINVAL);

err_unsupported:
        if (coupler->detach_regulator)
                coupler->detach_regulator(coupler, rdev);

        rdev_err(rdev,
                "Voltage balancing for multiple regulator couples is unimplemented\n");

        return ERR_PTR(-EPERM);
}

static void regulator_resolve_coupling(struct regulator_dev *rdev)
{
        struct regulator_coupler *coupler = rdev->coupling_desc.coupler;
        struct coupling_desc *c_desc = &rdev->coupling_desc;
        int n_coupled = c_desc->n_coupled;
        struct regulator_dev *c_rdev;
        int i;

        for (i = 1; i < n_coupled; i++) {
                /* already resolved */
                if (c_desc->coupled_rdevs[i])
                        continue;

                c_rdev = of_parse_coupled_regulator(rdev, i - 1);

                if (!c_rdev)
                        continue;

                if (c_rdev->coupling_desc.coupler != coupler) {
                        rdev_err(rdev, "coupler mismatch with %s\n",
                                 rdev_get_name(c_rdev));
                        return;
                }

                c_desc->coupled_rdevs[i] = c_rdev;
                c_desc->n_resolved++;

                regulator_resolve_coupling(c_rdev);
        }
}

static void regulator_remove_coupling(struct regulator_dev *rdev)
{
        struct regulator_coupler *coupler = rdev->coupling_desc.coupler;
        struct coupling_desc *__c_desc, *c_desc = &rdev->coupling_desc;
        struct regulator_dev *__c_rdev, *c_rdev;
        unsigned int __n_coupled, n_coupled;
        int i, k;
        int err;

        n_coupled = c_desc->n_coupled;

        for (i = 1; i < n_coupled; i++) {
                c_rdev = c_desc->coupled_rdevs[i];

                if (!c_rdev)
                        continue;

                regulator_lock(c_rdev);

                __c_desc = &c_rdev->coupling_desc;
                __n_coupled = __c_desc->n_coupled;

                for (k = 1; k < __n_coupled; k++) {
                        __c_rdev = __c_desc->coupled_rdevs[k];

                        if (__c_rdev == rdev) {
                                __c_desc->coupled_rdevs[k] = NULL;
                                __c_desc->n_resolved--;
                                break;
                        }
                }

                regulator_unlock(c_rdev);

                c_desc->coupled_rdevs[i] = NULL;
                c_desc->n_resolved--;
        }

        if (coupler && coupler->detach_regulator) {
                err = coupler->detach_regulator(coupler, rdev);
                if (err)
                        rdev_err(rdev, "failed to detach from coupler: %pe\n",
                                 ERR_PTR(err));
        }

        kfree(rdev->coupling_desc.coupled_rdevs);
        rdev->coupling_desc.coupled_rdevs = NULL;
}

static int regulator_init_coupling(struct regulator_dev *rdev)
{
        struct regulator_dev **coupled;
        int err, n_phandles;

        if (!IS_ENABLED(CONFIG_OF))
                n_phandles = 0;
        else
                n_phandles = of_get_n_coupled(rdev);

        coupled = kcalloc(n_phandles + 1, sizeof(*coupled), GFP_KERNEL);
        if (!coupled)
                return -ENOMEM;

        rdev->coupling_desc.coupled_rdevs = coupled;

        /*
         * Every regulator should always have coupling descriptor filled with
         * at least pointer to itself.
         */
        rdev->coupling_desc.coupled_rdevs[0] = rdev;
        rdev->coupling_desc.n_coupled = n_phandles + 1;
        rdev->coupling_desc.n_resolved++;

        /* regulator isn't coupled */
        if (n_phandles == 0)
                return 0;

        if (!of_check_coupling_data(rdev))
                return -EPERM;

        mutex_lock(&regulator_list_mutex);
        rdev->coupling_desc.coupler = regulator_find_coupler(rdev);
        mutex_unlock(&regulator_list_mutex);

        if (IS_ERR(rdev->coupling_desc.coupler)) {
                err = PTR_ERR(rdev->coupling_desc.coupler);
                rdev_err(rdev, "failed to get coupler: %pe\n", ERR_PTR(err));
                return err;
        }

        return 0;
}

static int generic_coupler_attach(struct regulator_coupler *coupler,
                                  struct regulator_dev *rdev)
{
        if (rdev->coupling_desc.n_coupled > 2) {
                rdev_err(rdev,
                         "Voltage balancing for multiple regulator couples is unimplemented\n");
                return -EPERM;
        }

        if (!rdev->constraints->always_on) {
                rdev_err(rdev,
                         "Coupling of a non always-on regulator is unimplemented\n");
                return -ENOTSUPP;
        }

        return 0;
}

static struct regulator_coupler generic_regulator_coupler = {
        .attach_regulator = generic_coupler_attach,
};

/**
 * regulator_register - register regulator
 * @dev: the device that drive the regulator
 * @regulator_desc: regulator to register
 * @cfg: runtime configuration for regulator
 *
 * Called by regulator drivers to register a regulator.
 *
 * Return: Pointer to a valid &struct regulator_dev on success or
 *           an ERR_PTR() encoded negative error number on failure.
 */
struct regulator_dev *
regulator_register(struct device *dev,
                   const struct regulator_desc *regulator_desc,
                   const struct regulator_config *cfg)
{
        const struct regulator_init_data *init_data;
        struct regulator_config *config = NULL;
        static atomic_t regulator_no = ATOMIC_INIT(-1);
        struct regulator_dev *rdev;
        bool dangling_cfg_gpiod = false;
        bool dangling_of_gpiod = false;
        int ret, i;
        bool resolved_early = false;

        if (cfg == NULL)
                return ERR_PTR(-EINVAL);
        if (cfg->ena_gpiod)
                dangling_cfg_gpiod = true;
        if (regulator_desc == NULL) {
                ret = -EINVAL;
                goto rinse;
        }

        WARN_ON(!dev || !cfg->dev);

        if (regulator_desc->name == NULL || regulator_desc->ops == NULL) {
                ret = -EINVAL;
                goto rinse;
        }

        if (regulator_desc->type != REGULATOR_VOLTAGE &&
            regulator_desc->type != REGULATOR_CURRENT) {
                ret = -EINVAL;
                goto rinse;
        }

        /* Only one of each should be implemented */
        WARN_ON(regulator_desc->ops->get_voltage &&
                regulator_desc->ops->get_voltage_sel);
        WARN_ON(regulator_desc->ops->set_voltage &&
                regulator_desc->ops->set_voltage_sel);

        /* If we're using selectors we must implement list_voltage. */
        if (regulator_desc->ops->get_voltage_sel &&
            !regulator_desc->ops->list_voltage) {
                ret = -EINVAL;
                goto rinse;
        }
        if (regulator_desc->ops->set_voltage_sel &&
            !regulator_desc->ops->list_voltage) {
                ret = -EINVAL;
                goto rinse;
        }

        rdev = kzalloc(sizeof(struct regulator_dev), GFP_KERNEL);
        if (rdev == NULL) {
                ret = -ENOMEM;
                goto rinse;
        }
        device_initialize(&rdev->dev);
        dev_set_drvdata(&rdev->dev, rdev);
        rdev->dev.class = &regulator_class;
        spin_lock_init(&rdev->err_lock);

        /*
         * Duplicate the config so the driver could override it after
         * parsing init data.
         */
        config = kmemdup(cfg, sizeof(*cfg), GFP_KERNEL);
        if (config == NULL) {
                ret = -ENOMEM;
                goto clean;
        }

        /*
         * DT may override the config->init_data provided if the platform
         * needs to do so. If so, config->init_data is completely ignored.
         */
        init_data = regulator_of_get_init_data(dev, regulator_desc, config,
                                               &rdev->dev.of_node);

        /*
         * Sometimes not all resources are probed already so we need to take
         * that into account. This happens most the time if the ena_gpiod comes
         * from a gpio extender or something else.
         */
        if (PTR_ERR(init_data) == -EPROBE_DEFER) {
                ret = -EPROBE_DEFER;
                goto clean;
        }

        /*
         * We need to keep track of any GPIO descriptor coming from the
         * device tree until we have handled it over to the core. If the
         * config that was passed in to this function DOES NOT contain
         * a descriptor, and the config after this call DOES contain
         * a descriptor, we definitely got one from parsing the device
         * tree.
         */
        if (!cfg->ena_gpiod && config->ena_gpiod)
                dangling_of_gpiod = true;
        if (!init_data) {
                init_data = config->init_data;
                rdev->dev.of_node = of_node_get(config->of_node);
        }

        ww_mutex_init(&rdev->mutex, &regulator_ww_class);
        rdev->reg_data = config->driver_data;
        rdev->owner = regulator_desc->owner;
        rdev->desc = regulator_desc;
        if (config->regmap)
                rdev->regmap = config->regmap;
        else if (dev_get_regmap(dev, NULL))
                rdev->regmap = dev_get_regmap(dev, NULL);
        else if (dev->parent)
                rdev->regmap = dev_get_regmap(dev->parent, NULL);
        INIT_LIST_HEAD(&rdev->consumer_list);
        INIT_LIST_HEAD(&rdev->list);
        BLOCKING_INIT_NOTIFIER_HEAD(&rdev->notifier);
        INIT_DELAYED_WORK(&rdev->disable_work, regulator_disable_work);

        if (init_data && init_data->supply_regulator)
                rdev->supply_name = init_data->supply_regulator;
        else if (regulator_desc->supply_name)
                rdev->supply_name = regulator_desc->supply_name;

        /* register with sysfs */
        rdev->dev.parent = config->dev;
        dev_set_name(&rdev->dev, "regulator.%lu",
                    (unsigned long) atomic_inc_return(&regulator_no));

        /* set regulator constraints */
        if (init_data)
                rdev->constraints = kmemdup(&init_data->constraints,
                                            sizeof(*rdev->constraints),
                                            GFP_KERNEL);
        else
                rdev->constraints = kzalloc(sizeof(*rdev->constraints),
                                            GFP_KERNEL);
        if (!rdev->constraints) {
                ret = -ENOMEM;
                goto wash;
        }

        if (regulator_desc->init_cb) {
                ret = regulator_desc->init_cb(rdev, config);
                if (ret < 0)
                        goto wash;
        }

        if ((rdev->supply_name && !rdev->supply) &&
                (rdev->constraints->always_on ||
                 rdev->constraints->boot_on)) {
                ret = regulator_resolve_supply(rdev);
                if (ret)
                        rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
                                         ERR_PTR(ret));

                resolved_early = true;
        }

        if (config->ena_gpiod) {
                ret = regulator_ena_gpio_request(rdev, config);
                if (ret != 0) {
                        rdev_err(rdev, "Failed to request enable GPIO: %pe\n",
                                 ERR_PTR(ret));
                        goto wash;
                }
                /* The regulator core took over the GPIO descriptor */
                dangling_cfg_gpiod = false;
                dangling_of_gpiod = false;
        }

        ret = set_machine_constraints(rdev);
        if (ret == -EPROBE_DEFER && !resolved_early) {
                /* Regulator might be in bypass mode and so needs its supply
                 * to set the constraints
                 */
                /* FIXME: this currently triggers a chicken-and-egg problem
                 * when creating -SUPPLY symlink in sysfs to a regulator
                 * that is just being created
                 */
                rdev_dbg(rdev, "will resolve supply early: %s\n",
                         rdev->supply_name);
                ret = regulator_resolve_supply(rdev);
                if (!ret)
                        ret = set_machine_constraints(rdev);
                else
                        rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
                                 ERR_PTR(ret));
        }
        if (ret < 0)
                goto wash;

        ret = regulator_init_coupling(rdev);
        if (ret < 0)
                goto wash;

        /* add consumers devices */
        if (init_data) {
                for (i = 0; i < init_data->num_consumer_supplies; i++) {
                        ret = set_consumer_device_supply(rdev,
                                init_data->consumer_supplies[i].dev_name,
                                init_data->consumer_supplies[i].supply);
                        if (ret < 0) {
                                dev_err(dev, "Failed to set supply %s\n",
                                        init_data->consumer_supplies[i].supply);
                                goto unset_supplies;
                        }
                }
        }

        if (!rdev->desc->ops->get_voltage &&
            !rdev->desc->ops->list_voltage &&
            !rdev->desc->fixed_uV)
                rdev->is_switch = true;

        ret = device_add(&rdev->dev);
        if (ret != 0)
                goto unset_supplies;

        rdev_init_debugfs(rdev);

        /* try to resolve regulators coupling since a new one was registered */
        mutex_lock(&regulator_list_mutex);
        regulator_resolve_coupling(rdev);
        mutex_unlock(&regulator_list_mutex);

        /* try to resolve regulators supply since a new one was registered */
        class_for_each_device(&regulator_class, NULL, NULL,
                              regulator_register_resolve_supply);
        kfree(config);
        return rdev;

unset_supplies:
        mutex_lock(&regulator_list_mutex);
        unset_regulator_supplies(rdev);
        regulator_remove_coupling(rdev);
        mutex_unlock(&regulator_list_mutex);
wash:
        regulator_put(rdev->supply);
        kfree(rdev->coupling_desc.coupled_rdevs);
        mutex_lock(&regulator_list_mutex);
        regulator_ena_gpio_free(rdev);
        mutex_unlock(&regulator_list_mutex);
clean:
        if (dangling_of_gpiod)
                gpiod_put(config->ena_gpiod);
        kfree(config);
        put_device(&rdev->dev);
rinse:
        if (dangling_cfg_gpiod)
                gpiod_put(cfg->ena_gpiod);
        return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(regulator_register);

/**
 * regulator_unregister - unregister regulator
 * @rdev: regulator to unregister
 *
 * Called by regulator drivers to unregister a regulator.
 */
void regulator_unregister(struct regulator_dev *rdev)
{
        if (rdev == NULL)
                return;

        if (rdev->supply) {
                while (rdev->use_count--)
                        regulator_disable(rdev->supply);
                regulator_put(rdev->supply);
        }

        flush_work(&rdev->disable_work.work);

        mutex_lock(&regulator_list_mutex);

        WARN_ON(rdev->open_count);
        regulator_remove_coupling(rdev);
        unset_regulator_supplies(rdev);
        list_del(&rdev->list);
        regulator_ena_gpio_free(rdev);
        device_unregister(&rdev->dev);

        mutex_unlock(&regulator_list_mutex);
}
EXPORT_SYMBOL_GPL(regulator_unregister);

#ifdef CONFIG_SUSPEND
/**
 * regulator_suspend - prepare regulators for system wide suspend
 * @dev: ``&struct device`` pointer that is passed to _regulator_suspend()
 *
 * Configure each regulator with it's suspend operating parameters for state.
 *
 * Return: 0 on success or a negative error number on failure.
 */
static int regulator_suspend(struct device *dev)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        suspend_state_t state = pm_suspend_target_state;
        int ret;
        const struct regulator_state *rstate;

        rstate = regulator_get_suspend_state_check(rdev, state);
        if (!rstate)
                return 0;

        regulator_lock(rdev);
        ret = __suspend_set_state(rdev, rstate);
        regulator_unlock(rdev);

        return ret;
}

static int regulator_resume(struct device *dev)
{
        suspend_state_t state = pm_suspend_target_state;
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct regulator_state *rstate;
        int ret = 0;

        rstate = regulator_get_suspend_state(rdev, state);
        if (rstate == NULL)
                return 0;

        /* Avoid grabbing the lock if we don't need to */
        if (!rdev->desc->ops->resume)
                return 0;

        regulator_lock(rdev);

        if (rstate->enabled == ENABLE_IN_SUSPEND ||
            rstate->enabled == DISABLE_IN_SUSPEND)
                ret = rdev->desc->ops->resume(rdev);

        regulator_unlock(rdev);

        return ret;
}
#else /* !CONFIG_SUSPEND */

#define regulator_suspend        NULL
#define regulator_resume        NULL

#endif /* !CONFIG_SUSPEND */

#ifdef CONFIG_PM
static const struct dev_pm_ops __maybe_unused regulator_pm_ops = {
        .suspend        = regulator_suspend,
        .resume                = regulator_resume,
};
#endif

const struct class regulator_class = {
        .name = "regulator",
        .dev_release = regulator_dev_release,
        .dev_groups = regulator_dev_groups,
#ifdef CONFIG_PM
        .pm = &regulator_pm_ops,
#endif
};
/**
 * regulator_has_full_constraints - the system has fully specified constraints
 *
 * Calling this function will cause the regulator API to disable all
 * regulators which have a zero use count and don't have an always_on
 * constraint in a late_initcall.
 *
 * The intention is that this will become the default behaviour in a
 * future kernel release so users are encouraged to use this facility
 * now.
 */
void regulator_has_full_constraints(void)
{
        has_full_constraints = 1;
}
EXPORT_SYMBOL_GPL(regulator_has_full_constraints);

/**
 * rdev_get_drvdata - get rdev regulator driver data
 * @rdev: regulator
 *
 * Get rdev regulator driver private data. This call can be used in the
 * regulator driver context.
 *
 * Return: Pointer to regulator driver private data.
 */
void *rdev_get_drvdata(struct regulator_dev *rdev)
{
        return rdev->reg_data;
}
EXPORT_SYMBOL_GPL(rdev_get_drvdata);

/**
 * regulator_get_drvdata - get regulator driver data
 * @regulator: regulator
 *
 * Get regulator driver private data. This call can be used in the consumer
 * driver context when non API regulator specific functions need to be called.
 *
 * Return: Pointer to regulator driver private data.
 */
void *regulator_get_drvdata(struct regulator *regulator)
{
        return regulator->rdev->reg_data;
}
EXPORT_SYMBOL_GPL(regulator_get_drvdata);

/**
 * regulator_set_drvdata - set regulator driver data
 * @regulator: regulator
 * @data: data
 */
void regulator_set_drvdata(struct regulator *regulator, void *data)
{
        regulator->rdev->reg_data = data;
}
EXPORT_SYMBOL_GPL(regulator_set_drvdata);

/**
 * rdev_get_id - get regulator ID
 * @rdev: regulator
 *
 * Return: Regulator ID for @rdev.
 */
int rdev_get_id(struct regulator_dev *rdev)
{
        return rdev->desc->id;
}
EXPORT_SYMBOL_GPL(rdev_get_id);

struct device *rdev_get_dev(struct regulator_dev *rdev)
{
        return &rdev->dev;
}
EXPORT_SYMBOL_GPL(rdev_get_dev);

struct regmap *rdev_get_regmap(struct regulator_dev *rdev)
{
        return rdev->regmap;
}
EXPORT_SYMBOL_GPL(rdev_get_regmap);

void *regulator_get_init_drvdata(struct regulator_init_data *reg_init_data)
{
        return reg_init_data->driver_data;
}
EXPORT_SYMBOL_GPL(regulator_get_init_drvdata);

#ifdef CONFIG_DEBUG_FS
static int supply_map_show(struct seq_file *sf, void *data)
{
        struct regulator_map *map;

        list_for_each_entry(map, &regulator_map_list, list) {
                seq_printf(sf, "%s -> %s.%s\n",
                                rdev_get_name(map->regulator), map->dev_name,
                                map->supply);
        }

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(supply_map);

struct summary_data {
        struct seq_file *s;
        struct regulator_dev *parent;
        int level;
};

static void regulator_summary_show_subtree(struct seq_file *s,
                                           struct regulator_dev *rdev,
                                           int level);

static int regulator_summary_show_children(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct summary_data *summary_data = data;

        if (rdev->supply && rdev->supply->rdev == summary_data->parent)
                regulator_summary_show_subtree(summary_data->s, rdev,
                                               summary_data->level + 1);

        return 0;
}

static void regulator_summary_show_subtree(struct seq_file *s,
                                           struct regulator_dev *rdev,
                                           int level)
{
        struct regulation_constraints *c;
        struct regulator *consumer;
        struct summary_data summary_data;
        unsigned int opmode;

        if (!rdev)
                return;

        opmode = _regulator_get_mode_unlocked(rdev);
        seq_printf(s, "%*s%-*s %3d %4d %6d %7s ",
                   level * 3 + 1, "",
                   30 - level * 3, rdev_get_name(rdev),
                   rdev->use_count, rdev->open_count, rdev->bypass_count,
                   regulator_opmode_to_str(opmode));

        seq_printf(s, "%5dmV ", regulator_get_voltage_rdev(rdev) / 1000);
        seq_printf(s, "%5dmA ",
                   _regulator_get_current_limit_unlocked(rdev) / 1000);

        c = rdev->constraints;
        if (c) {
                switch (rdev->desc->type) {
                case REGULATOR_VOLTAGE:
                        seq_printf(s, "%5dmV %5dmV ",
                                   c->min_uV / 1000, c->max_uV / 1000);
                        break;
                case REGULATOR_CURRENT:
                        seq_printf(s, "%5dmA %5dmA ",
                                   c->min_uA / 1000, c->max_uA / 1000);
                        break;
                }
        }

        seq_puts(s, "\n");

        list_for_each_entry(consumer, &rdev->consumer_list, list) {
                if (consumer->dev && consumer->dev->class == &regulator_class)
                        continue;

                seq_printf(s, "%*s%-*s ",
                           (level + 1) * 3 + 1, "",
                           30 - (level + 1) * 3,
                           consumer->supply_name ? consumer->supply_name :
                           consumer->dev ? dev_name(consumer->dev) : "deviceless");

                switch (rdev->desc->type) {
                case REGULATOR_VOLTAGE:
                        seq_printf(s, "%3d %33dmA%c%5dmV %5dmV",
                                   consumer->enable_count,
                                   consumer->uA_load / 1000,
                                   consumer->uA_load && !consumer->enable_count ?
                                   '*' : ' ',
                                   consumer->voltage[PM_SUSPEND_ON].min_uV / 1000,
                                   consumer->voltage[PM_SUSPEND_ON].max_uV / 1000);
                        break;
                case REGULATOR_CURRENT:
                        break;
                }

                seq_puts(s, "\n");
        }

        summary_data.s = s;
        summary_data.level = level;
        summary_data.parent = rdev;

        class_for_each_device(&regulator_class, NULL, &summary_data,
                              regulator_summary_show_children);
}

struct summary_lock_data {
        struct ww_acquire_ctx *ww_ctx;
        struct regulator_dev **new_contended_rdev;
        struct regulator_dev **old_contended_rdev;
};

static int regulator_summary_lock_one(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct summary_lock_data *lock_data = data;
        int ret = 0;

        if (rdev != *lock_data->old_contended_rdev) {
                ret = regulator_lock_nested(rdev, lock_data->ww_ctx);

                if (ret == -EDEADLK)
                        *lock_data->new_contended_rdev = rdev;
                else
                        WARN_ON_ONCE(ret);
        } else {
                *lock_data->old_contended_rdev = NULL;
        }

        return ret;
}

static int regulator_summary_unlock_one(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct summary_lock_data *lock_data = data;

        if (lock_data) {
                if (rdev == *lock_data->new_contended_rdev)
                        return -EDEADLK;
        }

        regulator_unlock(rdev);

        return 0;
}

static int regulator_summary_lock_all(struct ww_acquire_ctx *ww_ctx,
                                      struct regulator_dev **new_contended_rdev,
                                      struct regulator_dev **old_contended_rdev)
{
        struct summary_lock_data lock_data;
        int ret;

        lock_data.ww_ctx = ww_ctx;
        lock_data.new_contended_rdev = new_contended_rdev;
        lock_data.old_contended_rdev = old_contended_rdev;

        ret = class_for_each_device(&regulator_class, NULL, &lock_data,
                                    regulator_summary_lock_one);
        if (ret)
                class_for_each_device(&regulator_class, NULL, &lock_data,
                                      regulator_summary_unlock_one);

        return ret;
}

static void regulator_summary_lock(struct ww_acquire_ctx *ww_ctx)
{
        struct regulator_dev *new_contended_rdev = NULL;
        struct regulator_dev *old_contended_rdev = NULL;
        int err;

        mutex_lock(&regulator_list_mutex);

        ww_acquire_init(ww_ctx, &regulator_ww_class);

        do {
                if (new_contended_rdev) {
                        ww_mutex_lock_slow(&new_contended_rdev->mutex, ww_ctx);
                        old_contended_rdev = new_contended_rdev;
                        old_contended_rdev->ref_cnt++;
                        old_contended_rdev->mutex_owner = current;
                }

                err = regulator_summary_lock_all(ww_ctx,
                                                 &new_contended_rdev,
                                                 &old_contended_rdev);

                if (old_contended_rdev)
                        regulator_unlock(old_contended_rdev);

        } while (err == -EDEADLK);

        ww_acquire_done(ww_ctx);
}

static void regulator_summary_unlock(struct ww_acquire_ctx *ww_ctx)
{
        class_for_each_device(&regulator_class, NULL, NULL,
                              regulator_summary_unlock_one);
        ww_acquire_fini(ww_ctx);

        mutex_unlock(&regulator_list_mutex);
}

static int regulator_summary_show_roots(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct seq_file *s = data;

        if (!rdev->supply)
                regulator_summary_show_subtree(s, rdev, 0);

        return 0;
}

static int regulator_summary_show(struct seq_file *s, void *data)
{
        struct ww_acquire_ctx ww_ctx;

        seq_puts(s, " regulator                      use open bypass  opmode voltage current     min     max\n");
        seq_puts(s, "---------------------------------------------------------------------------------------\n");

        regulator_summary_lock(&ww_ctx);

        class_for_each_device(&regulator_class, NULL, s,
                              regulator_summary_show_roots);

        regulator_summary_unlock(&ww_ctx);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(regulator_summary);
#endif /* CONFIG_DEBUG_FS */

static int __init regulator_init(void)
{
        int ret;

        ret = class_register(&regulator_class);

        debugfs_root = debugfs_create_dir("regulator", NULL);
        if (IS_ERR(debugfs_root))
                pr_debug("regulator: Failed to create debugfs directory\n");

#ifdef CONFIG_DEBUG_FS
        debugfs_create_file("supply_map", 0444, debugfs_root, NULL,
                            &supply_map_fops);

        debugfs_create_file("regulator_summary", 0444, debugfs_root,
                            NULL, &regulator_summary_fops);
#endif
        regulator_dummy_init();

        regulator_coupler_register(&generic_regulator_coupler);

        return ret;
}

/* init early to allow our consumers to complete system booting */
core_initcall(regulator_init);

static int regulator_late_cleanup(struct device *dev, void *data)
{
        struct regulator_dev *rdev = dev_to_rdev(dev);
        struct regulation_constraints *c = rdev->constraints;
        int ret;

        if (c && c->always_on)
                return 0;

        if (!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS))
                return 0;

        regulator_lock(rdev);

        if (rdev->use_count)
                goto unlock;

        /* If reading the status failed, assume that it's off. */
        if (_regulator_is_enabled(rdev) <= 0)
                goto unlock;

        if (have_full_constraints()) {
                /* We log since this may kill the system if it goes
                 * wrong.
                 */
                rdev_info(rdev, "disabling\n");
                ret = _regulator_do_disable(rdev);
                if (ret != 0)
                        rdev_err(rdev, "couldn't disable: %pe\n", ERR_PTR(ret));
        } else {
                /* The intention is that in future we will
                 * assume that full constraints are provided
                 * so warn even if we aren't going to do
                 * anything here.
                 */
                rdev_warn(rdev, "incomplete constraints, leaving on\n");
        }

unlock:
        regulator_unlock(rdev);

        return 0;
}

static bool regulator_ignore_unused;
static int __init regulator_ignore_unused_setup(char *__unused)
{
        regulator_ignore_unused = true;
        return 1;
}
__setup("regulator_ignore_unused", regulator_ignore_unused_setup);

static void regulator_init_complete_work_function(struct work_struct *work)
{
        /*
         * Regulators may had failed to resolve their input supplies
         * when were registered, either because the input supply was
         * not registered yet or because its parent device was not
         * bound yet. So attempt to resolve the input supplies for
         * pending regulators before trying to disable unused ones.
         */
        class_for_each_device(&regulator_class, NULL, NULL,
                              regulator_register_resolve_supply);

        /*
         * For debugging purposes, it may be useful to prevent unused
         * regulators from being disabled.
         */
        if (regulator_ignore_unused) {
                pr_warn("regulator: Not disabling unused regulators\n");
                return;
        }

        /* If we have a full configuration then disable any regulators
         * we have permission to change the status for and which are
         * not in use or always_on.  This is effectively the default
         * for DT and ACPI as they have full constraints.
         */
        class_for_each_device(&regulator_class, NULL, NULL,
                              regulator_late_cleanup);
}

static DECLARE_DELAYED_WORK(regulator_init_complete_work,
                            regulator_init_complete_work_function);

static int __init regulator_init_complete(void)
{
        /*
         * Since DT doesn't provide an idiomatic mechanism for
         * enabling full constraints and since it's much more natural
         * with DT to provide them just assume that a DT enabled
         * system has full constraints.
         */
        if (of_have_populated_dt())
                has_full_constraints = true;

        /*
         * We punt completion for an arbitrary amount of time since
         * systems like distros will load many drivers from userspace
         * so consumers might not always be ready yet, this is
         * particularly an issue with laptops where this might bounce
         * the display off then on.  Ideally we'd get a notification
         * from userspace when this happens but we don't so just wait
         * a bit and hope we waited long enough.  It'd be better if
         * we'd only do this on systems that need it, and a kernel
         * command line option might be useful.
         */
        schedule_delayed_work(&regulator_init_complete_work,
                              msecs_to_jiffies(30000));

        return 0;
}
late_initcall_sync(regulator_init_complete);
















  252 





















  252 

























 4849 


























































































































































































































































































   64 
























  284 










































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM sched

#if !defined(_TRACE_SCHED_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SCHED_H

#include <linux/kthread.h>
#include <linux/sched/numa_balancing.h>
#include <linux/tracepoint.h>
#include <linux/binfmts.h>

/*
 * Tracepoint for calling kthread_stop, performed to end a kthread:
 */
TRACE_EVENT(sched_kthread_stop,

        TP_PROTO(struct task_struct *t),

        TP_ARGS(t),

        TP_STRUCT__entry(
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, t->comm, TASK_COMM_LEN);
                __entry->pid        = t->pid;
        ),

        TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
);

/*
 * Tracepoint for the return value of the kthread stopping:
 */
TRACE_EVENT(sched_kthread_stop_ret,

        TP_PROTO(int ret),

        TP_ARGS(ret),

        TP_STRUCT__entry(
                __field(        int,        ret        )
        ),

        TP_fast_assign(
                __entry->ret        = ret;
        ),

        TP_printk("ret=%d", __entry->ret)
);

/**
 * sched_kthread_work_queue_work - called when a work gets queued
 * @worker:        pointer to the kthread_worker
 * @work:        pointer to struct kthread_work
 *
 * This event occurs when a work is queued immediately or once a
 * delayed work is actually queued (ie: once the delay has been
 * reached).
 */
TRACE_EVENT(sched_kthread_work_queue_work,

        TP_PROTO(struct kthread_worker *worker,
                 struct kthread_work *work),

        TP_ARGS(worker, work),

        TP_STRUCT__entry(
                __field( void *,        work        )
                __field( void *,        function)
                __field( void *,        worker)
        ),

        TP_fast_assign(
                __entry->work                = work;
                __entry->function        = work->func;
                __entry->worker                = worker;
        ),

        TP_printk("work struct=%p function=%ps worker=%p",
                  __entry->work, __entry->function, __entry->worker)
);

/**
 * sched_kthread_work_execute_start - called immediately before the work callback
 * @work:        pointer to struct kthread_work
 *
 * Allows to track kthread work execution.
 */
TRACE_EVENT(sched_kthread_work_execute_start,

        TP_PROTO(struct kthread_work *work),

        TP_ARGS(work),

        TP_STRUCT__entry(
                __field( void *,        work        )
                __field( void *,        function)
        ),

        TP_fast_assign(
                __entry->work                = work;
                __entry->function        = work->func;
        ),

        TP_printk("work struct %p: function %ps", __entry->work, __entry->function)
);

/**
 * sched_kthread_work_execute_end - called immediately after the work callback
 * @work:        pointer to struct work_struct
 * @function:   pointer to worker function
 *
 * Allows to track workqueue execution.
 */
TRACE_EVENT(sched_kthread_work_execute_end,

        TP_PROTO(struct kthread_work *work, kthread_work_func_t function),

        TP_ARGS(work, function),

        TP_STRUCT__entry(
                __field( void *,        work        )
                __field( void *,        function)
        ),

        TP_fast_assign(
                __entry->work                = work;
                __entry->function        = function;
        ),

        TP_printk("work struct %p: function %ps", __entry->work, __entry->function)
);

/*
 * Tracepoint for waking up a task:
 */
DECLARE_EVENT_CLASS(sched_wakeup_template,

        TP_PROTO(struct task_struct *p),

        TP_ARGS(__perf_task(p)),

        TP_STRUCT__entry(
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
                __field(        int,        prio                        )
                __field(        int,        target_cpu                )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
                __entry->pid                = p->pid;
                __entry->prio                = p->prio; /* XXX SCHED_DEADLINE */
                __entry->target_cpu        = task_cpu(p);
        ),

        TP_printk("comm=%s pid=%d prio=%d target_cpu=%03d",
                  __entry->comm, __entry->pid, __entry->prio,
                  __entry->target_cpu)
);

/*
 * Tracepoint called when waking a task; this tracepoint is guaranteed to be
 * called from the waking context.
 */
DEFINE_EVENT(sched_wakeup_template, sched_waking,
             TP_PROTO(struct task_struct *p),
             TP_ARGS(p));

/*
 * Tracepoint called when the task is actually woken; p->state == TASK_RUNNING.
 * It is not always called from the waking context.
 */
DEFINE_EVENT(sched_wakeup_template, sched_wakeup,
             TP_PROTO(struct task_struct *p),
             TP_ARGS(p));

/*
 * Tracepoint for waking up a new task:
 */
DEFINE_EVENT(sched_wakeup_template, sched_wakeup_new,
             TP_PROTO(struct task_struct *p),
             TP_ARGS(p));

#ifdef CREATE_TRACE_POINTS
static inline long __trace_sched_switch_state(bool preempt,
                                              unsigned int prev_state,
                                              struct task_struct *p)
{
        unsigned int state;

#ifdef CONFIG_SCHED_DEBUG
        BUG_ON(p != current);
#endif /* CONFIG_SCHED_DEBUG */

        /*
         * Preemption ignores task state, therefore preempted tasks are always
         * RUNNING (we will not have dequeued if state != RUNNING).
         */
        if (preempt)
                return TASK_REPORT_MAX;

        /*
         * task_state_index() uses fls() and returns a value from 0-8 range.
         * Decrement it by 1 (except TASK_RUNNING state i.e 0) before using
         * it for left shift operation to get the correct task->state
         * mapping.
         */
        state = __task_state_index(prev_state, p->exit_state);

        return state ? (1 << (state - 1)) : state;
}
#endif /* CREATE_TRACE_POINTS */

/*
 * Tracepoint for task switches, performed by the scheduler:
 */
TRACE_EVENT(sched_switch,

        TP_PROTO(bool preempt,
                 struct task_struct *prev,
                 struct task_struct *next,
                 unsigned int prev_state),

        TP_ARGS(preempt, prev, next, prev_state),

        TP_STRUCT__entry(
                __array(        char,        prev_comm,        TASK_COMM_LEN        )
                __field(        pid_t,        prev_pid                        )
                __field(        int,        prev_prio                        )
                __field(        long,        prev_state                        )
                __array(        char,        next_comm,        TASK_COMM_LEN        )
                __field(        pid_t,        next_pid                        )
                __field(        int,        next_prio                        )
        ),

        TP_fast_assign(
                memcpy(__entry->prev_comm, prev->comm, TASK_COMM_LEN);
                __entry->prev_pid        = prev->pid;
                __entry->prev_prio        = prev->prio;
                __entry->prev_state        = __trace_sched_switch_state(preempt, prev_state, prev);
                memcpy(__entry->next_comm, next->comm, TASK_COMM_LEN);
                __entry->next_pid        = next->pid;
                __entry->next_prio        = next->prio;
                /* XXX SCHED_DEADLINE */
        ),

        TP_printk("prev_comm=%s prev_pid=%d prev_prio=%d prev_state=%s%s ==> next_comm=%s next_pid=%d next_prio=%d",
                __entry->prev_comm, __entry->prev_pid, __entry->prev_prio,

                (__entry->prev_state & (TASK_REPORT_MAX - 1)) ?
                  __print_flags(__entry->prev_state & (TASK_REPORT_MAX - 1), "|",
                                { TASK_INTERRUPTIBLE, "S" },
                                { TASK_UNINTERRUPTIBLE, "D" },
                                { __TASK_STOPPED, "T" },
                                { __TASK_TRACED, "t" },
                                { EXIT_DEAD, "X" },
                                { EXIT_ZOMBIE, "Z" },
                                { TASK_PARKED, "P" },
                                { TASK_DEAD, "I" }) :
                  "R",

                __entry->prev_state & TASK_REPORT_MAX ? "+" : "",
                __entry->next_comm, __entry->next_pid, __entry->next_prio)
);

/*
 * Tracepoint for a task being migrated:
 */
TRACE_EVENT(sched_migrate_task,

        TP_PROTO(struct task_struct *p, int dest_cpu),

        TP_ARGS(p, dest_cpu),

        TP_STRUCT__entry(
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
                __field(        int,        prio                        )
                __field(        int,        orig_cpu                )
                __field(        int,        dest_cpu                )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
                __entry->pid                = p->pid;
                __entry->prio                = p->prio; /* XXX SCHED_DEADLINE */
                __entry->orig_cpu        = task_cpu(p);
                __entry->dest_cpu        = dest_cpu;
        ),

        TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d",
                  __entry->comm, __entry->pid, __entry->prio,
                  __entry->orig_cpu, __entry->dest_cpu)
);

DECLARE_EVENT_CLASS(sched_process_template,

        TP_PROTO(struct task_struct *p),

        TP_ARGS(p),

        TP_STRUCT__entry(
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
                __field(        int,        prio                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
                __entry->pid                = p->pid;
                __entry->prio                = p->prio; /* XXX SCHED_DEADLINE */
        ),

        TP_printk("comm=%s pid=%d prio=%d",
                  __entry->comm, __entry->pid, __entry->prio)
);

/*
 * Tracepoint for freeing a task:
 */
DEFINE_EVENT(sched_process_template, sched_process_free,
             TP_PROTO(struct task_struct *p),
             TP_ARGS(p));

/*
 * Tracepoint for a task exiting:
 */
DEFINE_EVENT(sched_process_template, sched_process_exit,
             TP_PROTO(struct task_struct *p),
             TP_ARGS(p));

/*
 * Tracepoint for waiting on task to unschedule:
 */
DEFINE_EVENT(sched_process_template, sched_wait_task,
        TP_PROTO(struct task_struct *p),
        TP_ARGS(p));

/*
 * Tracepoint for a waiting task:
 */
TRACE_EVENT(sched_process_wait,

        TP_PROTO(struct pid *pid),

        TP_ARGS(pid),

        TP_STRUCT__entry(
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
                __field(        int,        prio                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
                __entry->pid                = pid_nr(pid);
                __entry->prio                = current->prio; /* XXX SCHED_DEADLINE */
        ),

        TP_printk("comm=%s pid=%d prio=%d",
                  __entry->comm, __entry->pid, __entry->prio)
);

/*
 * Tracepoint for kernel_clone:
 */
TRACE_EVENT(sched_process_fork,

        TP_PROTO(struct task_struct *parent, struct task_struct *child),

        TP_ARGS(parent, child),

        TP_STRUCT__entry(
                __array(        char,        parent_comm,        TASK_COMM_LEN        )
                __field(        pid_t,        parent_pid                        )
                __array(        char,        child_comm,        TASK_COMM_LEN        )
                __field(        pid_t,        child_pid                        )
        ),

        TP_fast_assign(
                memcpy(__entry->parent_comm, parent->comm, TASK_COMM_LEN);
                __entry->parent_pid        = parent->pid;
                memcpy(__entry->child_comm, child->comm, TASK_COMM_LEN);
                __entry->child_pid        = child->pid;
        ),

        TP_printk("comm=%s pid=%d child_comm=%s child_pid=%d",
                __entry->parent_comm, __entry->parent_pid,
                __entry->child_comm, __entry->child_pid)
);

/*
 * Tracepoint for exec:
 */
TRACE_EVENT(sched_process_exec,

        TP_PROTO(struct task_struct *p, pid_t old_pid,
                 struct linux_binprm *bprm),

        TP_ARGS(p, old_pid, bprm),

        TP_STRUCT__entry(
                __string(        filename,        bprm->filename        )
                __field(        pid_t,                pid                )
                __field(        pid_t,                old_pid                )
        ),

        TP_fast_assign(
                __assign_str(filename);
                __entry->pid                = p->pid;
                __entry->old_pid        = old_pid;
        ),

        TP_printk("filename=%s pid=%d old_pid=%d", __get_str(filename),
                  __entry->pid, __entry->old_pid)
);

/**
 * sched_prepare_exec - called before setting up new exec
 * @task:        pointer to the current task
 * @bprm:        pointer to linux_binprm used for new exec
 *
 * Called before flushing the old exec, where @task is still unchanged, but at
 * the point of no return during switching to the new exec. At the point it is
 * called the exec will either succeed, or on failure terminate the task. Also
 * see the "sched_process_exec" tracepoint, which is called right after @task
 * has successfully switched to the new exec.
 */
TRACE_EVENT(sched_prepare_exec,

        TP_PROTO(struct task_struct *task, struct linux_binprm *bprm),

        TP_ARGS(task, bprm),

        TP_STRUCT__entry(
                __string(        interp,                bprm->interp        )
                __string(        filename,        bprm->filename        )
                __field(        pid_t,                pid                )
                __string(        comm,                task->comm        )
        ),

        TP_fast_assign(
                __assign_str(interp);
                __assign_str(filename);
                __entry->pid = task->pid;
                __assign_str(comm);
        ),

        TP_printk("interp=%s filename=%s pid=%d comm=%s",
                  __get_str(interp), __get_str(filename),
                  __entry->pid, __get_str(comm))
);

#ifdef CONFIG_SCHEDSTATS
#define DEFINE_EVENT_SCHEDSTAT DEFINE_EVENT
#define DECLARE_EVENT_CLASS_SCHEDSTAT DECLARE_EVENT_CLASS
#else
#define DEFINE_EVENT_SCHEDSTAT DEFINE_EVENT_NOP
#define DECLARE_EVENT_CLASS_SCHEDSTAT DECLARE_EVENT_CLASS_NOP
#endif

/*
 * XXX the below sched_stat tracepoints only apply to SCHED_OTHER/BATCH/IDLE
 *     adding sched_stat support to SCHED_FIFO/RR would be welcome.
 */
DECLARE_EVENT_CLASS_SCHEDSTAT(sched_stat_template,

        TP_PROTO(struct task_struct *tsk, u64 delay),

        TP_ARGS(__perf_task(tsk), __perf_count(delay)),

        TP_STRUCT__entry(
                __array( char,        comm,        TASK_COMM_LEN        )
                __field( pid_t,        pid                        )
                __field( u64,        delay                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
                __entry->pid        = tsk->pid;
                __entry->delay        = delay;
        ),

        TP_printk("comm=%s pid=%d delay=%Lu [ns]",
                        __entry->comm, __entry->pid,
                        (unsigned long long)__entry->delay)
);

/*
 * Tracepoint for accounting wait time (time the task is runnable
 * but not actually running due to scheduler contention).
 */
DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_wait,
             TP_PROTO(struct task_struct *tsk, u64 delay),
             TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting sleep time (time the task is not runnable,
 * including iowait, see below).
 */
DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_sleep,
             TP_PROTO(struct task_struct *tsk, u64 delay),
             TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting iowait time (time the task is not runnable
 * due to waiting on IO to complete).
 */
DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_iowait,
             TP_PROTO(struct task_struct *tsk, u64 delay),
             TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting blocked time (time the task is in uninterruptible).
 */
DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_blocked,
             TP_PROTO(struct task_struct *tsk, u64 delay),
             TP_ARGS(tsk, delay));

/*
 * Tracepoint for accounting runtime (time the task is executing
 * on a CPU).
 */
DECLARE_EVENT_CLASS(sched_stat_runtime,

        TP_PROTO(struct task_struct *tsk, u64 runtime),

        TP_ARGS(tsk, __perf_count(runtime)),

        TP_STRUCT__entry(
                __array( char,        comm,        TASK_COMM_LEN        )
                __field( pid_t,        pid                        )
                __field( u64,        runtime                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
                __entry->pid                = tsk->pid;
                __entry->runtime        = runtime;
        ),

        TP_printk("comm=%s pid=%d runtime=%Lu [ns]",
                        __entry->comm, __entry->pid,
                        (unsigned long long)__entry->runtime)
);

DEFINE_EVENT(sched_stat_runtime, sched_stat_runtime,
             TP_PROTO(struct task_struct *tsk, u64 runtime),
             TP_ARGS(tsk, runtime));

/*
 * Tracepoint for showing priority inheritance modifying a tasks
 * priority.
 */
TRACE_EVENT(sched_pi_setprio,

        TP_PROTO(struct task_struct *tsk, struct task_struct *pi_task),

        TP_ARGS(tsk, pi_task),

        TP_STRUCT__entry(
                __array( char,        comm,        TASK_COMM_LEN        )
                __field( pid_t,        pid                        )
                __field( int,        oldprio                        )
                __field( int,        newprio                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
                __entry->pid                = tsk->pid;
                __entry->oldprio        = tsk->prio;
                __entry->newprio        = pi_task ?
                                min(tsk->normal_prio, pi_task->prio) :
                                tsk->normal_prio;
                /* XXX SCHED_DEADLINE bits missing */
        ),

        TP_printk("comm=%s pid=%d oldprio=%d newprio=%d",
                        __entry->comm, __entry->pid,
                        __entry->oldprio, __entry->newprio)
);

#ifdef CONFIG_DETECT_HUNG_TASK
TRACE_EVENT(sched_process_hang,
        TP_PROTO(struct task_struct *tsk),
        TP_ARGS(tsk),

        TP_STRUCT__entry(
                __array( char,        comm,        TASK_COMM_LEN        )
                __field( pid_t,        pid                        )
        ),

        TP_fast_assign(
                memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
                __entry->pid = tsk->pid;
        ),

        TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
);
#endif /* CONFIG_DETECT_HUNG_TASK */

/*
 * Tracks migration of tasks from one runqueue to another. Can be used to
 * detect if automatic NUMA balancing is bouncing between nodes.
 */
TRACE_EVENT(sched_move_numa,

        TP_PROTO(struct task_struct *tsk, int src_cpu, int dst_cpu),

        TP_ARGS(tsk, src_cpu, dst_cpu),

        TP_STRUCT__entry(
                __field( pid_t,        pid                        )
                __field( pid_t,        tgid                        )
                __field( pid_t,        ngid                        )
                __field( int,        src_cpu                        )
                __field( int,        src_nid                        )
                __field( int,        dst_cpu                        )
                __field( int,        dst_nid                        )
        ),

        TP_fast_assign(
                __entry->pid                = task_pid_nr(tsk);
                __entry->tgid                = task_tgid_nr(tsk);
                __entry->ngid                = task_numa_group_id(tsk);
                __entry->src_cpu        = src_cpu;
                __entry->src_nid        = cpu_to_node(src_cpu);
                __entry->dst_cpu        = dst_cpu;
                __entry->dst_nid        = cpu_to_node(dst_cpu);
        ),

        TP_printk("pid=%d tgid=%d ngid=%d src_cpu=%d src_nid=%d dst_cpu=%d dst_nid=%d",
                        __entry->pid, __entry->tgid, __entry->ngid,
                        __entry->src_cpu, __entry->src_nid,
                        __entry->dst_cpu, __entry->dst_nid)
);

DECLARE_EVENT_CLASS(sched_numa_pair_template,

        TP_PROTO(struct task_struct *src_tsk, int src_cpu,
                 struct task_struct *dst_tsk, int dst_cpu),

        TP_ARGS(src_tsk, src_cpu, dst_tsk, dst_cpu),

        TP_STRUCT__entry(
                __field( pid_t,        src_pid                        )
                __field( pid_t,        src_tgid                )
                __field( pid_t,        src_ngid                )
                __field( int,        src_cpu                        )
                __field( int,        src_nid                        )
                __field( pid_t,        dst_pid                        )
                __field( pid_t,        dst_tgid                )
                __field( pid_t,        dst_ngid                )
                __field( int,        dst_cpu                        )
                __field( int,        dst_nid                        )
        ),

        TP_fast_assign(
                __entry->src_pid        = task_pid_nr(src_tsk);
                __entry->src_tgid        = task_tgid_nr(src_tsk);
                __entry->src_ngid        = task_numa_group_id(src_tsk);
                __entry->src_cpu        = src_cpu;
                __entry->src_nid        = cpu_to_node(src_cpu);
                __entry->dst_pid        = dst_tsk ? task_pid_nr(dst_tsk) : 0;
                __entry->dst_tgid        = dst_tsk ? task_tgid_nr(dst_tsk) : 0;
                __entry->dst_ngid        = dst_tsk ? task_numa_group_id(dst_tsk) : 0;
                __entry->dst_cpu        = dst_cpu;
                __entry->dst_nid        = dst_cpu >= 0 ? cpu_to_node(dst_cpu) : -1;
        ),

        TP_printk("src_pid=%d src_tgid=%d src_ngid=%d src_cpu=%d src_nid=%d dst_pid=%d dst_tgid=%d dst_ngid=%d dst_cpu=%d dst_nid=%d",
                        __entry->src_pid, __entry->src_tgid, __entry->src_ngid,
                        __entry->src_cpu, __entry->src_nid,
                        __entry->dst_pid, __entry->dst_tgid, __entry->dst_ngid,
                        __entry->dst_cpu, __entry->dst_nid)
);

DEFINE_EVENT(sched_numa_pair_template, sched_stick_numa,

        TP_PROTO(struct task_struct *src_tsk, int src_cpu,
                 struct task_struct *dst_tsk, int dst_cpu),

        TP_ARGS(src_tsk, src_cpu, dst_tsk, dst_cpu)
);

DEFINE_EVENT(sched_numa_pair_template, sched_swap_numa,

        TP_PROTO(struct task_struct *src_tsk, int src_cpu,
                 struct task_struct *dst_tsk, int dst_cpu),

        TP_ARGS(src_tsk, src_cpu, dst_tsk, dst_cpu)
);

#ifdef CONFIG_NUMA_BALANCING
#define NUMAB_SKIP_REASON                                        \
        EM( NUMAB_SKIP_UNSUITABLE,                "unsuitable" )        \
        EM( NUMAB_SKIP_SHARED_RO,                "shared_ro" )        \
        EM( NUMAB_SKIP_INACCESSIBLE,                "inaccessible" )        \
        EM( NUMAB_SKIP_SCAN_DELAY,                "scan_delay" )        \
        EM( NUMAB_SKIP_PID_INACTIVE,                "pid_inactive" )        \
        EM( NUMAB_SKIP_IGNORE_PID,                "ignore_pid_inactive" )                \
        EMe(NUMAB_SKIP_SEQ_COMPLETED,                "seq_completed" )

/* Redefine for export. */
#undef EM
#undef EMe
#define EM(a, b)        TRACE_DEFINE_ENUM(a);
#define EMe(a, b)        TRACE_DEFINE_ENUM(a);

NUMAB_SKIP_REASON

/* Redefine for symbolic printing. */
#undef EM
#undef EMe
#define EM(a, b)        { a, b },
#define EMe(a, b)        { a, b }

TRACE_EVENT(sched_skip_vma_numa,

        TP_PROTO(struct mm_struct *mm, struct vm_area_struct *vma,
                 enum numa_vmaskip_reason reason),

        TP_ARGS(mm, vma, reason),

        TP_STRUCT__entry(
                __field(unsigned long, numa_scan_offset)
                __field(unsigned long, vm_start)
                __field(unsigned long, vm_end)
                __field(enum numa_vmaskip_reason, reason)
        ),

        TP_fast_assign(
                __entry->numa_scan_offset        = mm->numa_scan_offset;
                __entry->vm_start                = vma->vm_start;
                __entry->vm_end                        = vma->vm_end;
                __entry->reason                        = reason;
        ),

        TP_printk("numa_scan_offset=%lX vm_start=%lX vm_end=%lX reason=%s",
                  __entry->numa_scan_offset,
                  __entry->vm_start,
                  __entry->vm_end,
                  __print_symbolic(__entry->reason, NUMAB_SKIP_REASON))
);
#endif /* CONFIG_NUMA_BALANCING */

/*
 * Tracepoint for waking a polling cpu without an IPI.
 */
TRACE_EVENT(sched_wake_idle_without_ipi,

        TP_PROTO(int cpu),

        TP_ARGS(cpu),

        TP_STRUCT__entry(
                __field(        int,        cpu        )
        ),

        TP_fast_assign(
                __entry->cpu        = cpu;
        ),

        TP_printk("cpu=%d", __entry->cpu)
);

/*
 * Following tracepoints are not exported in tracefs and provide hooking
 * mechanisms only for testing and debugging purposes.
 *
 * Postfixed with _tp to make them easily identifiable in the code.
 */
DECLARE_TRACE(pelt_cfs_tp,
        TP_PROTO(struct cfs_rq *cfs_rq),
        TP_ARGS(cfs_rq));

DECLARE_TRACE(pelt_rt_tp,
        TP_PROTO(struct rq *rq),
        TP_ARGS(rq));

DECLARE_TRACE(pelt_dl_tp,
        TP_PROTO(struct rq *rq),
        TP_ARGS(rq));

DECLARE_TRACE(pelt_hw_tp,
        TP_PROTO(struct rq *rq),
        TP_ARGS(rq));

DECLARE_TRACE(pelt_irq_tp,
        TP_PROTO(struct rq *rq),
        TP_ARGS(rq));

DECLARE_TRACE(pelt_se_tp,
        TP_PROTO(struct sched_entity *se),
        TP_ARGS(se));

DECLARE_TRACE(sched_cpu_capacity_tp,
        TP_PROTO(struct rq *rq),
        TP_ARGS(rq));

DECLARE_TRACE(sched_overutilized_tp,
        TP_PROTO(struct root_domain *rd, bool overutilized),
        TP_ARGS(rd, overutilized));

DECLARE_TRACE(sched_util_est_cfs_tp,
        TP_PROTO(struct cfs_rq *cfs_rq),
        TP_ARGS(cfs_rq));

DECLARE_TRACE(sched_util_est_se_tp,
        TP_PROTO(struct sched_entity *se),
        TP_ARGS(se));

DECLARE_TRACE(sched_update_nr_running_tp,
        TP_PROTO(struct rq *rq, int change),
        TP_ARGS(rq, change));

DECLARE_TRACE(sched_compute_energy_tp,
        TP_PROTO(struct task_struct *p, int dst_cpu, unsigned long energy,
                 unsigned long max_util, unsigned long busy_time),
        TP_ARGS(p, dst_cpu, energy, max_util, busy_time));

#endif /* _TRACE_SCHED_H */

/* This part must be outside protection */
#include <trace/define_trace.h>


















































































































    7 





    7 






























































































    7 
    7 

























































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
// SPDX-License-Identifier: GPL-2.0
//
// Register cache access API
//
// Copyright 2011 Wolfson Microelectronics plc
//
// Author: Dimitris Papastamos <dp@opensource.wolfsonmicro.com>

#include <linux/bsearch.h>
#include <linux/device.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/sort.h>

#include "trace.h"
#include "internal.h"

static const struct regcache_ops *cache_types[] = {
        &regcache_rbtree_ops,
        &regcache_maple_ops,
        &regcache_flat_ops,
};

static int regcache_hw_init(struct regmap *map)
{
        int i, j;
        int ret;
        int count;
        unsigned int reg, val;
        void *tmp_buf;

        if (!map->num_reg_defaults_raw)
                return -EINVAL;

        /* calculate the size of reg_defaults */
        for (count = 0, i = 0; i < map->num_reg_defaults_raw; i++)
                if (regmap_readable(map, i * map->reg_stride) &&
                    !regmap_volatile(map, i * map->reg_stride))
                        count++;

        /* all registers are unreadable or volatile, so just bypass */
        if (!count) {
                map->cache_bypass = true;
                return 0;
        }

        map->num_reg_defaults = count;
        map->reg_defaults = kmalloc_array(count, sizeof(struct reg_default),
                                          GFP_KERNEL);
        if (!map->reg_defaults)
                return -ENOMEM;

        if (!map->reg_defaults_raw) {
                bool cache_bypass = map->cache_bypass;
                dev_warn(map->dev, "No cache defaults, reading back from HW\n");

                /* Bypass the cache access till data read from HW */
                map->cache_bypass = true;
                tmp_buf = kmalloc(map->cache_size_raw, GFP_KERNEL);
                if (!tmp_buf) {
                        ret = -ENOMEM;
                        goto err_free;
                }
                ret = regmap_raw_read(map, 0, tmp_buf,
                                      map->cache_size_raw);
                map->cache_bypass = cache_bypass;
                if (ret == 0) {
                        map->reg_defaults_raw = tmp_buf;
                        map->cache_free = true;
                } else {
                        kfree(tmp_buf);
                }
        }

        /* fill the reg_defaults */
        for (i = 0, j = 0; i < map->num_reg_defaults_raw; i++) {
                reg = i * map->reg_stride;

                if (!regmap_readable(map, reg))
                        continue;

                if (regmap_volatile(map, reg))
                        continue;

                if (map->reg_defaults_raw) {
                        val = regcache_get_val(map, map->reg_defaults_raw, i);
                } else {
                        bool cache_bypass = map->cache_bypass;

                        map->cache_bypass = true;
                        ret = regmap_read(map, reg, &val);
                        map->cache_bypass = cache_bypass;
                        if (ret != 0) {
                                dev_err(map->dev, "Failed to read %d: %d\n",
                                        reg, ret);
                                goto err_free;
                        }
                }

                map->reg_defaults[j].reg = reg;
                map->reg_defaults[j].def = val;
                j++;
        }

        return 0;

err_free:
        kfree(map->reg_defaults);

        return ret;
}

int regcache_init(struct regmap *map, const struct regmap_config *config)
{
        int ret;
        int i;
        void *tmp_buf;

        if (map->cache_type == REGCACHE_NONE) {
                if (config->reg_defaults || config->num_reg_defaults_raw)
                        dev_warn(map->dev,
                                 "No cache used with register defaults set!\n");

                map->cache_bypass = true;
                return 0;
        }

        if (config->reg_defaults && !config->num_reg_defaults) {
                dev_err(map->dev,
                         "Register defaults are set without the number!\n");
                return -EINVAL;
        }

        if (config->num_reg_defaults && !config->reg_defaults) {
                dev_err(map->dev,
                        "Register defaults number are set without the reg!\n");
                return -EINVAL;
        }

        for (i = 0; i < config->num_reg_defaults; i++)
                if (config->reg_defaults[i].reg % map->reg_stride)
                        return -EINVAL;

        for (i = 0; i < ARRAY_SIZE(cache_types); i++)
                if (cache_types[i]->type == map->cache_type)
                        break;

        if (i == ARRAY_SIZE(cache_types)) {
                dev_err(map->dev, "Could not match cache type: %d\n",
                        map->cache_type);
                return -EINVAL;
        }

        map->num_reg_defaults = config->num_reg_defaults;
        map->num_reg_defaults_raw = config->num_reg_defaults_raw;
        map->reg_defaults_raw = config->reg_defaults_raw;
        map->cache_word_size = BITS_TO_BYTES(config->val_bits);
        map->cache_size_raw = map->cache_word_size * config->num_reg_defaults_raw;

        map->cache = NULL;
        map->cache_ops = cache_types[i];

        if (!map->cache_ops->read ||
            !map->cache_ops->write ||
            !map->cache_ops->name)
                return -EINVAL;

        /* We still need to ensure that the reg_defaults
         * won't vanish from under us.  We'll need to make
         * a copy of it.
         */
        if (config->reg_defaults) {
                tmp_buf = kmemdup_array(config->reg_defaults, map->num_reg_defaults,
                                        sizeof(*map->reg_defaults), GFP_KERNEL);
                if (!tmp_buf)
                        return -ENOMEM;
                map->reg_defaults = tmp_buf;
        } else if (map->num_reg_defaults_raw) {
                /* Some devices such as PMICs don't have cache defaults,
                 * we cope with this by reading back the HW registers and
                 * crafting the cache defaults by hand.
                 */
                ret = regcache_hw_init(map);
                if (ret < 0)
                        return ret;
                if (map->cache_bypass)
                        return 0;
        }

        if (!map->max_register_is_set && map->num_reg_defaults_raw) {
                map->max_register = (map->num_reg_defaults_raw  - 1) * map->reg_stride;
                map->max_register_is_set = true;
        }

        if (map->cache_ops->init) {
                dev_dbg(map->dev, "Initializing %s cache\n",
                        map->cache_ops->name);
                map->lock(map->lock_arg);
                ret = map->cache_ops->init(map);
                map->unlock(map->lock_arg);
                if (ret)
                        goto err_free;
        }
        return 0;

err_free:
        kfree(map->reg_defaults);
        if (map->cache_free)
                kfree(map->reg_defaults_raw);

        return ret;
}

void regcache_exit(struct regmap *map)
{
        if (map->cache_type == REGCACHE_NONE)
                return;

        BUG_ON(!map->cache_ops);

        kfree(map->reg_defaults);
        if (map->cache_free)
                kfree(map->reg_defaults_raw);

        if (map->cache_ops->exit) {
                dev_dbg(map->dev, "Destroying %s cache\n",
                        map->cache_ops->name);
                map->lock(map->lock_arg);
                map->cache_ops->exit(map);
                map->unlock(map->lock_arg);
        }
}

/**
 * regcache_read - Fetch the value of a given register from the cache.
 *
 * @map: map to configure.
 * @reg: The register index.
 * @value: The value to be returned.
 *
 * Return a negative value on failure, 0 on success.
 */
int regcache_read(struct regmap *map,
                  unsigned int reg, unsigned int *value)
{
        int ret;

        if (map->cache_type == REGCACHE_NONE)
                return -EINVAL;

        BUG_ON(!map->cache_ops);

        if (!regmap_volatile(map, reg)) {
                ret = map->cache_ops->read(map, reg, value);

                if (ret == 0)
                        trace_regmap_reg_read_cache(map, reg, *value);

                return ret;
        }

        return -EINVAL;
}

/**
 * regcache_write - Set the value of a given register in the cache.
 *
 * @map: map to configure.
 * @reg: The register index.
 * @value: The new register value.
 *
 * Return a negative value on failure, 0 on success.
 */
int regcache_write(struct regmap *map,
                   unsigned int reg, unsigned int value)
{
        if (map->cache_type == REGCACHE_NONE)
                return 0;

        BUG_ON(!map->cache_ops);

        if (!regmap_volatile(map, reg))
                return map->cache_ops->write(map, reg, value);

        return 0;
}

bool regcache_reg_needs_sync(struct regmap *map, unsigned int reg,
                             unsigned int val)
{
        int ret;

        if (!regmap_writeable(map, reg))
                return false;

        /* If we don't know the chip just got reset, then sync everything. */
        if (!map->no_sync_defaults)
                return true;

        /* Is this the hardware default?  If so skip. */
        ret = regcache_lookup_reg(map, reg);
        if (ret >= 0 && val == map->reg_defaults[ret].def)
                return false;
        return true;
}

static int regcache_default_sync(struct regmap *map, unsigned int min,
                                 unsigned int max)
{
        unsigned int reg;

        for (reg = min; reg <= max; reg += map->reg_stride) {
                unsigned int val;
                int ret;

                if (regmap_volatile(map, reg) ||
                    !regmap_writeable(map, reg))
                        continue;

                ret = regcache_read(map, reg, &val);
                if (ret == -ENOENT)
                        continue;
                if (ret)
                        return ret;

                if (!regcache_reg_needs_sync(map, reg, val))
                        continue;

                map->cache_bypass = true;
                ret = _regmap_write(map, reg, val);
                map->cache_bypass = false;
                if (ret) {
                        dev_err(map->dev, "Unable to sync register %#x. %d\n",
                                reg, ret);
                        return ret;
                }
                dev_dbg(map->dev, "Synced register %#x, value %#x\n", reg, val);
        }

        return 0;
}

static int rbtree_all(const void *key, const struct rb_node *node)
{
        return 0;
}

/**
 * regcache_sync - Sync the register cache with the hardware.
 *
 * @map: map to configure.
 *
 * Any registers that should not be synced should be marked as
 * volatile.  In general drivers can choose not to use the provided
 * syncing functionality if they so require.
 *
 * Return a negative value on failure, 0 on success.
 */
int regcache_sync(struct regmap *map)
{
        int ret = 0;
        unsigned int i;
        const char *name;
        bool bypass;
        struct rb_node *node;

        if (WARN_ON(map->cache_type == REGCACHE_NONE))
                return -EINVAL;

        BUG_ON(!map->cache_ops);

        map->lock(map->lock_arg);
        /* Remember the initial bypass state */
        bypass = map->cache_bypass;
        dev_dbg(map->dev, "Syncing %s cache\n",
                map->cache_ops->name);
        name = map->cache_ops->name;
        trace_regcache_sync(map, name, "start");

        if (!map->cache_dirty)
                goto out;

        /* Apply any patch first */
        map->cache_bypass = true;
        for (i = 0; i < map->patch_regs; i++) {
                ret = _regmap_write(map, map->patch[i].reg, map->patch[i].def);
                if (ret != 0) {
                        dev_err(map->dev, "Failed to write %x = %x: %d\n",
                                map->patch[i].reg, map->patch[i].def, ret);
                        goto out;
                }
        }
        map->cache_bypass = false;

        if (map->cache_ops->sync)
                ret = map->cache_ops->sync(map, 0, map->max_register);
        else
                ret = regcache_default_sync(map, 0, map->max_register);

        if (ret == 0)
                map->cache_dirty = false;

out:
        /* Restore the bypass state */
        map->cache_bypass = bypass;
        map->no_sync_defaults = false;

        /*
         * If we did any paging with cache bypassed and a cached
         * paging register then the register and cache state might
         * have gone out of sync, force writes of all the paging
         * registers.
         */
        rb_for_each(node, NULL, &map->range_tree, rbtree_all) {
                struct regmap_range_node *this =
                        rb_entry(node, struct regmap_range_node, node);

                /* If there's nothing in the cache there's nothing to sync */
                if (regcache_read(map, this->selector_reg, &i) != 0)
                        continue;

                ret = _regmap_write(map, this->selector_reg, i);
                if (ret != 0) {
                        dev_err(map->dev, "Failed to write %x = %x: %d\n",
                                this->selector_reg, i, ret);
                        break;
                }
        }

        map->unlock(map->lock_arg);

        regmap_async_complete(map);

        trace_regcache_sync(map, name, "stop");

        return ret;
}
EXPORT_SYMBOL_GPL(regcache_sync);

/**
 * regcache_sync_region - Sync part  of the register cache with the hardware.
 *
 * @map: map to sync.
 * @min: first register to sync
 * @max: last register to sync
 *
 * Write all non-default register values in the specified region to
 * the hardware.
 *
 * Return a negative value on failure, 0 on success.
 */
int regcache_sync_region(struct regmap *map, unsigned int min,
                         unsigned int max)
{
        int ret = 0;
        const char *name;
        bool bypass;

        if (WARN_ON(map->cache_type == REGCACHE_NONE))
                return -EINVAL;

        BUG_ON(!map->cache_ops);

        map->lock(map->lock_arg);

        /* Remember the initial bypass state */
        bypass = map->cache_bypass;

        name = map->cache_ops->name;
        dev_dbg(map->dev, "Syncing %s cache from %d-%d\n", name, min, max);

        trace_regcache_sync(map, name, "start region");

        if (!map->cache_dirty)
                goto out;

        map->async = true;

        if (map->cache_ops->sync)
                ret = map->cache_ops->sync(map, min, max);
        else
                ret = regcache_default_sync(map, min, max);

out:
        /* Restore the bypass state */
        map->cache_bypass = bypass;
        map->async = false;
        map->no_sync_defaults = false;
        map->unlock(map->lock_arg);

        regmap_async_complete(map);

        trace_regcache_sync(map, name, "stop region");

        return ret;
}
EXPORT_SYMBOL_GPL(regcache_sync_region);

/**
 * regcache_drop_region - Discard part of the register cache
 *
 * @map: map to operate on
 * @min: first register to discard
 * @max: last register to discard
 *
 * Discard part of the register cache.
 *
 * Return a negative value on failure, 0 on success.
 */
int regcache_drop_region(struct regmap *map, unsigned int min,
                         unsigned int max)
{
        int ret = 0;

        if (!map->cache_ops || !map->cache_ops->drop)
                return -EINVAL;

        map->lock(map->lock_arg);

        trace_regcache_drop_region(map, min, max);

        ret = map->cache_ops->drop(map, min, max);

        map->unlock(map->lock_arg);

        return ret;
}
EXPORT_SYMBOL_GPL(regcache_drop_region);

/**
 * regcache_cache_only - Put a register map into cache only mode
 *
 * @map: map to configure
 * @enable: flag if changes should be written to the hardware
 *
 * When a register map is marked as cache only writes to the register
 * map API will only update the register cache, they will not cause
 * any hardware changes.  This is useful for allowing portions of
 * drivers to act as though the device were functioning as normal when
 * it is disabled for power saving reasons.
 */
void regcache_cache_only(struct regmap *map, bool enable)
{
        map->lock(map->lock_arg);
        WARN_ON(map->cache_type != REGCACHE_NONE &&
                map->cache_bypass && enable);
        map->cache_only = enable;
        trace_regmap_cache_only(map, enable);
        map->unlock(map->lock_arg);
}
EXPORT_SYMBOL_GPL(regcache_cache_only);

/**
 * regcache_mark_dirty - Indicate that HW registers were reset to default values
 *
 * @map: map to mark
 *
 * Inform regcache that the device has been powered down or reset, so that
 * on resume, regcache_sync() knows to write out all non-default values
 * stored in the cache.
 *
 * If this function is not called, regcache_sync() will assume that
 * the hardware state still matches the cache state, modulo any writes that
 * happened when cache_only was true.
 */
void regcache_mark_dirty(struct regmap *map)
{
        map->lock(map->lock_arg);
        map->cache_dirty = true;
        map->no_sync_defaults = true;
        map->unlock(map->lock_arg);
}
EXPORT_SYMBOL_GPL(regcache_mark_dirty);

/**
 * regcache_cache_bypass - Put a register map into cache bypass mode
 *
 * @map: map to configure
 * @enable: flag if changes should not be written to the cache
 *
 * When a register map is marked with the cache bypass option, writes
 * to the register map API will only update the hardware and not
 * the cache directly.  This is useful when syncing the cache back to
 * the hardware.
 */
void regcache_cache_bypass(struct regmap *map, bool enable)
{
        map->lock(map->lock_arg);
        WARN_ON(map->cache_only && enable);
        map->cache_bypass = enable;
        trace_regmap_cache_bypass(map, enable);
        map->unlock(map->lock_arg);
}
EXPORT_SYMBOL_GPL(regcache_cache_bypass);

/**
 * regcache_reg_cached - Check if a register is cached
 *
 * @map: map to check
 * @reg: register to check
 *
 * Reports if a register is cached.
 */
bool regcache_reg_cached(struct regmap *map, unsigned int reg)
{
        unsigned int val;
        int ret;

        map->lock(map->lock_arg);

        ret = regcache_read(map, reg, &val);

        map->unlock(map->lock_arg);

        return ret == 0;
}
EXPORT_SYMBOL_GPL(regcache_reg_cached);

void regcache_set_val(struct regmap *map, void *base, unsigned int idx,
                      unsigned int val)
{
        /* Use device native format if possible */
        if (map->format.format_val) {
                map->format.format_val(base + (map->cache_word_size * idx),
                                       val, 0);
                return;
        }

        switch (map->cache_word_size) {
        case 1: {
                u8 *cache = base;

                cache[idx] = val;
                break;
        }
        case 2: {
                u16 *cache = base;

                cache[idx] = val;
                break;
        }
        case 4: {
                u32 *cache = base;

                cache[idx] = val;
                break;
        }
        default:
                BUG();
        }
}

unsigned int regcache_get_val(struct regmap *map, const void *base,
                              unsigned int idx)
{
        if (!base)
                return -EINVAL;

        /* Use device native format if possible */
        if (map->format.parse_val)
                return map->format.parse_val(regcache_get_val_addr(map, base,
                                                                   idx));

        switch (map->cache_word_size) {
        case 1: {
                const u8 *cache = base;

                return cache[idx];
        }
        case 2: {
                const u16 *cache = base;

                return cache[idx];
        }
        case 4: {
                const u32 *cache = base;

                return cache[idx];
        }
        default:
                BUG();
        }
        /* unreachable */
        return -1;
}

static int regcache_default_cmp(const void *a, const void *b)
{
        const struct reg_default *_a = a;
        const struct reg_default *_b = b;

        return _a->reg - _b->reg;
}

int regcache_lookup_reg(struct regmap *map, unsigned int reg)
{
        struct reg_default key;
        struct reg_default *r;

        key.reg = reg;
        key.def = 0;

        r = bsearch(&key, map->reg_defaults, map->num_reg_defaults,
                    sizeof(struct reg_default), regcache_default_cmp);

        if (r)
                return r - map->reg_defaults;
        else
                return -ENOENT;
}

static bool regcache_reg_present(unsigned long *cache_present, unsigned int idx)
{
        if (!cache_present)
                return true;

        return test_bit(idx, cache_present);
}

int regcache_sync_val(struct regmap *map, unsigned int reg, unsigned int val)
{
        int ret;

        if (!regcache_reg_needs_sync(map, reg, val))
                return 0;

        map->cache_bypass = true;

        ret = _regmap_write(map, reg, val);

        map->cache_bypass = false;

        if (ret != 0) {
                dev_err(map->dev, "Unable to sync register %#x. %d\n",
                        reg, ret);
                return ret;
        }
        dev_dbg(map->dev, "Synced register %#x, value %#x\n",
                reg, val);

        return 0;
}

static int regcache_sync_block_single(struct regmap *map, void *block,
                                      unsigned long *cache_present,
                                      unsigned int block_base,
                                      unsigned int start, unsigned int end)
{
        unsigned int i, regtmp, val;
        int ret;

        for (i = start; i < end; i++) {
                regtmp = block_base + (i * map->reg_stride);

                if (!regcache_reg_present(cache_present, i) ||
                    !regmap_writeable(map, regtmp))
                        continue;

                val = regcache_get_val(map, block, i);
                ret = regcache_sync_val(map, regtmp, val);
                if (ret != 0)
                        return ret;
        }

        return 0;
}

static int regcache_sync_block_raw_flush(struct regmap *map, const void **data,
                                         unsigned int base, unsigned int cur)
{
        size_t val_bytes = map->format.val_bytes;
        int ret, count;

        if (*data == NULL)
                return 0;

        count = (cur - base) / map->reg_stride;

        dev_dbg(map->dev, "Writing %zu bytes for %d registers from 0x%x-0x%x\n",
                count * val_bytes, count, base, cur - map->reg_stride);

        map->cache_bypass = true;

        ret = _regmap_raw_write(map, base, *data, count * val_bytes, false);
        if (ret)
                dev_err(map->dev, "Unable to sync registers %#x-%#x. %d\n",
                        base, cur - map->reg_stride, ret);

        map->cache_bypass = false;

        *data = NULL;

        return ret;
}

static int regcache_sync_block_raw(struct regmap *map, void *block,
                            unsigned long *cache_present,
                            unsigned int block_base, unsigned int start,
                            unsigned int end)
{
        unsigned int i, val;
        unsigned int regtmp = 0;
        unsigned int base = 0;
        const void *data = NULL;
        int ret;

        for (i = start; i < end; i++) {
                regtmp = block_base + (i * map->reg_stride);

                if (!regcache_reg_present(cache_present, i) ||
                    !regmap_writeable(map, regtmp)) {
                        ret = regcache_sync_block_raw_flush(map, &data,
                                                            base, regtmp);
                        if (ret != 0)
                                return ret;
                        continue;
                }

                val = regcache_get_val(map, block, i);
                if (!regcache_reg_needs_sync(map, regtmp, val)) {
                        ret = regcache_sync_block_raw_flush(map, &data,
                                                            base, regtmp);
                        if (ret != 0)
                                return ret;
                        continue;
                }

                if (!data) {
                        data = regcache_get_val_addr(map, block, i);
                        base = regtmp;
                }
        }

        return regcache_sync_block_raw_flush(map, &data, base, regtmp +
                        map->reg_stride);
}

int regcache_sync_block(struct regmap *map, void *block,
                        unsigned long *cache_present,
                        unsigned int block_base, unsigned int start,
                        unsigned int end)
{
        if (regmap_can_raw_write(map) && !map->use_single_write)
                return regcache_sync_block_raw(map, block, cache_present,
                                               block_base, start, end);
        else
                return regcache_sync_block_single(map, block, cache_present,
                                                  block_base, start, end);
}




























































    7 




























    4 











    7 






















 10024 





















   71 


   67 

    8 
























   13 









    5 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
// SPDX-License-Identifier: GPL-2.0
#include <linux/err.h>
#include <linux/bug.h>
#include <linux/atomic.h>
#include <linux/errseq.h>
#include <linux/log2.h>

/*
 * An errseq_t is a way of recording errors in one place, and allowing any
 * number of "subscribers" to tell whether it has changed since a previous
 * point where it was sampled.
 *
 * It's implemented as an unsigned 32-bit value. The low order bits are
 * designated to hold an error code (between 0 and -MAX_ERRNO). The upper bits
 * are used as a counter. This is done with atomics instead of locking so that
 * these functions can be called from any context.
 *
 * The general idea is for consumers to sample an errseq_t value. That value
 * can later be used to tell whether any new errors have occurred since that
 * sampling was done.
 *
 * Note that there is a risk of collisions if new errors are being recorded
 * frequently, since we have so few bits to use as a counter.
 *
 * To mitigate this, one bit is used as a flag to tell whether the value has
 * been sampled since a new value was recorded. That allows us to avoid bumping
 * the counter if no one has sampled it since the last time an error was
 * recorded.
 *
 * A new errseq_t should always be zeroed out.  A errseq_t value of all zeroes
 * is the special (but common) case where there has never been an error. An all
 * zero value thus serves as the "epoch" if one wishes to know whether there
 * has ever been an error set since it was first initialized.
 */

/* The low bits are designated for error code (max of MAX_ERRNO) */
#define ERRSEQ_SHIFT                ilog2(MAX_ERRNO + 1)

/* This bit is used as a flag to indicate whether the value has been seen */
#define ERRSEQ_SEEN                (1 << ERRSEQ_SHIFT)

/* The lowest bit of the counter */
#define ERRSEQ_CTR_INC                (1 << (ERRSEQ_SHIFT + 1))

/**
 * errseq_set - set a errseq_t for later reporting
 * @eseq: errseq_t field that should be set
 * @err: error to set (must be between -1 and -MAX_ERRNO)
 *
 * This function sets the error in @eseq, and increments the sequence counter
 * if the last sequence was sampled at some point in the past.
 *
 * Any error set will always overwrite an existing error.
 *
 * Return: The previous value, primarily for debugging purposes. The
 * return value should not be used as a previously sampled value in later
 * calls as it will not have the SEEN flag set.
 */
errseq_t errseq_set(errseq_t *eseq, int err)
{
        errseq_t cur, old;

        /* MAX_ERRNO must be able to serve as a mask */
        BUILD_BUG_ON_NOT_POWER_OF_2(MAX_ERRNO + 1);

        /*
         * Ensure the error code actually fits where we want it to go. If it
         * doesn't then just throw a warning and don't record anything. We
         * also don't accept zero here as that would effectively clear a
         * previous error.
         */
        old = READ_ONCE(*eseq);

        if (WARN(unlikely(err == 0 || (unsigned int)-err > MAX_ERRNO),
                                "err = %d\n", err))
                return old;

        for (;;) {
                errseq_t new;

                /* Clear out error bits and set new error */
                new = (old & ~(MAX_ERRNO|ERRSEQ_SEEN)) | -err;

                /* Only increment if someone has looked at it */
                if (old & ERRSEQ_SEEN)
                        new += ERRSEQ_CTR_INC;

                /* If there would be no change, then call it done */
                if (new == old) {
                        cur = new;
                        break;
                }

                /* Try to swap the new value into place */
                cur = cmpxchg(eseq, old, new);

                /*
                 * Call it success if we did the swap or someone else beat us
                 * to it for the same value.
                 */
                if (likely(cur == old || cur == new))
                        break;

                /* Raced with an update, try again */
                old = cur;
        }
        return cur;
}
EXPORT_SYMBOL(errseq_set);

/**
 * errseq_sample() - Grab current errseq_t value.
 * @eseq: Pointer to errseq_t to be sampled.
 *
 * This function allows callers to initialise their errseq_t variable.
 * If the error has been "seen", new callers will not see an old error.
 * If there is an unseen error in @eseq, the caller of this function will
 * see it the next time it checks for an error.
 *
 * Context: Any context.
 * Return: The current errseq value.
 */
errseq_t errseq_sample(errseq_t *eseq)
{
        errseq_t old = READ_ONCE(*eseq);

        /* If nobody has seen this error yet, then we can be the first. */
        if (!(old & ERRSEQ_SEEN))
                old = 0;
        return old;
}
EXPORT_SYMBOL(errseq_sample);

/**
 * errseq_check() - Has an error occurred since a particular sample point?
 * @eseq: Pointer to errseq_t value to be checked.
 * @since: Previously-sampled errseq_t from which to check.
 *
 * Grab the value that eseq points to, and see if it has changed @since
 * the given value was sampled. The @since value is not advanced, so there
 * is no need to mark the value as seen.
 *
 * Return: The latest error set in the errseq_t or 0 if it hasn't changed.
 */
int errseq_check(errseq_t *eseq, errseq_t since)
{
        errseq_t cur = READ_ONCE(*eseq);

        if (likely(cur == since))
                return 0;
        return -(cur & MAX_ERRNO);
}
EXPORT_SYMBOL(errseq_check);

/**
 * errseq_check_and_advance() - Check an errseq_t and advance to current value.
 * @eseq: Pointer to value being checked and reported.
 * @since: Pointer to previously-sampled errseq_t to check against and advance.
 *
 * Grab the eseq value, and see whether it matches the value that @since
 * points to. If it does, then just return 0.
 *
 * If it doesn't, then the value has changed. Set the "seen" flag, and try to
 * swap it into place as the new eseq value. Then, set that value as the new
 * "since" value, and return whatever the error portion is set to.
 *
 * Note that no locking is provided here for concurrent updates to the "since"
 * value. The caller must provide that if necessary. Because of this, callers
 * may want to do a lockless errseq_check before taking the lock and calling
 * this.
 *
 * Return: Negative errno if one has been stored, or 0 if no new error has
 * occurred.
 */
int errseq_check_and_advance(errseq_t *eseq, errseq_t *since)
{
        int err = 0;
        errseq_t old, new;

        /*
         * Most callers will want to use the inline wrapper to check this,
         * so that the common case of no error is handled without needing
         * to take the lock that protects the "since" value.
         */
        old = READ_ONCE(*eseq);
        if (old != *since) {
                /*
                 * Set the flag and try to swap it into place if it has
                 * changed.
                 *
                 * We don't care about the outcome of the swap here. If the
                 * swap doesn't occur, then it has either been updated by a
                 * writer who is altering the value in some way (updating
                 * counter or resetting the error), or another reader who is
                 * just setting the "seen" flag. Either outcome is OK, and we
                 * can advance "since" and return an error based on what we
                 * have.
                 */
                new = old | ERRSEQ_SEEN;
                if (new != old)
                        cmpxchg(eseq, old, new);
                *since = new;
                err = -(new & MAX_ERRNO);
        }
        return err;
}
EXPORT_SYMBOL(errseq_check_and_advance);
















































































   19 












   19 




















































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
/*
 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <rdma/rdma_user_ioctl.h>
#include <rdma/uverbs_ioctl.h>
#include "rdma_core.h"
#include "uverbs.h"

struct bundle_alloc_head {
        struct_group_tagged(bundle_alloc_head_hdr, hdr,
                struct bundle_alloc_head *next;
        );
        u8 data[];
};

struct bundle_priv {
        /* Must be first */
        struct bundle_alloc_head_hdr alloc_head;
        struct bundle_alloc_head *allocated_mem;
        size_t internal_avail;
        size_t internal_used;

        struct radix_tree_root *radix;
        const struct uverbs_api_ioctl_method *method_elm;
        void __rcu **radix_slots;
        unsigned long radix_slots_len;
        u32 method_key;

        struct ib_uverbs_attr __user *user_attrs;
        struct ib_uverbs_attr *uattrs;

        DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
        DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
        DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);

        /*
         * Must be last. bundle ends in a flex array which overlaps
         * internal_buffer.
         */
        struct uverbs_attr_bundle_hdr bundle;
        u64 internal_buffer[32];
};

/*
 * Each method has an absolute minimum amount of memory it needs to allocate,
 * precompute that amount and determine if the onstack memory can be used or
 * if allocation is need.
 */
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
                              unsigned int num_attrs)
{
        struct bundle_priv *pbundle;
        struct uverbs_attr_bundle *bundle;
        size_t bundle_size =
                offsetof(struct bundle_priv, internal_buffer) +
                sizeof(*bundle->attrs) * method_elm->key_bitmap_len +
                sizeof(*pbundle->uattrs) * num_attrs;

        method_elm->use_stack = bundle_size <= sizeof(*pbundle);
        method_elm->bundle_size =
                ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));

        /* Do not want order-2 allocations for this. */
        WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
}

/**
 * _uverbs_alloc() - Quickly allocate memory for use with a bundle
 * @bundle: The bundle
 * @size: Number of bytes to allocate
 * @flags: Allocator flags
 *
 * The bundle allocator is intended for allocations that are connected with
 * processing the system call related to the bundle. The allocated memory is
 * always freed once the system call completes, and cannot be freed any other
 * way.
 *
 * This tries to use a small pool of pre-allocated memory for performance.
 */
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
                             gfp_t flags)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        size_t new_used;
        void *res;

        if (check_add_overflow(size, pbundle->internal_used, &new_used))
                return ERR_PTR(-EOVERFLOW);

        if (new_used > pbundle->internal_avail) {
                struct bundle_alloc_head *buf;

                buf = kvmalloc(struct_size(buf, data, size), flags);
                if (!buf)
                        return ERR_PTR(-ENOMEM);
                buf->next = pbundle->allocated_mem;
                pbundle->allocated_mem = buf;
                return buf->data;
        }

        res = (void *)pbundle->internal_buffer + pbundle->internal_used;
        pbundle->internal_used =
                ALIGN(new_used, sizeof(*pbundle->internal_buffer));
        if (want_init_on_alloc(flags))
                memset(res, 0, size);
        return res;
}
EXPORT_SYMBOL(_uverbs_alloc);

static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
                                   u16 len)
{
        if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
                return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
                                            uattr->len - len);

        return !memchr_inv((const void *)&uattr->data + len,
                           0, uattr->len - len);
}

static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
                             const struct uverbs_attr *attr)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        u16 flags;

        flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
                UVERBS_ATTR_F_VALID_OUTPUT;
        if (put_user(flags,
                     &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
                return -EFAULT;
        return 0;
}

static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
                                     const struct uverbs_api_attr *attr_uapi,
                                     struct uverbs_objs_arr_attr *attr,
                                     struct ib_uverbs_attr *uattr,
                                     u32 attr_bkey)
{
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        size_t array_len;
        u32 *idr_vals;
        int ret = 0;
        size_t i;

        if (uattr->attr_data.reserved)
                return -EINVAL;

        if (uattr->len % sizeof(u32))
                return -EINVAL;

        array_len = uattr->len / sizeof(u32);
        if (array_len < spec->u2.objs_arr.min_len ||
            array_len > spec->u2.objs_arr.max_len)
                return -EINVAL;

        attr->uobjects =
                uverbs_alloc(bundle,
                             array_size(array_len, sizeof(*attr->uobjects)));
        if (IS_ERR(attr->uobjects))
                return PTR_ERR(attr->uobjects);

        /*
         * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
         * to store idrs array and avoid additional memory allocation. The
         * idrs array is offset to the end of the uobjects array so we will be
         * able to read idr and replace with a pointer.
         */
        idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;

        if (uattr->len > sizeof(uattr->data)) {
                ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
                                     uattr->len);
                if (ret)
                        return -EFAULT;
        } else {
                memcpy(idr_vals, &uattr->data, uattr->len);
        }

        for (i = 0; i != array_len; i++) {
                attr->uobjects[i] = uverbs_get_uobject_from_file(
                        spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
                        idr_vals[i], bundle);
                if (IS_ERR(attr->uobjects[i])) {
                        ret = PTR_ERR(attr->uobjects[i]);
                        break;
                }
        }

        attr->len = i;
        __set_bit(attr_bkey, pbundle->spec_finalize);
        return ret;
}

static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
                                   struct uverbs_objs_arr_attr *attr,
                                   bool commit,
                                   struct uverbs_attr_bundle *attrs)
{
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        size_t i;

        for (i = 0; i != attr->len; i++)
                uverbs_finalize_object(attr->uobjects[i],
                                       spec->u2.objs_arr.access, false, commit,
                                       attrs);
}

static int uverbs_process_attr(struct bundle_priv *pbundle,
                               const struct uverbs_api_attr *attr_uapi,
                               struct ib_uverbs_attr *uattr, u32 attr_bkey)
{
        const struct uverbs_attr_spec *spec = &attr_uapi->spec;
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        struct uverbs_attr *e = &bundle->attrs[attr_bkey];
        const struct uverbs_attr_spec *val_spec = spec;
        struct uverbs_obj_attr *o_attr;

        switch (spec->type) {
        case UVERBS_ATTR_TYPE_ENUM_IN:
                if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
                        return -EOPNOTSUPP;

                if (uattr->attr_data.enum_data.reserved)
                        return -EINVAL;

                val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];

                /* Currently we only support PTR_IN based enums */
                if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
                        return -EOPNOTSUPP;

                e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
                fallthrough;
        case UVERBS_ATTR_TYPE_PTR_IN:
                /* Ensure that any data provided by userspace beyond the known
                 * struct is zero. Userspace that knows how to use some future
                 * longer struct will fail here if used with an old kernel and
                 * non-zero content, making ABI compat/discovery simpler.
                 */
                if (uattr->len > val_spec->u.ptr.len &&
                    val_spec->zero_trailing &&
                    !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
                        return -EOPNOTSUPP;

                fallthrough;
        case UVERBS_ATTR_TYPE_PTR_OUT:
                if (uattr->len < val_spec->u.ptr.min_len ||
                    (!val_spec->zero_trailing &&
                     uattr->len > val_spec->u.ptr.len))
                        return -EINVAL;

                if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
                    uattr->attr_data.reserved)
                        return -EINVAL;

                e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
                e->ptr_attr.len = uattr->len;

                if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
                        void *p;

                        p = uverbs_alloc(bundle, uattr->len);
                        if (IS_ERR(p))
                                return PTR_ERR(p);

                        e->ptr_attr.ptr = p;

                        if (copy_from_user(p, u64_to_user_ptr(uattr->data),
                                           uattr->len))
                                return -EFAULT;
                } else {
                        e->ptr_attr.data = uattr->data;
                }
                break;

        case UVERBS_ATTR_TYPE_IDR:
        case UVERBS_ATTR_TYPE_FD:
                if (uattr->attr_data.reserved)
                        return -EINVAL;

                if (uattr->len != 0)
                        return -EINVAL;

                o_attr = &e->obj_attr;
                o_attr->attr_elm = attr_uapi;

                /*
                 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
                 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
                 * here without caring about truncation as we know that the
                 * IDR implementation today rejects negative IDs
                 */
                o_attr->uobject = uverbs_get_uobject_from_file(
                        spec->u.obj.obj_type, spec->u.obj.access,
                        uattr->data_s64, bundle);
                if (IS_ERR(o_attr->uobject))
                        return PTR_ERR(o_attr->uobject);
                __set_bit(attr_bkey, pbundle->uobj_finalize);

                if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
                        unsigned int uattr_idx = uattr - pbundle->uattrs;
                        s64 id = o_attr->uobject->id;

                        /* Copy the allocated id to the user-space */
                        if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
                                return -EFAULT;
                }

                break;

        case UVERBS_ATTR_TYPE_RAW_FD:
                if (uattr->attr_data.reserved || uattr->len != 0 ||
                    uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
                        return -EINVAL;
                /* _uverbs_get_const_signed() is the accessor */
                e->ptr_attr.data = uattr->data_s64;
                break;

        case UVERBS_ATTR_TYPE_IDRS_ARRAY:
                return uverbs_process_idrs_array(pbundle, attr_uapi,
                                                 &e->objs_arr_attr, uattr,
                                                 attr_bkey);
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

/*
 * We search the radix tree with the method prefix and now we want to fast
 * search the suffix bits to get a particular attribute pointer. It is not
 * totally clear to me if this breaks the radix tree encasulation or not, but
 * it uses the iter data to determine if the method iter points at the same
 * chunk that will store the attribute, if so it just derefs it directly. By
 * construction in most kernel configs the method and attrs will all fit in a
 * single radix chunk, so in most cases this will have no search. Other cases
 * this falls back to a full search.
 */
static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
                                             u32 attr_key)
{
        void __rcu **slot;

        if (likely(attr_key < pbundle->radix_slots_len)) {
                void *entry;

                slot = pbundle->radix_slots + attr_key;
                entry = rcu_dereference_raw(*slot);
                if (likely(!radix_tree_is_internal_node(entry) && entry))
                        return slot;
        }

        return radix_tree_lookup_slot(pbundle->radix,
                                      pbundle->method_key | attr_key);
}

static int uverbs_set_attr(struct bundle_priv *pbundle,
                           struct ib_uverbs_attr *uattr)
{
        u32 attr_key = uapi_key_attr(uattr->attr_id);
        u32 attr_bkey = uapi_bkey_attr(attr_key);
        const struct uverbs_api_attr *attr;
        void __rcu **slot;
        int ret;

        slot = uapi_get_attr_for_method(pbundle, attr_key);
        if (!slot) {
                /*
                 * Kernel does not support the attribute but user-space says it
                 * is mandatory
                 */
                if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
                        return -EPROTONOSUPPORT;
                return 0;
        }
        attr = rcu_dereference_protected(*slot, true);

        /* Reject duplicate attributes from user-space */
        if (test_bit(attr_bkey, pbundle->bundle.attr_present))
                return -EINVAL;

        ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
        if (ret)
                return ret;

        __set_bit(attr_bkey, pbundle->bundle.attr_present);

        return 0;
}

static int ib_uverbs_run_method(struct bundle_priv *pbundle,
                                unsigned int num_attrs)
{
        int (*handler)(struct uverbs_attr_bundle *attrs);
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
        unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
        unsigned int i;
        int ret;

        /* See uverbs_disassociate_api() */
        handler = srcu_dereference(
                pbundle->method_elm->handler,
                &pbundle->bundle.ufile->device->disassociate_srcu);
        if (!handler)
                return -EIO;

        pbundle->uattrs = uverbs_alloc(bundle, uattrs_size);
        if (IS_ERR(pbundle->uattrs))
                return PTR_ERR(pbundle->uattrs);
        if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
                return -EFAULT;

        for (i = 0; i != num_attrs; i++) {
                ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
                if (unlikely(ret))
                        return ret;
        }

        /* User space did not provide all the mandatory attributes */
        if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
                                    pbundle->bundle.attr_present,
                                    pbundle->method_elm->key_bitmap_len)))
                return -EINVAL;

        if (pbundle->method_elm->has_udata)
                uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,
                                  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
        else
                pbundle->bundle.driver_udata = (struct ib_udata){};

        if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
                struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr;

                ret = uobj_destroy(destroy_attr->uobject, bundle);
                if (ret)
                        return ret;
                __clear_bit(destroy_bkey, pbundle->uobj_finalize);

                ret = handler(bundle);
                uobj_put_destroy(destroy_attr->uobject);
        } else {
                ret = handler(bundle);
        }

        /*
         * Until the drivers are revised to use the bundle directly we have to
         * assume that the driver wrote to its UHW_OUT and flag userspace
         * appropriately.
         */
        if (!ret && pbundle->method_elm->has_udata) {
                const struct uverbs_attr *attr =
                        uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);

                if (!IS_ERR(attr))
                        ret = uverbs_set_output(bundle, attr);
        }

        /*
         * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
         * not invoke the method because the request is not supported.  No
         * other cases should return this code.
         */
        if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
                return -EINVAL;

        return ret;
}

static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
{
        unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
        struct uverbs_attr_bundle *bundle =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        struct bundle_alloc_head *memblock;
        unsigned int i;

        /* fast path for simple uobjects */
        i = -1;
        while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
                                  i + 1)) < key_bitmap_len) {
                struct uverbs_attr *attr = &bundle->attrs[i];

                uverbs_finalize_object(
                        attr->obj_attr.uobject,
                        attr->obj_attr.attr_elm->spec.u.obj.access,
                        test_bit(i, pbundle->uobj_hw_obj_valid),
                        commit, bundle);
        }

        i = -1;
        while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
                                  i + 1)) < key_bitmap_len) {
                struct uverbs_attr *attr = &bundle->attrs[i];
                const struct uverbs_api_attr *attr_uapi;
                void __rcu **slot;

                slot = uapi_get_attr_for_method(
                        pbundle,
                        pbundle->method_key | uapi_bkey_to_key_attr(i));
                if (WARN_ON(!slot))
                        continue;

                attr_uapi = rcu_dereference_protected(*slot, true);

                if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
                        uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
                                               commit, bundle);
                }
        }

        for (memblock = pbundle->allocated_mem; memblock;) {
                struct bundle_alloc_head *tmp = memblock;

                memblock = memblock->next;
                kvfree(tmp);
        }
}

static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
                               struct ib_uverbs_ioctl_hdr *hdr,
                               struct ib_uverbs_attr __user *user_attrs)
{
        const struct uverbs_api_ioctl_method *method_elm;
        struct uverbs_api *uapi = ufile->device->uapi;
        struct radix_tree_iter attrs_iter;
        struct bundle_priv *pbundle;
        struct bundle_priv onstack;
        void __rcu **slot;
        int ret;

        if (unlikely(hdr->driver_id != uapi->driver_id))
                return -EINVAL;

        slot = radix_tree_iter_lookup(
                &uapi->radix, &attrs_iter,
                uapi_key_obj(hdr->object_id) |
                        uapi_key_ioctl_method(hdr->method_id));
        if (unlikely(!slot))
                return -EPROTONOSUPPORT;
        method_elm = rcu_dereference_protected(*slot, true);

        if (!method_elm->use_stack) {
                pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
                if (!pbundle)
                        return -ENOMEM;
                pbundle->internal_avail =
                        method_elm->bundle_size -
                        offsetof(struct bundle_priv, internal_buffer);
                pbundle->alloc_head.next = NULL;
                pbundle->allocated_mem = container_of(&pbundle->alloc_head,
                                                struct bundle_alloc_head, hdr);
        } else {
                pbundle = &onstack;
                pbundle->internal_avail = sizeof(pbundle->internal_buffer);
                pbundle->allocated_mem = NULL;
        }

        /* Space for the pbundle->bundle.attrs flex array */
        pbundle->method_elm = method_elm;
        pbundle->method_key = attrs_iter.index;
        pbundle->bundle.ufile = ufile;
        pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
        pbundle->radix = &uapi->radix;
        pbundle->radix_slots = slot;
        pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
        pbundle->user_attrs = user_attrs;

        pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
                                               sizeof(*container_of(&pbundle->bundle,
                                                        struct uverbs_attr_bundle, hdr)->attrs),
                                               sizeof(*pbundle->internal_buffer));
        memset(pbundle->bundle.attr_present, 0,
               sizeof(pbundle->bundle.attr_present));
        memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
        memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
        memset(pbundle->uobj_hw_obj_valid, 0,
               sizeof(pbundle->uobj_hw_obj_valid));

        ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
        bundle_destroy(pbundle, ret == 0);
        return ret;
}

long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
        struct ib_uverbs_file *file = filp->private_data;
        struct ib_uverbs_ioctl_hdr __user *user_hdr =
                (struct ib_uverbs_ioctl_hdr __user *)arg;
        struct ib_uverbs_ioctl_hdr hdr;
        int srcu_key;
        int err;

        if (unlikely(cmd != RDMA_VERBS_IOCTL))
                return -ENOIOCTLCMD;

        err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
        if (err)
                return -EFAULT;

        if (hdr.length > PAGE_SIZE ||
            hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
                return -EINVAL;

        if (hdr.reserved1 || hdr.reserved2)
                return -EPROTONOSUPPORT;

        srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
        err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
        srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
        return err;
}

int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
                       size_t idx, u64 allowed_bits)
{
        const struct uverbs_attr *attr;
        u64 flags;

        attr = uverbs_attr_get(attrs_bundle, idx);
        /* Missing attribute means 0 flags */
        if (IS_ERR(attr)) {
                *to = 0;
                return 0;
        }

        /*
         * New userspace code should use 8 bytes to pass flags, but we
         * transparently support old userspaces that were using 4 bytes as
         * well.
         */
        if (attr->ptr_attr.len == 8)
                flags = attr->ptr_attr.data;
        else if (attr->ptr_attr.len == 4)
                flags = *(u32 *)&attr->ptr_attr.data;
        else
                return -EINVAL;

        if (flags & ~allowed_bits)
                return -EINVAL;

        *to = flags;
        return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);

int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
                       size_t idx, u64 allowed_bits)
{
        u64 flags;
        int ret;

        ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
        if (ret)
                return ret;

        if (flags > U32_MAX)
                return -EINVAL;
        *to = flags;

        return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);

/*
 * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
 * This is primarily used to convert the UVERBS_ATTR_UHW() into the
 * ib_udata format used by the drivers.
 */
void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
                       struct ib_udata *udata, unsigned int attr_in,
                       unsigned int attr_out)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);
        struct uverbs_attr_bundle *bundle_aux =
                container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
        const struct uverbs_attr *in =
                uverbs_attr_get(bundle_aux, attr_in);
        const struct uverbs_attr *out =
                uverbs_attr_get(bundle_aux, attr_out);

        if (!IS_ERR(in)) {
                udata->inlen = in->ptr_attr.len;
                if (uverbs_attr_ptr_is_inline(in))
                        udata->inbuf =
                                &pbundle->user_attrs[in->ptr_attr.uattr_idx]
                                         .data;
                else
                        udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
        } else {
                udata->inbuf = NULL;
                udata->inlen = 0;
        }

        if (!IS_ERR(out)) {
                udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
                udata->outlen = out->ptr_attr.len;
        } else {
                udata->outbuf = NULL;
                udata->outlen = 0;
        }
}

int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
                   const void *from, size_t size)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
        size_t min_size;

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        min_size = min_t(size_t, attr->ptr_attr.len, size);
        if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
                return -EFAULT;

        return uverbs_set_output(bundle, attr);
}
EXPORT_SYMBOL(uverbs_copy_to);


/*
 * This is only used if the caller has directly used copy_to_use to write the
 * data.  It signals to user space that the buffer is filled in.
 */
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        return uverbs_set_output(bundle, attr);
}

int _uverbs_get_const_signed(s64 *to,
                             const struct uverbs_attr_bundle *attrs_bundle,
                             size_t idx, s64 lower_bound, u64 upper_bound,
                             s64  *def_val)
{
        const struct uverbs_attr *attr;

        attr = uverbs_attr_get(attrs_bundle, idx);
        if (IS_ERR(attr)) {
                if ((PTR_ERR(attr) != -ENOENT) || !def_val)
                        return PTR_ERR(attr);

                *to = *def_val;
        } else {
                *to = attr->ptr_attr.data;
        }

        if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
                return -EINVAL;

        return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_signed);

int _uverbs_get_const_unsigned(u64 *to,
                               const struct uverbs_attr_bundle *attrs_bundle,
                               size_t idx, u64 upper_bound, u64 *def_val)
{
        const struct uverbs_attr *attr;

        attr = uverbs_attr_get(attrs_bundle, idx);
        if (IS_ERR(attr)) {
                if ((PTR_ERR(attr) != -ENOENT) || !def_val)
                        return PTR_ERR(attr);

                *to = *def_val;
        } else {
                *to = attr->ptr_attr.data;
        }

        if (*to > upper_bound)
                return -EINVAL;

        return 0;
}
EXPORT_SYMBOL(_uverbs_get_const_unsigned);

int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
                                  size_t idx, const void *from, size_t size)
{
        const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);

        if (IS_ERR(attr))
                return PTR_ERR(attr);

        if (size < attr->ptr_attr.len) {
                if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
                               attr->ptr_attr.len - size))
                        return -EFAULT;
        }
        return uverbs_copy_to(bundle, idx, from, size);
}
EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);

/* Once called an abort will call through to the type's destroy_hw() */
void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
                                 u16 idx)
{
        struct bundle_priv *pbundle =
                container_of(&bundle->hdr, struct bundle_priv, bundle);

        __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
                  pbundle->uobj_hw_obj_valid);
}
EXPORT_SYMBOL(uverbs_finalize_uobj_create);































































































   60 
  179 
  179 







  178 

    8 

   27 








  119 

    9 

   19 




   73 



   73 












   56 
   64 














   56 








  163 





   67 












  117 

  163 
    6 





  110 

    9 

   67 

















  109 





   82 











  145 








  145 


  145 
   27 
  119 






  145 

  144 





   26 


  119 








   52 









  143 






  144 








   93 
   52 
























   67 


































    9 






   16 




   56 



  179 








    9 







  179 


   67 

   49 


    5 



    5 






   52 

   52 













   68 





































































































   68 
   68 
   68 






   23 



   68 















   13 





   23 










   10 
   58 




   38 



   38 























  213 
   11 

  212 



    5 

    7 

    5 

















   87 

























  110 














 4823 




 4822 
 4780 


  154 





 4815 












  116 













   90 
   87 




  116 



  116 

  110 







  109 




    2 









    7 








    6 






















  116 



  116 










  116 
  116 


  116 



  116 
































  172 
  172 



   15 



  138 


   16 
   16 

   41 












   50 






   40 












    9 
    2 
    2 






   49 




   19 


   19 






 4689 






 4686 








   19 
   19 

   19 
    2 









   19 


   19 
   19 







  123 

































   40 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
 */

/*
 * fsnotify inode mark locking/lifetime/and refcnting
 *
 * REFCNT:
 * The group->recnt and mark->refcnt tell how many "things" in the kernel
 * currently are referencing the objects. Both kind of objects typically will
 * live inside the kernel with a refcnt of 2, one for its creation and one for
 * the reference a group and a mark hold to each other.
 * If you are holding the appropriate locks, you can take a reference and the
 * object itself is guaranteed to survive until the reference is dropped.
 *
 * LOCKING:
 * There are 3 locks involved with fsnotify inode marks and they MUST be taken
 * in order as follows:
 *
 * group->mark_mutex
 * mark->lock
 * mark->connector->lock
 *
 * group->mark_mutex protects the marks_list anchored inside a given group and
 * each mark is hooked via the g_list.  It also protects the groups private
 * data (i.e group limits).

 * mark->lock protects the marks attributes like its masks and flags.
 * Furthermore it protects the access to a reference of the group that the mark
 * is assigned to as well as the access to a reference of the inode/vfsmount
 * that is being watched by the mark.
 *
 * mark->connector->lock protects the list of marks anchored inside an
 * inode / vfsmount and each mark is hooked via the i_list.
 *
 * A list of notification marks relating to inode / mnt is contained in
 * fsnotify_mark_connector. That structure is alive as long as there are any
 * marks in the list and is also protected by fsnotify_mark_srcu. A mark gets
 * detached from fsnotify_mark_connector when last reference to the mark is
 * dropped.  Thus having mark reference is enough to protect mark->connector
 * pointer and to make sure fsnotify_mark_connector cannot disappear. Also
 * because we remove mark from g_list before dropping mark reference associated
 * with that, any mark found through g_list is guaranteed to have
 * mark->connector set until we drop group->mark_mutex.
 *
 * LIFETIME:
 * Inode marks survive between when they are added to an inode and when their
 * refcnt==0. Marks are also protected by fsnotify_mark_srcu.
 *
 * The inode mark can be cleared for a number of different reasons including:
 * - The inode is unlinked for the last time.  (fsnotify_inode_remove)
 * - The inode is being evicted from cache. (fsnotify_inode_delete)
 * - The fs the inode is on is unmounted.  (fsnotify_inode_delete/fsnotify_unmount_inodes)
 * - Something explicitly requests that it be removed.  (fsnotify_destroy_mark)
 * - The fsnotify_group associated with the mark is going away and all such marks
 *   need to be cleaned up. (fsnotify_clear_marks_by_group)
 *
 * This has the very interesting property of being able to run concurrently with
 * any (or all) other directions.
 */

#include <linux/fs.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/srcu.h>
#include <linux/ratelimit.h>

#include <linux/atomic.h>

#include <linux/fsnotify_backend.h>
#include "fsnotify.h"

#define FSNOTIFY_REAPER_DELAY        (1)        /* 1 jiffy */

struct srcu_struct fsnotify_mark_srcu;
struct kmem_cache *fsnotify_mark_connector_cachep;

static DEFINE_SPINLOCK(destroy_lock);
static LIST_HEAD(destroy_list);
static struct fsnotify_mark_connector *connector_destroy_list;

static void fsnotify_mark_destroy_workfn(struct work_struct *work);
static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn);

static void fsnotify_connector_destroy_workfn(struct work_struct *work);
static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn);

void fsnotify_get_mark(struct fsnotify_mark *mark)
{
        WARN_ON_ONCE(!refcount_read(&mark->refcnt));
        refcount_inc(&mark->refcnt);
}

static fsnotify_connp_t *fsnotify_object_connp(void *obj,
                                enum fsnotify_obj_type obj_type)
{
        switch (obj_type) {
        case FSNOTIFY_OBJ_TYPE_INODE:
                return &((struct inode *)obj)->i_fsnotify_marks;
        case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
                return &real_mount(obj)->mnt_fsnotify_marks;
        case FSNOTIFY_OBJ_TYPE_SB:
                return fsnotify_sb_marks(obj);
        default:
                return NULL;
        }
}

static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)
{
        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
                return &fsnotify_conn_inode(conn)->i_fsnotify_mask;
        else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
                return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;
        else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)
                return &fsnotify_conn_sb(conn)->s_fsnotify_mask;
        return NULL;
}

__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)
{
        if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))
                return 0;

        return READ_ONCE(*fsnotify_conn_mask_p(conn));
}

static void fsnotify_get_sb_watched_objects(struct super_block *sb)
{
        atomic_long_inc(fsnotify_sb_watched_objects(sb));
}

static void fsnotify_put_sb_watched_objects(struct super_block *sb)
{
        atomic_long_t *watched_objects = fsnotify_sb_watched_objects(sb);

        /* the superblock can go away after this decrement */
        if (atomic_long_dec_and_test(watched_objects))
                wake_up_var(watched_objects);
}

static void fsnotify_get_inode_ref(struct inode *inode)
{
        ihold(inode);
        fsnotify_get_sb_watched_objects(inode->i_sb);
}

static void fsnotify_put_inode_ref(struct inode *inode)
{
        /* read ->i_sb before the inode can go away */
        struct super_block *sb = inode->i_sb;

        iput(inode);
        fsnotify_put_sb_watched_objects(sb);
}

/*
 * Grab or drop watched objects reference depending on whether the connector
 * is attached and has any marks attached.
 */
static void fsnotify_update_sb_watchers(struct super_block *sb,
                                        struct fsnotify_mark_connector *conn)
{
        struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
        bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED;
        struct fsnotify_mark *first_mark = NULL;
        unsigned int highest_prio = 0;

        if (conn->obj)
                first_mark = hlist_entry_safe(conn->list.first,
                                              struct fsnotify_mark, obj_list);
        if (first_mark)
                highest_prio = first_mark->group->priority;
        if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM))
                highest_prio = 0;

        /*
         * If the highest priority of group watching this object is prio,
         * then watched object has a reference on counters [0..prio].
         * Update priority >= 1 watched objects counters.
         */
        for (unsigned int p = conn->prio + 1; p <= highest_prio; p++)
                atomic_long_inc(&sbinfo->watched_objects[p]);
        for (unsigned int p = conn->prio; p > highest_prio; p--)
                atomic_long_dec(&sbinfo->watched_objects[p]);
        conn->prio = highest_prio;

        /* Update priority >= 0 (a.k.a total) watched objects counter */
        BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0);
        if (first_mark && !is_watched) {
                conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_get_sb_watched_objects(sb);
        } else if (!first_mark && is_watched) {
                conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_put_sb_watched_objects(sb);
        }
}

/*
 * Grab or drop inode reference for the connector if needed.
 *
 * When it's time to drop the reference, we only clear the HAS_IREF flag and
 * return the inode object. fsnotify_drop_object() will be resonsible for doing
 * iput() outside of spinlocks. This happens when last mark that wanted iref is
 * detached.
 */
static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
                                          bool want_iref)
{
        bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
        struct inode *inode = NULL;

        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||
            want_iref == has_iref)
                return NULL;

        if (want_iref) {
                /* Pin inode if any mark wants inode refcount held */
                fsnotify_get_inode_ref(fsnotify_conn_inode(conn));
                conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;
        } else {
                /* Unpin inode after detach of last mark that wanted iref */
                inode = fsnotify_conn_inode(conn);
                conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;
        }

        return inode;
}

static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
        u32 new_mask = 0;
        bool want_iref = false;
        struct fsnotify_mark *mark;

        assert_spin_locked(&conn->lock);
        /* We can get detached connector here when inode is getting unlinked. */
        if (!fsnotify_valid_obj_type(conn->type))
                return NULL;
        hlist_for_each_entry(mark, &conn->list, obj_list) {
                if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED))
                        continue;
                new_mask |= fsnotify_calc_mask(mark);
                if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&
                    !(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF))
                        want_iref = true;
        }
        /*
         * We use WRITE_ONCE() to prevent silly compiler optimizations from
         * confusing readers not holding conn->lock with partial updates.
         */
        WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask);

        return fsnotify_update_iref(conn, want_iref);
}

static bool fsnotify_conn_watches_children(
                                        struct fsnotify_mark_connector *conn)
{
        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
                return false;

        return fsnotify_inode_watches_children(fsnotify_conn_inode(conn));
}

static void fsnotify_conn_set_children_dentry_flags(
                                        struct fsnotify_mark_connector *conn)
{
        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
                return;

        fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn));
}

/*
 * Calculate mask of events for a list of marks. The caller must make sure
 * connector and connector->obj cannot disappear under us.  Callers achieve
 * this by holding a mark->lock or mark->group->mark_mutex for a mark on this
 * list.
 */
void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
        bool update_children;

        if (!conn)
                return;

        spin_lock(&conn->lock);
        update_children = !fsnotify_conn_watches_children(conn);
        __fsnotify_recalc_mask(conn);
        update_children &= fsnotify_conn_watches_children(conn);
        spin_unlock(&conn->lock);
        /*
         * Set children's PARENT_WATCHED flags only if parent started watching.
         * When parent stops watching, we clear false positive PARENT_WATCHED
         * flags lazily in __fsnotify_parent().
         */
        if (update_children)
                fsnotify_conn_set_children_dentry_flags(conn);
}

/* Free all connectors queued for freeing once SRCU period ends */
static void fsnotify_connector_destroy_workfn(struct work_struct *work)
{
        struct fsnotify_mark_connector *conn, *free;

        spin_lock(&destroy_lock);
        conn = connector_destroy_list;
        connector_destroy_list = NULL;
        spin_unlock(&destroy_lock);

        synchronize_srcu(&fsnotify_mark_srcu);
        while (conn) {
                free = conn;
                conn = conn->destroy_next;
                kmem_cache_free(fsnotify_mark_connector_cachep, free);
        }
}

static void *fsnotify_detach_connector_from_object(
                                        struct fsnotify_mark_connector *conn,
                                        unsigned int *type)
{
        fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type);
        struct super_block *sb = fsnotify_connector_sb(conn);
        struct inode *inode = NULL;

        *type = conn->type;
        if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)
                return NULL;

        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
                inode = fsnotify_conn_inode(conn);
                inode->i_fsnotify_mask = 0;

                /* Unpin inode when detaching from connector */
                if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))
                        inode = NULL;
        } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
                fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
        } else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
                fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
        }

        rcu_assign_pointer(*connp, NULL);
        conn->obj = NULL;
        conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
        fsnotify_update_sb_watchers(sb, conn);

        return inode;
}

static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
{
        struct fsnotify_group *group = mark->group;

        if (WARN_ON_ONCE(!group))
                return;
        group->ops->free_mark(mark);
        fsnotify_put_group(group);
}

/* Drop object reference originally held by a connector */
static void fsnotify_drop_object(unsigned int type, void *objp)
{
        if (!objp)
                return;
        /* Currently only inode references are passed to be dropped */
        if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
                return;
        fsnotify_put_inode_ref(objp);
}

void fsnotify_put_mark(struct fsnotify_mark *mark)
{
        struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);
        void *objp = NULL;
        unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED;
        bool free_conn = false;

        /* Catch marks that were actually never attached to object */
        if (!conn) {
                if (refcount_dec_and_test(&mark->refcnt))
                        fsnotify_final_mark_destroy(mark);
                return;
        }

        /*
         * We have to be careful so that traversals of obj_list under lock can
         * safely grab mark reference.
         */
        if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))
                return;

        hlist_del_init_rcu(&mark->obj_list);
        if (hlist_empty(&conn->list)) {
                objp = fsnotify_detach_connector_from_object(conn, &type);
                free_conn = true;
        } else {
                struct super_block *sb = fsnotify_connector_sb(conn);

                /* Update watched objects after detaching mark */
                if (sb)
                        fsnotify_update_sb_watchers(sb, conn);
                objp = __fsnotify_recalc_mask(conn);
                type = conn->type;
        }
        WRITE_ONCE(mark->connector, NULL);
        spin_unlock(&conn->lock);

        fsnotify_drop_object(type, objp);

        if (free_conn) {
                spin_lock(&destroy_lock);
                conn->destroy_next = connector_destroy_list;
                connector_destroy_list = conn;
                spin_unlock(&destroy_lock);
                queue_work(system_unbound_wq, &connector_reaper_work);
        }
        /*
         * Note that we didn't update flags telling whether inode cares about
         * what's happening with children. We update these flags from
         * __fsnotify_parent() lazily when next event happens on one of our
         * children.
         */
        spin_lock(&destroy_lock);
        list_add(&mark->g_list, &destroy_list);
        spin_unlock(&destroy_lock);
        queue_delayed_work(system_unbound_wq, &reaper_work,
                           FSNOTIFY_REAPER_DELAY);
}
EXPORT_SYMBOL_GPL(fsnotify_put_mark);

/*
 * Get mark reference when we found the mark via lockless traversal of object
 * list. Mark can be already removed from the list by now and on its way to be
 * destroyed once SRCU period ends.
 *
 * Also pin the group so it doesn't disappear under us.
 */
static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
{
        if (!mark)
                return true;

        if (refcount_inc_not_zero(&mark->refcnt)) {
                spin_lock(&mark->lock);
                if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
                        /* mark is attached, group is still alive then */
                        atomic_inc(&mark->group->user_waits);
                        spin_unlock(&mark->lock);
                        return true;
                }
                spin_unlock(&mark->lock);
                fsnotify_put_mark(mark);
        }
        return false;
}

/*
 * Puts marks and wakes up group destruction if necessary.
 *
 * Pairs with fsnotify_get_mark_safe()
 */
static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
{
        if (mark) {
                struct fsnotify_group *group = mark->group;

                fsnotify_put_mark(mark);
                /*
                 * We abuse notification_waitq on group shutdown for waiting for
                 * all marks pinned when waiting for userspace.
                 */
                if (atomic_dec_and_test(&group->user_waits) && group->shutdown)
                        wake_up(&group->notification_waitq);
        }
}

bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
        __releases(&fsnotify_mark_srcu)
{
        int type;

        fsnotify_foreach_iter_type(type) {
                /* This can fail if mark is being removed */
                if (!fsnotify_get_mark_safe(iter_info->marks[type])) {
                        __release(&fsnotify_mark_srcu);
                        goto fail;
                }
        }

        /*
         * Now that both marks are pinned by refcount in the inode / vfsmount
         * lists, we can drop SRCU lock, and safely resume the list iteration
         * once userspace returns.
         */
        srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);

        return true;

fail:
        for (type--; type >= 0; type--)
                fsnotify_put_mark_wake(iter_info->marks[type]);
        return false;
}

void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
        __acquires(&fsnotify_mark_srcu)
{
        int type;

        iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
        fsnotify_foreach_iter_type(type)
                fsnotify_put_mark_wake(iter_info->marks[type]);
}

/*
 * Mark mark as detached, remove it from group list. Mark still stays in object
 * list until its last reference is dropped. Note that we rely on mark being
 * removed from group list before corresponding reference to it is dropped. In
 * particular we rely on mark->connector being valid while we hold
 * group->mark_mutex if we found the mark through g_list.
 *
 * Must be called with group->mark_mutex held. The caller must either hold
 * reference to the mark or be protected by fsnotify_mark_srcu.
 */
void fsnotify_detach_mark(struct fsnotify_mark *mark)
{
        fsnotify_group_assert_locked(mark->group);
        WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) &&
                     refcount_read(&mark->refcnt) < 1 +
                        !!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));

        spin_lock(&mark->lock);
        /* something else already called this function on this mark */
        if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
                spin_unlock(&mark->lock);
                return;
        }
        mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;
        list_del_init(&mark->g_list);
        spin_unlock(&mark->lock);

        /* Drop mark reference acquired in fsnotify_add_mark_locked() */
        fsnotify_put_mark(mark);
}

/*
 * Free fsnotify mark. The mark is actually only marked as being freed.  The
 * freeing is actually happening only once last reference to the mark is
 * dropped from a workqueue which first waits for srcu period end.
 *
 * Caller must have a reference to the mark or be protected by
 * fsnotify_mark_srcu.
 */
void fsnotify_free_mark(struct fsnotify_mark *mark)
{
        struct fsnotify_group *group = mark->group;

        spin_lock(&mark->lock);
        /* something else already called this function on this mark */
        if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {
                spin_unlock(&mark->lock);
                return;
        }
        mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;
        spin_unlock(&mark->lock);

        /*
         * Some groups like to know that marks are being freed.  This is a
         * callback to the group function to let it know that this mark
         * is being freed.
         */
        if (group->ops->freeing_mark)
                group->ops->freeing_mark(mark, group);
}

void fsnotify_destroy_mark(struct fsnotify_mark *mark,
                           struct fsnotify_group *group)
{
        fsnotify_group_lock(group);
        fsnotify_detach_mark(mark);
        fsnotify_group_unlock(group);
        fsnotify_free_mark(mark);
}
EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);

/*
 * Sorting function for lists of fsnotify marks.
 *
 * Fanotify supports different notification classes (reflected as priority of
 * notification group). Events shall be passed to notification groups in
 * decreasing priority order. To achieve this marks in notification lists for
 * inodes and vfsmounts are sorted so that priorities of corresponding groups
 * are descending.
 *
 * Furthermore correct handling of the ignore mask requires processing inode
 * and vfsmount marks of each group together. Using the group address as
 * further sort criterion provides a unique sorting order and thus we can
 * merge inode and vfsmount lists of marks in linear time and find groups
 * present in both lists.
 *
 * A return value of 1 signifies that b has priority over a.
 * A return value of 0 signifies that the two marks have to be handled together.
 * A return value of -1 signifies that a has priority over b.
 */
int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)
{
        if (a == b)
                return 0;
        if (!a)
                return 1;
        if (!b)
                return -1;
        if (a->priority < b->priority)
                return 1;
        if (a->priority > b->priority)
                return -1;
        if (a < b)
                return 1;
        return -1;
}

static int fsnotify_attach_info_to_sb(struct super_block *sb)
{
        struct fsnotify_sb_info *sbinfo;

        /* sb info is freed on fsnotify_sb_delete() */
        sbinfo = kzalloc(sizeof(*sbinfo), GFP_KERNEL);
        if (!sbinfo)
                return -ENOMEM;

        /*
         * cmpxchg() provides the barrier so that callers of fsnotify_sb_info()
         * will observe an initialized structure
         */
        if (cmpxchg(&sb->s_fsnotify_info, NULL, sbinfo)) {
                /* Someone else created sbinfo for us */
                kfree(sbinfo);
        }
        return 0;
}

static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
                                               void *obj, unsigned int obj_type)
{
        struct fsnotify_mark_connector *conn;

        conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL);
        if (!conn)
                return -ENOMEM;
        spin_lock_init(&conn->lock);
        INIT_HLIST_HEAD(&conn->list);
        conn->flags = 0;
        conn->prio = 0;
        conn->type = obj_type;
        conn->obj = obj;

        /*
         * cmpxchg() provides the barrier so that readers of *connp can see
         * only initialized structure
         */
        if (cmpxchg(connp, NULL, conn)) {
                /* Someone else created list structure for us */
                kmem_cache_free(fsnotify_mark_connector_cachep, conn);
        }
        return 0;
}

/*
 * Get mark connector, make sure it is alive and return with its lock held.
 * This is for users that get connector pointer from inode or mount. Users that
 * hold reference to a mark on the list may directly lock connector->lock as
 * they are sure list cannot go away under them.
 */
static struct fsnotify_mark_connector *fsnotify_grab_connector(
                                                fsnotify_connp_t *connp)
{
        struct fsnotify_mark_connector *conn;
        int idx;

        idx = srcu_read_lock(&fsnotify_mark_srcu);
        conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
        if (!conn)
                goto out;
        spin_lock(&conn->lock);
        if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
                spin_unlock(&conn->lock);
                srcu_read_unlock(&fsnotify_mark_srcu, idx);
                return NULL;
        }
out:
        srcu_read_unlock(&fsnotify_mark_srcu, idx);
        return conn;
}

/*
 * Add mark into proper place in given list of marks. These marks may be used
 * for the fsnotify backend to determine which event types should be delivered
 * to which group and for which inodes. These marks are ordered according to
 * priority, highest number first, and then by the group's location in memory.
 */
static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj,
                                  unsigned int obj_type, int add_flags)
{
        struct super_block *sb = fsnotify_object_sb(obj, obj_type);
        struct fsnotify_mark *lmark, *last = NULL;
        struct fsnotify_mark_connector *conn;
        fsnotify_connp_t *connp;
        int cmp;
        int err = 0;

        if (WARN_ON(!fsnotify_valid_obj_type(obj_type)))
                return -EINVAL;

        /*
         * Attach the sb info before attaching a connector to any object on sb.
         * The sb info will remain attached as long as sb lives.
         */
        if (!fsnotify_sb_info(sb)) {
                err = fsnotify_attach_info_to_sb(sb);
                if (err)
                        return err;
        }

        connp = fsnotify_object_connp(obj, obj_type);
restart:
        spin_lock(&mark->lock);
        conn = fsnotify_grab_connector(connp);
        if (!conn) {
                spin_unlock(&mark->lock);
                err = fsnotify_attach_connector_to_object(connp, obj, obj_type);
                if (err)
                        return err;
                goto restart;
        }

        /* is mark the first mark? */
        if (hlist_empty(&conn->list)) {
                hlist_add_head_rcu(&mark->obj_list, &conn->list);
                goto added;
        }

        /* should mark be in the middle of the current list? */
        hlist_for_each_entry(lmark, &conn->list, obj_list) {
                last = lmark;

                if ((lmark->group == mark->group) &&
                    (lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) &&
                    !(mark->group->flags & FSNOTIFY_GROUP_DUPS)) {
                        err = -EEXIST;
                        goto out_err;
                }

                cmp = fsnotify_compare_groups(lmark->group, mark->group);
                if (cmp >= 0) {
                        hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);
                        goto added;
                }
        }

        BUG_ON(last == NULL);
        /* mark should be the last entry.  last is the current last entry */
        hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);
added:
        fsnotify_update_sb_watchers(sb, conn);
        /*
         * Since connector is attached to object using cmpxchg() we are
         * guaranteed that connector initialization is fully visible by anyone
         * seeing mark->connector set.
         */
        WRITE_ONCE(mark->connector, conn);
out_err:
        spin_unlock(&conn->lock);
        spin_unlock(&mark->lock);
        return err;
}

/*
 * Attach an initialized mark to a given group and fs object.
 * These marks may be used for the fsnotify backend to determine which
 * event types should be delivered to which group.
 */
int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
                             void *obj, unsigned int obj_type,
                             int add_flags)
{
        struct fsnotify_group *group = mark->group;
        int ret = 0;

        fsnotify_group_assert_locked(group);

        /*
         * LOCKING ORDER!!!!
         * group->mark_mutex
         * mark->lock
         * mark->connector->lock
         */
        spin_lock(&mark->lock);
        mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;

        list_add(&mark->g_list, &group->marks_list);
        fsnotify_get_mark(mark); /* for g_list */
        spin_unlock(&mark->lock);

        ret = fsnotify_add_mark_list(mark, obj, obj_type, add_flags);
        if (ret)
                goto err;

        fsnotify_recalc_mask(mark->connector);

        return ret;
err:
        spin_lock(&mark->lock);
        mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |
                         FSNOTIFY_MARK_FLAG_ATTACHED);
        list_del_init(&mark->g_list);
        spin_unlock(&mark->lock);

        fsnotify_put_mark(mark);
        return ret;
}

int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj,
                      unsigned int obj_type, int add_flags)
{
        int ret;
        struct fsnotify_group *group = mark->group;

        fsnotify_group_lock(group);
        ret = fsnotify_add_mark_locked(mark, obj, obj_type, add_flags);
        fsnotify_group_unlock(group);
        return ret;
}
EXPORT_SYMBOL_GPL(fsnotify_add_mark);

/*
 * Given a list of marks, find the mark associated with given group. If found
 * take a reference to that mark and return it, else return NULL.
 */
struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type,
                                         struct fsnotify_group *group)
{
        fsnotify_connp_t *connp = fsnotify_object_connp(obj, obj_type);
        struct fsnotify_mark_connector *conn;
        struct fsnotify_mark *mark;

        if (!connp)
                return NULL;

        conn = fsnotify_grab_connector(connp);
        if (!conn)
                return NULL;

        hlist_for_each_entry(mark, &conn->list, obj_list) {
                if (mark->group == group &&
                    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
                        fsnotify_get_mark(mark);
                        spin_unlock(&conn->lock);
                        return mark;
                }
        }
        spin_unlock(&conn->lock);
        return NULL;
}
EXPORT_SYMBOL_GPL(fsnotify_find_mark);

/* Clear any marks in a group with given type mask */
void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
                                   unsigned int obj_type)
{
        struct fsnotify_mark *lmark, *mark;
        LIST_HEAD(to_free);
        struct list_head *head = &to_free;

        /* Skip selection step if we want to clear all marks. */
        if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) {
                head = &group->marks_list;
                goto clear;
        }
        /*
         * We have to be really careful here. Anytime we drop mark_mutex, e.g.
         * fsnotify_clear_marks_by_inode() can come and free marks. Even in our
         * to_free list so we have to use mark_mutex even when accessing that
         * list. And freeing mark requires us to drop mark_mutex. So we can
         * reliably free only the first mark in the list. That's why we first
         * move marks to free to to_free list in one go and then free marks in
         * to_free list one by one.
         */
        fsnotify_group_lock(group);
        list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
                if (mark->connector->type == obj_type)
                        list_move(&mark->g_list, &to_free);
        }
        fsnotify_group_unlock(group);

clear:
        while (1) {
                fsnotify_group_lock(group);
                if (list_empty(head)) {
                        fsnotify_group_unlock(group);
                        break;
                }
                mark = list_first_entry(head, struct fsnotify_mark, g_list);
                fsnotify_get_mark(mark);
                fsnotify_detach_mark(mark);
                fsnotify_group_unlock(group);
                fsnotify_free_mark(mark);
                fsnotify_put_mark(mark);
        }
}

/* Destroy all marks attached to an object via connector */
void fsnotify_destroy_marks(fsnotify_connp_t *connp)
{
        struct fsnotify_mark_connector *conn;
        struct fsnotify_mark *mark, *old_mark = NULL;
        void *objp;
        unsigned int type;

        conn = fsnotify_grab_connector(connp);
        if (!conn)
                return;
        /*
         * We have to be careful since we can race with e.g.
         * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the
         * list can get modified. However we are holding mark reference and
         * thus our mark cannot be removed from obj_list so we can continue
         * iteration after regaining conn->lock.
         */
        hlist_for_each_entry(mark, &conn->list, obj_list) {
                fsnotify_get_mark(mark);
                spin_unlock(&conn->lock);
                if (old_mark)
                        fsnotify_put_mark(old_mark);
                old_mark = mark;
                fsnotify_destroy_mark(mark, mark->group);
                spin_lock(&conn->lock);
        }
        /*
         * Detach list from object now so that we don't pin inode until all
         * mark references get dropped. It would lead to strange results such
         * as delaying inode deletion or blocking unmount.
         */
        objp = fsnotify_detach_connector_from_object(conn, &type);
        spin_unlock(&conn->lock);
        if (old_mark)
                fsnotify_put_mark(old_mark);
        fsnotify_drop_object(type, objp);
}

/*
 * Nothing fancy, just initialize lists and locks and counters.
 */
void fsnotify_init_mark(struct fsnotify_mark *mark,
                        struct fsnotify_group *group)
{
        memset(mark, 0, sizeof(*mark));
        spin_lock_init(&mark->lock);
        refcount_set(&mark->refcnt, 1);
        fsnotify_get_group(group);
        mark->group = group;
        WRITE_ONCE(mark->connector, NULL);
}
EXPORT_SYMBOL_GPL(fsnotify_init_mark);

/*
 * Destroy all marks in destroy_list, waits for SRCU period to finish before
 * actually freeing marks.
 */
static void fsnotify_mark_destroy_workfn(struct work_struct *work)
{
        struct fsnotify_mark *mark, *next;
        struct list_head private_destroy_list;

        spin_lock(&destroy_lock);
        /* exchange the list head */
        list_replace_init(&destroy_list, &private_destroy_list);
        spin_unlock(&destroy_lock);

        synchronize_srcu(&fsnotify_mark_srcu);

        list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {
                list_del_init(&mark->g_list);
                fsnotify_final_mark_destroy(mark);
        }
}

/* Wait for all marks queued for destruction to be actually destroyed */
void fsnotify_wait_marks_destroyed(void)
{
        flush_delayed_work(&reaper_work);
}
EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed);






















































    2 










    1 


    1 







































    1 













    1 


















































































































































































































































































   35 














    2 


















   37 

















   32 
    5 



   21 

   15 
   37 



   37 






   37 





   35 






    2 






    2 




    2 




























   37 












   37 




   37 





   37 


   37 




   15 

   22 






   20 
    5 
   13 

   37 



   28 


    9 

   32 




    5 

   22 



   15 

   37 











   37 








   37 




   37 














    1 
































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2005 Marc Kleine-Budde, Pengutronix
 * Copyright (C) 2006 Andrey Volkov, Varma Electronics
 * Copyright (C) 2008-2009 Wolfgang Grandegger <wg@grandegger.com>
 * Copyright (C) 2021 Vincent Mailhol <mailhol.vincent@wanadoo.fr>
 */

#include <linux/can/dev.h>
#include <net/rtnetlink.h>

static const struct nla_policy can_policy[IFLA_CAN_MAX + 1] = {
        [IFLA_CAN_STATE] = { .type = NLA_U32 },
        [IFLA_CAN_CTRLMODE] = { .len = sizeof(struct can_ctrlmode) },
        [IFLA_CAN_RESTART_MS] = { .type = NLA_U32 },
        [IFLA_CAN_RESTART] = { .type = NLA_U32 },
        [IFLA_CAN_BITTIMING] = { .len = sizeof(struct can_bittiming) },
        [IFLA_CAN_BITTIMING_CONST] = { .len = sizeof(struct can_bittiming_const) },
        [IFLA_CAN_CLOCK] = { .len = sizeof(struct can_clock) },
        [IFLA_CAN_BERR_COUNTER] = { .len = sizeof(struct can_berr_counter) },
        [IFLA_CAN_DATA_BITTIMING] = { .len = sizeof(struct can_bittiming) },
        [IFLA_CAN_DATA_BITTIMING_CONST]        = { .len = sizeof(struct can_bittiming_const) },
        [IFLA_CAN_TERMINATION] = { .type = NLA_U16 },
        [IFLA_CAN_TDC] = { .type = NLA_NESTED },
        [IFLA_CAN_CTRLMODE_EXT] = { .type = NLA_NESTED },
};

static const struct nla_policy can_tdc_policy[IFLA_CAN_TDC_MAX + 1] = {
        [IFLA_CAN_TDC_TDCV_MIN] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCV_MAX] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCO_MIN] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCO_MAX] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCF_MIN] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCF_MAX] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCV] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCO] = { .type = NLA_U32 },
        [IFLA_CAN_TDC_TDCF] = { .type = NLA_U32 },
};

static int can_validate_bittiming(const struct can_bittiming *bt,
                                  struct netlink_ext_ack *extack)
{
        /* sample point is in one-tenth of a percent */
        if (bt->sample_point >= 1000) {
                NL_SET_ERR_MSG(extack, "sample point must be between 0 and 100%");

                return -EINVAL;
        }

        return 0;
}

static int can_validate(struct nlattr *tb[], struct nlattr *data[],
                        struct netlink_ext_ack *extack)
{
        bool is_can_fd = false;
        int err;

        /* Make sure that valid CAN FD configurations always consist of
         * - nominal/arbitration bittiming
         * - data bittiming
         * - control mode with CAN_CTRLMODE_FD set
         * - TDC parameters are coherent (details below)
         */

        if (!data)
                return 0;

        if (data[IFLA_CAN_CTRLMODE]) {
                struct can_ctrlmode *cm = nla_data(data[IFLA_CAN_CTRLMODE]);
                u32 tdc_flags = cm->flags & CAN_CTRLMODE_TDC_MASK;

                is_can_fd = cm->flags & cm->mask & CAN_CTRLMODE_FD;

                /* CAN_CTRLMODE_TDC_{AUTO,MANUAL} are mutually exclusive */
                if (tdc_flags == CAN_CTRLMODE_TDC_MASK)
                        return -EOPNOTSUPP;
                /* If one of the CAN_CTRLMODE_TDC_* flag is set then
                 * TDC must be set and vice-versa
                 */
                if (!!tdc_flags != !!data[IFLA_CAN_TDC])
                        return -EOPNOTSUPP;
                /* If providing TDC parameters, at least TDCO is
                 * needed. TDCV is needed if and only if
                 * CAN_CTRLMODE_TDC_MANUAL is set
                 */
                if (data[IFLA_CAN_TDC]) {
                        struct nlattr *tb_tdc[IFLA_CAN_TDC_MAX + 1];

                        err = nla_parse_nested(tb_tdc, IFLA_CAN_TDC_MAX,
                                               data[IFLA_CAN_TDC],
                                               can_tdc_policy, extack);
                        if (err)
                                return err;

                        if (tb_tdc[IFLA_CAN_TDC_TDCV]) {
                                if (tdc_flags & CAN_CTRLMODE_TDC_AUTO)
                                        return -EOPNOTSUPP;
                        } else {
                                if (tdc_flags & CAN_CTRLMODE_TDC_MANUAL)
                                        return -EOPNOTSUPP;
                        }

                        if (!tb_tdc[IFLA_CAN_TDC_TDCO])
                                return -EOPNOTSUPP;
                }
        }

        if (data[IFLA_CAN_BITTIMING]) {
                struct can_bittiming bt;

                memcpy(&bt, nla_data(data[IFLA_CAN_BITTIMING]), sizeof(bt));
                err = can_validate_bittiming(&bt, extack);
                if (err)
                        return err;
        }

        if (is_can_fd) {
                if (!data[IFLA_CAN_BITTIMING] || !data[IFLA_CAN_DATA_BITTIMING])
                        return -EOPNOTSUPP;
        }

        if (data[IFLA_CAN_DATA_BITTIMING] || data[IFLA_CAN_TDC]) {
                if (!is_can_fd)
                        return -EOPNOTSUPP;
        }

        if (data[IFLA_CAN_DATA_BITTIMING]) {
                struct can_bittiming bt;

                memcpy(&bt, nla_data(data[IFLA_CAN_DATA_BITTIMING]), sizeof(bt));
                err = can_validate_bittiming(&bt, extack);
                if (err)
                        return err;
        }

        return 0;
}

static int can_tdc_changelink(struct can_priv *priv, const struct nlattr *nla,
                              struct netlink_ext_ack *extack)
{
        struct nlattr *tb_tdc[IFLA_CAN_TDC_MAX + 1];
        struct can_tdc tdc = { 0 };
        const struct can_tdc_const *tdc_const = priv->tdc_const;
        int err;

        if (!tdc_const || !can_tdc_is_enabled(priv))
                return -EOPNOTSUPP;

        err = nla_parse_nested(tb_tdc, IFLA_CAN_TDC_MAX, nla,
                               can_tdc_policy, extack);
        if (err)
                return err;

        if (tb_tdc[IFLA_CAN_TDC_TDCV]) {
                u32 tdcv = nla_get_u32(tb_tdc[IFLA_CAN_TDC_TDCV]);

                if (tdcv < tdc_const->tdcv_min || tdcv > tdc_const->tdcv_max)
                        return -EINVAL;

                tdc.tdcv = tdcv;
        }

        if (tb_tdc[IFLA_CAN_TDC_TDCO]) {
                u32 tdco = nla_get_u32(tb_tdc[IFLA_CAN_TDC_TDCO]);

                if (tdco < tdc_const->tdco_min || tdco > tdc_const->tdco_max)
                        return -EINVAL;

                tdc.tdco = tdco;
        }

        if (tb_tdc[IFLA_CAN_TDC_TDCF]) {
                u32 tdcf = nla_get_u32(tb_tdc[IFLA_CAN_TDC_TDCF]);

                if (tdcf < tdc_const->tdcf_min || tdcf > tdc_const->tdcf_max)
                        return -EINVAL;

                tdc.tdcf = tdcf;
        }

        priv->tdc = tdc;

        return 0;
}

static int can_changelink(struct net_device *dev, struct nlattr *tb[],
                          struct nlattr *data[],
                          struct netlink_ext_ack *extack)
{
        struct can_priv *priv = netdev_priv(dev);
        u32 tdc_mask = 0;
        int err;

        /* We need synchronization with dev->stop() */
        ASSERT_RTNL();

        if (data[IFLA_CAN_CTRLMODE]) {
                struct can_ctrlmode *cm;
                u32 ctrlstatic;
                u32 maskedflags;

                /* Do not allow changing controller mode while running */
                if (dev->flags & IFF_UP)
                        return -EBUSY;
                cm = nla_data(data[IFLA_CAN_CTRLMODE]);
                ctrlstatic = can_get_static_ctrlmode(priv);
                maskedflags = cm->flags & cm->mask;

                /* check whether provided bits are allowed to be passed */
                if (maskedflags & ~(priv->ctrlmode_supported | ctrlstatic))
                        return -EOPNOTSUPP;

                /* do not check for static fd-non-iso if 'fd' is disabled */
                if (!(maskedflags & CAN_CTRLMODE_FD))
                        ctrlstatic &= ~CAN_CTRLMODE_FD_NON_ISO;

                /* make sure static options are provided by configuration */
                if ((maskedflags & ctrlstatic) != ctrlstatic)
                        return -EOPNOTSUPP;

                /* clear bits to be modified and copy the flag values */
                priv->ctrlmode &= ~cm->mask;
                priv->ctrlmode |= maskedflags;

                /* CAN_CTRLMODE_FD can only be set when driver supports FD */
                if (priv->ctrlmode & CAN_CTRLMODE_FD) {
                        dev->mtu = CANFD_MTU;
                } else {
                        dev->mtu = CAN_MTU;
                        memset(&priv->data_bittiming, 0,
                               sizeof(priv->data_bittiming));
                        priv->ctrlmode &= ~CAN_CTRLMODE_TDC_MASK;
                        memset(&priv->tdc, 0, sizeof(priv->tdc));
                }

                tdc_mask = cm->mask & CAN_CTRLMODE_TDC_MASK;
                /* CAN_CTRLMODE_TDC_{AUTO,MANUAL} are mutually
                 * exclusive: make sure to turn the other one off
                 */
                if (tdc_mask)
                        priv->ctrlmode &= cm->flags | ~CAN_CTRLMODE_TDC_MASK;
        }

        if (data[IFLA_CAN_BITTIMING]) {
                struct can_bittiming bt;

                /* Do not allow changing bittiming while running */
                if (dev->flags & IFF_UP)
                        return -EBUSY;

                /* Calculate bittiming parameters based on
                 * bittiming_const if set, otherwise pass bitrate
                 * directly via do_set_bitrate(). Bail out if neither
                 * is given.
                 */
                if (!priv->bittiming_const && !priv->do_set_bittiming &&
                    !priv->bitrate_const)
                        return -EOPNOTSUPP;

                memcpy(&bt, nla_data(data[IFLA_CAN_BITTIMING]), sizeof(bt));
                err = can_get_bittiming(dev, &bt,
                                        priv->bittiming_const,
                                        priv->bitrate_const,
                                        priv->bitrate_const_cnt,
                                        extack);
                if (err)
                        return err;

                if (priv->bitrate_max && bt.bitrate > priv->bitrate_max) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "arbitration bitrate %u bps surpasses transceiver capabilities of %u bps",
                                           bt.bitrate, priv->bitrate_max);
                        return -EINVAL;
                }

                memcpy(&priv->bittiming, &bt, sizeof(bt));

                if (priv->do_set_bittiming) {
                        /* Finally, set the bit-timing registers */
                        err = priv->do_set_bittiming(dev);
                        if (err)
                                return err;
                }
        }

        if (data[IFLA_CAN_RESTART_MS]) {
                /* Do not allow changing restart delay while running */
                if (dev->flags & IFF_UP)
                        return -EBUSY;
                priv->restart_ms = nla_get_u32(data[IFLA_CAN_RESTART_MS]);
        }

        if (data[IFLA_CAN_RESTART]) {
                /* Do not allow a restart while not running */
                if (!(dev->flags & IFF_UP))
                        return -EINVAL;
                err = can_restart_now(dev);
                if (err)
                        return err;
        }

        if (data[IFLA_CAN_DATA_BITTIMING]) {
                struct can_bittiming dbt;

                /* Do not allow changing bittiming while running */
                if (dev->flags & IFF_UP)
                        return -EBUSY;

                /* Calculate bittiming parameters based on
                 * data_bittiming_const if set, otherwise pass bitrate
                 * directly via do_set_bitrate(). Bail out if neither
                 * is given.
                 */
                if (!priv->data_bittiming_const && !priv->do_set_data_bittiming &&
                    !priv->data_bitrate_const)
                        return -EOPNOTSUPP;

                memcpy(&dbt, nla_data(data[IFLA_CAN_DATA_BITTIMING]),
                       sizeof(dbt));
                err = can_get_bittiming(dev, &dbt,
                                        priv->data_bittiming_const,
                                        priv->data_bitrate_const,
                                        priv->data_bitrate_const_cnt,
                                        extack);
                if (err)
                        return err;

                if (priv->bitrate_max && dbt.bitrate > priv->bitrate_max) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "CANFD data bitrate %u bps surpasses transceiver capabilities of %u bps",
                                           dbt.bitrate, priv->bitrate_max);
                        return -EINVAL;
                }

                memset(&priv->tdc, 0, sizeof(priv->tdc));
                if (data[IFLA_CAN_TDC]) {
                        /* TDC parameters are provided: use them */
                        err = can_tdc_changelink(priv, data[IFLA_CAN_TDC],
                                                 extack);
                        if (err) {
                                priv->ctrlmode &= ~CAN_CTRLMODE_TDC_MASK;
                                return err;
                        }
                } else if (!tdc_mask) {
                        /* Neither of TDC parameters nor TDC flags are
                         * provided: do calculation
                         */
                        can_calc_tdco(&priv->tdc, priv->tdc_const, &dbt,
                                      &priv->ctrlmode, priv->ctrlmode_supported);
                } /* else: both CAN_CTRLMODE_TDC_{AUTO,MANUAL} are explicitly
                   * turned off. TDC is disabled: do nothing
                   */

                memcpy(&priv->data_bittiming, &dbt, sizeof(dbt));

                if (priv->do_set_data_bittiming) {
                        /* Finally, set the bit-timing registers */
                        err = priv->do_set_data_bittiming(dev);
                        if (err)
                                return err;
                }
        }

        if (data[IFLA_CAN_TERMINATION]) {
                const u16 termval = nla_get_u16(data[IFLA_CAN_TERMINATION]);
                const unsigned int num_term = priv->termination_const_cnt;
                unsigned int i;

                if (!priv->do_set_termination)
                        return -EOPNOTSUPP;

                /* check whether given value is supported by the interface */
                for (i = 0; i < num_term; i++) {
                        if (termval == priv->termination_const[i])
                                break;
                }
                if (i >= num_term)
                        return -EINVAL;

                /* Finally, set the termination value */
                err = priv->do_set_termination(dev, termval);
                if (err)
                        return err;

                priv->termination = termval;
        }

        return 0;
}

static size_t can_tdc_get_size(const struct net_device *dev)
{
        struct can_priv *priv = netdev_priv(dev);
        size_t size;

        if (!priv->tdc_const)
                return 0;

        size = nla_total_size(0);                        /* nest IFLA_CAN_TDC */
        if (priv->ctrlmode_supported & CAN_CTRLMODE_TDC_MANUAL) {
                size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCV_MIN */
                size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCV_MAX */
        }
        size += nla_total_size(sizeof(u32));                /* IFLA_CAN_TDCO_MIN */
        size += nla_total_size(sizeof(u32));                /* IFLA_CAN_TDCO_MAX */
        if (priv->tdc_const->tdcf_max) {
                size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCF_MIN */
                size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCF_MAX */
        }

        if (can_tdc_is_enabled(priv)) {
                if (priv->ctrlmode & CAN_CTRLMODE_TDC_MANUAL ||
                    priv->do_get_auto_tdcv)
                        size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCV */
                size += nla_total_size(sizeof(u32));                /* IFLA_CAN_TDCO */
                if (priv->tdc_const->tdcf_max)
                        size += nla_total_size(sizeof(u32));        /* IFLA_CAN_TDCF */
        }

        return size;
}

static size_t can_ctrlmode_ext_get_size(void)
{
        return nla_total_size(0) +                /* nest IFLA_CAN_CTRLMODE_EXT */
                nla_total_size(sizeof(u32));        /* IFLA_CAN_CTRLMODE_SUPPORTED */
}

static size_t can_get_size(const struct net_device *dev)
{
        struct can_priv *priv = netdev_priv(dev);
        size_t size = 0;

        if (priv->bittiming.bitrate)                                /* IFLA_CAN_BITTIMING */
                size += nla_total_size(sizeof(struct can_bittiming));
        if (priv->bittiming_const)                                /* IFLA_CAN_BITTIMING_CONST */
                size += nla_total_size(sizeof(struct can_bittiming_const));
        size += nla_total_size(sizeof(struct can_clock));        /* IFLA_CAN_CLOCK */
        size += nla_total_size(sizeof(u32));                        /* IFLA_CAN_STATE */
        size += nla_total_size(sizeof(struct can_ctrlmode));        /* IFLA_CAN_CTRLMODE */
        size += nla_total_size(sizeof(u32));                        /* IFLA_CAN_RESTART_MS */
        if (priv->do_get_berr_counter)                                /* IFLA_CAN_BERR_COUNTER */
                size += nla_total_size(sizeof(struct can_berr_counter));
        if (priv->data_bittiming.bitrate)                        /* IFLA_CAN_DATA_BITTIMING */
                size += nla_total_size(sizeof(struct can_bittiming));
        if (priv->data_bittiming_const)                                /* IFLA_CAN_DATA_BITTIMING_CONST */
                size += nla_total_size(sizeof(struct can_bittiming_const));
        if (priv->termination_const) {
                size += nla_total_size(sizeof(priv->termination));                /* IFLA_CAN_TERMINATION */
                size += nla_total_size(sizeof(*priv->termination_const) *        /* IFLA_CAN_TERMINATION_CONST */
                                       priv->termination_const_cnt);
        }
        if (priv->bitrate_const)                                /* IFLA_CAN_BITRATE_CONST */
                size += nla_total_size(sizeof(*priv->bitrate_const) *
                                       priv->bitrate_const_cnt);
        if (priv->data_bitrate_const)                                /* IFLA_CAN_DATA_BITRATE_CONST */
                size += nla_total_size(sizeof(*priv->data_bitrate_const) *
                                       priv->data_bitrate_const_cnt);
        size += sizeof(priv->bitrate_max);                        /* IFLA_CAN_BITRATE_MAX */
        size += can_tdc_get_size(dev);                                /* IFLA_CAN_TDC */
        size += can_ctrlmode_ext_get_size();                        /* IFLA_CAN_CTRLMODE_EXT */

        return size;
}

static int can_tdc_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct nlattr *nest;
        struct can_priv *priv = netdev_priv(dev);
        struct can_tdc *tdc = &priv->tdc;
        const struct can_tdc_const *tdc_const = priv->tdc_const;

        if (!tdc_const)
                return 0;

        nest = nla_nest_start(skb, IFLA_CAN_TDC);
        if (!nest)
                return -EMSGSIZE;

        if (priv->ctrlmode_supported & CAN_CTRLMODE_TDC_MANUAL &&
            (nla_put_u32(skb, IFLA_CAN_TDC_TDCV_MIN, tdc_const->tdcv_min) ||
             nla_put_u32(skb, IFLA_CAN_TDC_TDCV_MAX, tdc_const->tdcv_max)))
                goto err_cancel;
        if (nla_put_u32(skb, IFLA_CAN_TDC_TDCO_MIN, tdc_const->tdco_min) ||
            nla_put_u32(skb, IFLA_CAN_TDC_TDCO_MAX, tdc_const->tdco_max))
                goto err_cancel;
        if (tdc_const->tdcf_max &&
            (nla_put_u32(skb, IFLA_CAN_TDC_TDCF_MIN, tdc_const->tdcf_min) ||
             nla_put_u32(skb, IFLA_CAN_TDC_TDCF_MAX, tdc_const->tdcf_max)))
                goto err_cancel;

        if (can_tdc_is_enabled(priv)) {
                u32 tdcv;
                int err = -EINVAL;

                if (priv->ctrlmode & CAN_CTRLMODE_TDC_MANUAL) {
                        tdcv = tdc->tdcv;
                        err = 0;
                } else if (priv->do_get_auto_tdcv) {
                        err = priv->do_get_auto_tdcv(dev, &tdcv);
                }
                if (!err && nla_put_u32(skb, IFLA_CAN_TDC_TDCV, tdcv))
                        goto err_cancel;
                if (nla_put_u32(skb, IFLA_CAN_TDC_TDCO, tdc->tdco))
                        goto err_cancel;
                if (tdc_const->tdcf_max &&
                    nla_put_u32(skb, IFLA_CAN_TDC_TDCF, tdc->tdcf))
                        goto err_cancel;
        }

        nla_nest_end(skb, nest);
        return 0;

err_cancel:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int can_ctrlmode_ext_fill_info(struct sk_buff *skb,
                                      const struct can_priv *priv)
{
        struct nlattr *nest;

        nest = nla_nest_start(skb, IFLA_CAN_CTRLMODE_EXT);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u32(skb, IFLA_CAN_CTRLMODE_SUPPORTED,
                        priv->ctrlmode_supported)) {
                nla_nest_cancel(skb, nest);
                return -EMSGSIZE;
        }

        nla_nest_end(skb, nest);
        return 0;
}

static int can_fill_info(struct sk_buff *skb, const struct net_device *dev)
{
        struct can_priv *priv = netdev_priv(dev);
        struct can_ctrlmode cm = {.flags = priv->ctrlmode};
        struct can_berr_counter bec = { };
        enum can_state state = priv->state;

        if (priv->do_get_state)
                priv->do_get_state(dev, &state);

        if ((priv->bittiming.bitrate != CAN_BITRATE_UNSET &&
             priv->bittiming.bitrate != CAN_BITRATE_UNKNOWN &&
             nla_put(skb, IFLA_CAN_BITTIMING,
                     sizeof(priv->bittiming), &priv->bittiming)) ||

            (priv->bittiming_const &&
             nla_put(skb, IFLA_CAN_BITTIMING_CONST,
                     sizeof(*priv->bittiming_const), priv->bittiming_const)) ||

            nla_put(skb, IFLA_CAN_CLOCK, sizeof(priv->clock), &priv->clock) ||
            nla_put_u32(skb, IFLA_CAN_STATE, state) ||
            nla_put(skb, IFLA_CAN_CTRLMODE, sizeof(cm), &cm) ||
            nla_put_u32(skb, IFLA_CAN_RESTART_MS, priv->restart_ms) ||

            (priv->do_get_berr_counter &&
             !priv->do_get_berr_counter(dev, &bec) &&
             nla_put(skb, IFLA_CAN_BERR_COUNTER, sizeof(bec), &bec)) ||

            (priv->data_bittiming.bitrate &&
             nla_put(skb, IFLA_CAN_DATA_BITTIMING,
                     sizeof(priv->data_bittiming), &priv->data_bittiming)) ||

            (priv->data_bittiming_const &&
             nla_put(skb, IFLA_CAN_DATA_BITTIMING_CONST,
                     sizeof(*priv->data_bittiming_const),
                     priv->data_bittiming_const)) ||

            (priv->termination_const &&
             (nla_put_u16(skb, IFLA_CAN_TERMINATION, priv->termination) ||
              nla_put(skb, IFLA_CAN_TERMINATION_CONST,
                      sizeof(*priv->termination_const) *
                      priv->termination_const_cnt,
                      priv->termination_const))) ||

            (priv->bitrate_const &&
             nla_put(skb, IFLA_CAN_BITRATE_CONST,
                     sizeof(*priv->bitrate_const) *
                     priv->bitrate_const_cnt,
                     priv->bitrate_const)) ||

            (priv->data_bitrate_const &&
             nla_put(skb, IFLA_CAN_DATA_BITRATE_CONST,
                     sizeof(*priv->data_bitrate_const) *
                     priv->data_bitrate_const_cnt,
                     priv->data_bitrate_const)) ||

            (nla_put(skb, IFLA_CAN_BITRATE_MAX,
                     sizeof(priv->bitrate_max),
                     &priv->bitrate_max)) ||

            can_tdc_fill_info(skb, dev) ||

            can_ctrlmode_ext_fill_info(skb, priv)
            )

                return -EMSGSIZE;

        return 0;
}

static size_t can_get_xstats_size(const struct net_device *dev)
{
        return sizeof(struct can_device_stats);
}

static int can_fill_xstats(struct sk_buff *skb, const struct net_device *dev)
{
        struct can_priv *priv = netdev_priv(dev);

        if (nla_put(skb, IFLA_INFO_XSTATS,
                    sizeof(priv->can_stats), &priv->can_stats))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int can_newlink(struct net *src_net, struct net_device *dev,
                       struct nlattr *tb[], struct nlattr *data[],
                       struct netlink_ext_ack *extack)
{
        return -EOPNOTSUPP;
}

static void can_dellink(struct net_device *dev, struct list_head *head)
{
}

struct rtnl_link_ops can_link_ops __read_mostly = {
        .kind                = "can",
        .netns_refund        = true,
        .maxtype        = IFLA_CAN_MAX,
        .policy                = can_policy,
        .setup                = can_setup,
        .validate        = can_validate,
        .newlink        = can_newlink,
        .changelink        = can_changelink,
        .dellink        = can_dellink,
        .get_size        = can_get_size,
        .fill_info        = can_fill_info,
        .get_xstats_size = can_get_xstats_size,
        .fill_xstats        = can_fill_xstats,
};

int can_netlink_register(void)
{
        return rtnl_link_register(&can_link_ops);
}

void can_netlink_unregister(void)
{
        rtnl_link_unregister(&can_link_ops);
}





































   12 



    2 

   10 



























   35 




    1 


    3 





   30 
    1 


















    1 





    2 

   25 


    1 

    2 


















































   40 



    1 

   39 

















   47 


    2 


   45 













































































































  130 
  130 
   39 

    3 










  130 



  130 
  130 








    3 



    3 
    3 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
// SPDX-License-Identifier: GPL-2.0-or-later
//
// Validation of USB-audio class descriptors
//

#include <linux/init.h>
#include <linux/usb.h>
#include <linux/usb/audio.h>
#include <linux/usb/audio-v2.h>
#include <linux/usb/audio-v3.h>
#include <linux/usb/midi.h>
#include "usbaudio.h"
#include "helper.h"

struct usb_desc_validator {
        unsigned char protocol;
        unsigned char type;
        bool (*func)(const void *p, const struct usb_desc_validator *v);
        size_t size;
};

#define UAC_VERSION_ALL                (unsigned char)(-1)

/* UAC1 only */
static bool validate_uac1_header(const void *p,
                                 const struct usb_desc_validator *v)
{
        const struct uac1_ac_header_descriptor *d = p;

        return d->bLength >= sizeof(*d) &&
                d->bLength >= sizeof(*d) + d->bInCollection;
}

/* for mixer unit; covering all UACs */
static bool validate_mixer_unit(const void *p,
                                const struct usb_desc_validator *v)
{
        const struct uac_mixer_unit_descriptor *d = p;
        size_t len;

        if (d->bLength < sizeof(*d) || !d->bNrInPins)
                return false;
        len = sizeof(*d) + d->bNrInPins;
        /* We can't determine the bitmap size only from this unit descriptor,
         * so just check with the remaining length.
         * The actual bitmap is checked at mixer unit parser.
         */
        switch (v->protocol) {
        case UAC_VERSION_1:
        default:
                len += 2 + 1; /* wChannelConfig, iChannelNames */
                /* bmControls[n*m] */
                len += 1; /* iMixer */
                break;
        case UAC_VERSION_2:
                len += 4 + 1; /* bmChannelConfig, iChannelNames */
                /* bmMixerControls[n*m] */
                len += 1 + 1; /* bmControls, iMixer */
                break;
        case UAC_VERSION_3:
                len += 2; /* wClusterDescrID */
                /* bmMixerControls[n*m] */
                break;
        }
        return d->bLength >= len;
}

/* both for processing and extension units; covering all UACs */
static bool validate_processing_unit(const void *p,
                                     const struct usb_desc_validator *v)
{
        const struct uac_processing_unit_descriptor *d = p;
        const unsigned char *hdr = p;
        size_t len, m;

        if (d->bLength < sizeof(*d))
                return false;
        len = sizeof(*d) + d->bNrInPins;
        if (d->bLength < len)
                return false;
        switch (v->protocol) {
        case UAC_VERSION_1:
        default:
                /* bNrChannels, wChannelConfig, iChannelNames */
                len += 1 + 2 + 1;
                if (d->bLength < len + 1) /* bControlSize */
                        return false;
                m = hdr[len];
                len += 1 + m + 1; /* bControlSize, bmControls, iProcessing */
                break;
        case UAC_VERSION_2:
                /* bNrChannels, bmChannelConfig, iChannelNames */
                len += 1 + 4 + 1;
                if (v->type == UAC2_PROCESSING_UNIT_V2)
                        len += 2; /* bmControls -- 2 bytes for PU */
                else
                        len += 1; /* bmControls -- 1 byte for EU */
                len += 1; /* iProcessing */
                break;
        case UAC_VERSION_3:
                /* wProcessingDescrStr, bmControls */
                len += 2 + 4;
                break;
        }
        if (d->bLength < len)
                return false;

        switch (v->protocol) {
        case UAC_VERSION_1:
        default:
                if (v->type == UAC1_EXTENSION_UNIT)
                        return true; /* OK */
                switch (le16_to_cpu(d->wProcessType)) {
                case UAC_PROCESS_UP_DOWNMIX:
                case UAC_PROCESS_DOLBY_PROLOGIC:
                        if (d->bLength < len + 1) /* bNrModes */
                                return false;
                        m = hdr[len];
                        len += 1 + m * 2; /* bNrModes, waModes(n) */
                        break;
                default:
                        break;
                }
                break;
        case UAC_VERSION_2:
                if (v->type == UAC2_EXTENSION_UNIT_V2)
                        return true; /* OK */
                switch (le16_to_cpu(d->wProcessType)) {
                case UAC2_PROCESS_UP_DOWNMIX:
                case UAC2_PROCESS_DOLBY_PROLOCIC: /* SiC! */
                        if (d->bLength < len + 1) /* bNrModes */
                                return false;
                        m = hdr[len];
                        len += 1 + m * 4; /* bNrModes, daModes(n) */
                        break;
                default:
                        break;
                }
                break;
        case UAC_VERSION_3:
                if (v->type == UAC3_EXTENSION_UNIT) {
                        len += 2; /* wClusterDescrID */
                        break;
                }
                switch (le16_to_cpu(d->wProcessType)) {
                case UAC3_PROCESS_UP_DOWNMIX:
                        if (d->bLength < len + 1) /* bNrModes */
                                return false;
                        m = hdr[len];
                        len += 1 + m * 2; /* bNrModes, waClusterDescrID(n) */
                        break;
                case UAC3_PROCESS_MULTI_FUNCTION:
                        len += 2 + 4; /* wClusterDescrID, bmAlgorighms */
                        break;
                default:
                        break;
                }
                break;
        }
        if (d->bLength < len)
                return false;

        return true;
}

/* both for selector and clock selector units; covering all UACs */
static bool validate_selector_unit(const void *p,
                                   const struct usb_desc_validator *v)
{
        const struct uac_selector_unit_descriptor *d = p;
        size_t len;

        if (d->bLength < sizeof(*d))
                return false;
        len = sizeof(*d) + d->bNrInPins;
        switch (v->protocol) {
        case UAC_VERSION_1:
        default:
                len += 1; /* iSelector */
                break;
        case UAC_VERSION_2:
                len += 1 + 1; /* bmControls, iSelector */
                break;
        case UAC_VERSION_3:
                len += 4 + 2; /* bmControls, wSelectorDescrStr */
                break;
        }
        return d->bLength >= len;
}

static bool validate_uac1_feature_unit(const void *p,
                                       const struct usb_desc_validator *v)
{
        const struct uac_feature_unit_descriptor *d = p;

        if (d->bLength < sizeof(*d) || !d->bControlSize)
                return false;
        /* at least bmaControls(0) for master channel + iFeature */
        return d->bLength >= sizeof(*d) + d->bControlSize + 1;
}

static bool validate_uac2_feature_unit(const void *p,
                                       const struct usb_desc_validator *v)
{
        const struct uac2_feature_unit_descriptor *d = p;

        if (d->bLength < sizeof(*d))
                return false;
        /* at least bmaControls(0) for master channel + iFeature */
        return d->bLength >= sizeof(*d) + 4 + 1;
}

static bool validate_uac3_feature_unit(const void *p,
                                       const struct usb_desc_validator *v)
{
        const struct uac3_feature_unit_descriptor *d = p;

        if (d->bLength < sizeof(*d))
                return false;
        /* at least bmaControls(0) for master channel + wFeatureDescrStr */
        return d->bLength >= sizeof(*d) + 4 + 2;
}

static bool validate_midi_out_jack(const void *p,
                                   const struct usb_desc_validator *v)
{
        const struct usb_midi_out_jack_descriptor *d = p;

        return d->bLength >= sizeof(*d) &&
                d->bLength >= sizeof(*d) + d->bNrInputPins * 2;
}

#define FIXED(p, t, s) { .protocol = (p), .type = (t), .size = sizeof(s) }
#define FUNC(p, t, f) { .protocol = (p), .type = (t), .func = (f) }

static const struct usb_desc_validator audio_validators[] = {
        /* UAC1 */
        FUNC(UAC_VERSION_1, UAC_HEADER, validate_uac1_header),
        FIXED(UAC_VERSION_1, UAC_INPUT_TERMINAL,
              struct uac_input_terminal_descriptor),
        FIXED(UAC_VERSION_1, UAC_OUTPUT_TERMINAL,
              struct uac1_output_terminal_descriptor),
        FUNC(UAC_VERSION_1, UAC_MIXER_UNIT, validate_mixer_unit),
        FUNC(UAC_VERSION_1, UAC_SELECTOR_UNIT, validate_selector_unit),
        FUNC(UAC_VERSION_1, UAC_FEATURE_UNIT, validate_uac1_feature_unit),
        FUNC(UAC_VERSION_1, UAC1_PROCESSING_UNIT, validate_processing_unit),
        FUNC(UAC_VERSION_1, UAC1_EXTENSION_UNIT, validate_processing_unit),

        /* UAC2 */
        FIXED(UAC_VERSION_2, UAC_HEADER, struct uac2_ac_header_descriptor),
        FIXED(UAC_VERSION_2, UAC_INPUT_TERMINAL,
              struct uac2_input_terminal_descriptor),
        FIXED(UAC_VERSION_2, UAC_OUTPUT_TERMINAL,
              struct uac2_output_terminal_descriptor),
        FUNC(UAC_VERSION_2, UAC_MIXER_UNIT, validate_mixer_unit),
        FUNC(UAC_VERSION_2, UAC_SELECTOR_UNIT, validate_selector_unit),
        FUNC(UAC_VERSION_2, UAC_FEATURE_UNIT, validate_uac2_feature_unit),
        /* UAC_VERSION_2, UAC2_EFFECT_UNIT: not implemented yet */
        FUNC(UAC_VERSION_2, UAC2_PROCESSING_UNIT_V2, validate_processing_unit),
        FUNC(UAC_VERSION_2, UAC2_EXTENSION_UNIT_V2, validate_processing_unit),
        FIXED(UAC_VERSION_2, UAC2_CLOCK_SOURCE,
              struct uac_clock_source_descriptor),
        FUNC(UAC_VERSION_2, UAC2_CLOCK_SELECTOR, validate_selector_unit),
        FIXED(UAC_VERSION_2, UAC2_CLOCK_MULTIPLIER,
              struct uac_clock_multiplier_descriptor),
        /* UAC_VERSION_2, UAC2_SAMPLE_RATE_CONVERTER: not implemented yet */

        /* UAC3 */
        FIXED(UAC_VERSION_2, UAC_HEADER, struct uac3_ac_header_descriptor),
        FIXED(UAC_VERSION_3, UAC_INPUT_TERMINAL,
              struct uac3_input_terminal_descriptor),
        FIXED(UAC_VERSION_3, UAC_OUTPUT_TERMINAL,
              struct uac3_output_terminal_descriptor),
        /* UAC_VERSION_3, UAC3_EXTENDED_TERMINAL: not implemented yet */
        FUNC(UAC_VERSION_3, UAC3_MIXER_UNIT, validate_mixer_unit),
        FUNC(UAC_VERSION_3, UAC3_SELECTOR_UNIT, validate_selector_unit),
        FUNC(UAC_VERSION_3, UAC_FEATURE_UNIT, validate_uac3_feature_unit),
        /*  UAC_VERSION_3, UAC3_EFFECT_UNIT: not implemented yet */
        FUNC(UAC_VERSION_3, UAC3_PROCESSING_UNIT, validate_processing_unit),
        FUNC(UAC_VERSION_3, UAC3_EXTENSION_UNIT, validate_processing_unit),
        FIXED(UAC_VERSION_3, UAC3_CLOCK_SOURCE,
              struct uac3_clock_source_descriptor),
        FUNC(UAC_VERSION_3, UAC3_CLOCK_SELECTOR, validate_selector_unit),
        FIXED(UAC_VERSION_3, UAC3_CLOCK_MULTIPLIER,
              struct uac3_clock_multiplier_descriptor),
        /* UAC_VERSION_3, UAC3_SAMPLE_RATE_CONVERTER: not implemented yet */
        /* UAC_VERSION_3, UAC3_CONNECTORS: not implemented yet */
        { } /* terminator */
};

static const struct usb_desc_validator midi_validators[] = {
        FIXED(UAC_VERSION_ALL, USB_MS_HEADER,
              struct usb_ms_header_descriptor),
        FIXED(UAC_VERSION_ALL, USB_MS_MIDI_IN_JACK,
              struct usb_midi_in_jack_descriptor),
        FUNC(UAC_VERSION_ALL, USB_MS_MIDI_OUT_JACK,
             validate_midi_out_jack),
        { } /* terminator */
};


/* Validate the given unit descriptor, return true if it's OK */
static bool validate_desc(unsigned char *hdr, int protocol,
                          const struct usb_desc_validator *v)
{
        if (hdr[1] != USB_DT_CS_INTERFACE)
                return true; /* don't care */

        for (; v->type; v++) {
                if (v->type == hdr[2] &&
                    (v->protocol == UAC_VERSION_ALL ||
                     v->protocol == protocol)) {
                        if (v->func)
                                return v->func(hdr, v);
                        /* check for the fixed size */
                        return hdr[0] >= v->size;
                }
        }

        return true; /* not matching, skip validation */
}

bool snd_usb_validate_audio_desc(void *p, int protocol)
{
        unsigned char *c = p;
        bool valid;

        valid = validate_desc(p, protocol, audio_validators);
        if (!valid && snd_usb_skip_validation) {
                print_hex_dump(KERN_ERR, "USB-audio: buggy audio desc: ",
                               DUMP_PREFIX_NONE, 16, 1, c, c[0], true);
                valid = true;
        }
        return valid;
}

bool snd_usb_validate_midi_desc(void *p)
{
        unsigned char *c = p;
        bool valid;

        valid = validate_desc(p, UAC_VERSION_1, midi_validators);
        if (!valid && snd_usb_skip_validation) {
                print_hex_dump(KERN_ERR, "USB-audio: buggy midi desc: ",
                               DUMP_PREFIX_NONE, 16, 1, c, c[0], true);
                valid = true;
        }
        return valid;
}
































































































































































































































  688 





































































   17 























































    8 
  144 




   13 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * NET                Generic infrastructure for INET connection oriented protocols.
 *
 *                Definitions for inet_connection_sock 
 *
 * Authors:        Many people, see the TCP sources
 *
 *                 From code originally in TCP
 */
#ifndef _INET_CONNECTION_SOCK_H
#define _INET_CONNECTION_SOCK_H

#include <linux/compiler.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/poll.h>
#include <linux/kernel.h>
#include <linux/sockptr.h>

#include <net/inet_sock.h>
#include <net/request_sock.h>

/* Cancel timers, when they are not required. */
#undef INET_CSK_CLEAR_TIMERS

struct inet_bind_bucket;
struct inet_bind2_bucket;
struct tcp_congestion_ops;

/*
 * Pointers to address related TCP functions
 * (i.e. things that depend on the address family)
 */
struct inet_connection_sock_af_ops {
        int            (*queue_xmit)(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
        void            (*send_check)(struct sock *sk, struct sk_buff *skb);
        int            (*rebuild_header)(struct sock *sk);
        void            (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb);
        int            (*conn_request)(struct sock *sk, struct sk_buff *skb);
        struct sock *(*syn_recv_sock)(const struct sock *sk, struct sk_buff *skb,
                                      struct request_sock *req,
                                      struct dst_entry *dst,
                                      struct request_sock *req_unhash,
                                      bool *own_req);
        u16            net_header_len;
        u16            sockaddr_len;
        int            (*setsockopt)(struct sock *sk, int level, int optname,
                                  sockptr_t optval, unsigned int optlen);
        int            (*getsockopt)(struct sock *sk, int level, int optname,
                                  char __user *optval, int __user *optlen);
        void            (*addr2sockaddr)(struct sock *sk, struct sockaddr *);
        void            (*mtu_reduced)(struct sock *sk);
};

/** inet_connection_sock - INET connection oriented sock
 *
 * @icsk_accept_queue:           FIFO of established children
 * @icsk_bind_hash:           Bind node
 * @icsk_bind2_hash:           Bind node in the bhash2 table
 * @icsk_timeout:           Timeout
 * @icsk_retransmit_timer: Resend (no ack)
 * @icsk_rto:                   Retransmit timeout
 * @icsk_pmtu_cookie           Last pmtu seen by socket
 * @icsk_ca_ops                   Pluggable congestion control hook
 * @icsk_af_ops                   Operations which are AF_INET{4,6} specific
 * @icsk_ulp_ops           Pluggable ULP control hook
 * @icsk_ulp_data           ULP private data
 * @icsk_clean_acked           Clean acked data hook
 * @icsk_ca_state:           Congestion control state
 * @icsk_retransmits:           Number of unrecovered [RTO] timeouts
 * @icsk_pending:           Scheduled timer event
 * @icsk_backoff:           Backoff
 * @icsk_syn_retries:      Number of allowed SYN (or equivalent) retries
 * @icsk_probes_out:           unanswered 0 window probes
 * @icsk_ext_hdr_len:           Network protocol overhead (IP/IPv6 options)
 * @icsk_ack:                   Delayed ACK control data
 * @icsk_mtup;                   MTU probing control data
 * @icsk_probes_tstamp:    Probe timestamp (cleared by non-zero window ack)
 * @icsk_user_timeout:           TCP_USER_TIMEOUT value
 */
struct inet_connection_sock {
        /* inet_sock has to be the first member! */
        struct inet_sock          icsk_inet;
        struct request_sock_queue icsk_accept_queue;
        struct inet_bind_bucket          *icsk_bind_hash;
        struct inet_bind2_bucket  *icsk_bind2_hash;
        unsigned long                  icsk_timeout;
         struct timer_list          icsk_retransmit_timer;
         struct timer_list          icsk_delack_timer;
        __u32                          icsk_rto;
        __u32                     icsk_rto_min;
        __u32                     icsk_delack_max;
        __u32                          icsk_pmtu_cookie;
        const struct tcp_congestion_ops *icsk_ca_ops;
        const struct inet_connection_sock_af_ops *icsk_af_ops;
        const struct tcp_ulp_ops  *icsk_ulp_ops;
        void __rcu                  *icsk_ulp_data;
        void (*icsk_clean_acked)(struct sock *sk, u32 acked_seq);
        unsigned int                  (*icsk_sync_mss)(struct sock *sk, u32 pmtu);
        __u8                          icsk_ca_state:5,
                                  icsk_ca_initialized:1,
                                  icsk_ca_setsockopt:1,
                                  icsk_ca_dst_locked:1;
        __u8                          icsk_retransmits;
        __u8                          icsk_pending;
        __u8                          icsk_backoff;
        __u8                          icsk_syn_retries;
        __u8                          icsk_probes_out;
        __u16                          icsk_ext_hdr_len;
        struct {
                __u8                  pending;         /* ACK is pending                           */
                __u8                  quick;         /* Scheduled number of quick acks           */
                __u8                  pingpong;         /* The session is interactive                   */
                __u8                  retry;         /* Number of attempts                           */
                #define ATO_BITS 8
                __u32                  ato:ATO_BITS,         /* Predicted tick of soft clock           */
                                  lrcv_flowlabel:20, /* last received ipv6 flowlabel           */
                                  unused:4;
                unsigned long          timeout;         /* Currently scheduled timeout                   */
                __u32                  lrcvtime;         /* timestamp of last received data packet */
                __u16                  last_seg_size; /* Size of last incoming segment           */
                __u16                  rcv_mss;         /* MSS used for delayed ACK decisions           */
        } icsk_ack;
        struct {
                /* Range of MTUs to search */
                int                  search_high;
                int                  search_low;

                /* Information on the current probe. */
                u32                  probe_size:31,
                /* Is the MTUP feature enabled for this connection? */
                                  enabled:1;

                u32                  probe_timestamp;
        } icsk_mtup;
        u32                          icsk_probes_tstamp;
        u32                          icsk_user_timeout;

        u64                          icsk_ca_priv[104 / sizeof(u64)];
#define ICSK_CA_PRIV_SIZE          sizeof_field(struct inet_connection_sock, icsk_ca_priv)
};

#define ICSK_TIME_RETRANS        1        /* Retransmit timer */
#define ICSK_TIME_DACK                2        /* Delayed ack timer */
#define ICSK_TIME_PROBE0        3        /* Zero window probe timer */
#define ICSK_TIME_LOSS_PROBE        5        /* Tail loss probe timer */
#define ICSK_TIME_REO_TIMEOUT        6        /* Reordering timer */

#define inet_csk(ptr) container_of_const(ptr, struct inet_connection_sock, icsk_inet.sk)

static inline void *inet_csk_ca(const struct sock *sk)
{
        return (void *)inet_csk(sk)->icsk_ca_priv;
}

struct sock *inet_csk_clone_lock(const struct sock *sk,
                                 const struct request_sock *req,
                                 const gfp_t priority);

enum inet_csk_ack_state_t {
        ICSK_ACK_SCHED        = 1,
        ICSK_ACK_TIMER  = 2,
        ICSK_ACK_PUSHED = 4,
        ICSK_ACK_PUSHED2 = 8,
        ICSK_ACK_NOW = 16,        /* Send the next ACK immediately (once) */
        ICSK_ACK_NOMEM = 32,
};

void inet_csk_init_xmit_timers(struct sock *sk,
                               void (*retransmit_handler)(struct timer_list *),
                               void (*delack_handler)(struct timer_list *),
                               void (*keepalive_handler)(struct timer_list *));
void inet_csk_clear_xmit_timers(struct sock *sk);
void inet_csk_clear_xmit_timers_sync(struct sock *sk);

static inline void inet_csk_schedule_ack(struct sock *sk)
{
        inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_SCHED;
}

static inline int inet_csk_ack_scheduled(const struct sock *sk)
{
        return inet_csk(sk)->icsk_ack.pending & ICSK_ACK_SCHED;
}

static inline void inet_csk_delack_init(struct sock *sk)
{
        memset(&inet_csk(sk)->icsk_ack, 0, sizeof(inet_csk(sk)->icsk_ack));
}

void inet_csk_delete_keepalive_timer(struct sock *sk);
void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long timeout);

static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0) {
                smp_store_release(&icsk->icsk_pending, 0);
#ifdef INET_CSK_CLEAR_TIMERS
                sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
#endif
        } else if (what == ICSK_TIME_DACK) {
                smp_store_release(&icsk->icsk_ack.pending, 0);
                icsk->icsk_ack.retry = 0;
#ifdef INET_CSK_CLEAR_TIMERS
                sk_stop_timer(sk, &icsk->icsk_delack_timer);
#endif
        } else {
                pr_debug("inet_csk BUG: unknown timer value\n");
        }
}

/*
 *        Reset the retransmission timer
 */
static inline void inet_csk_reset_xmit_timer(struct sock *sk, const int what,
                                             unsigned long when,
                                             const unsigned long max_when)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        if (when > max_when) {
                pr_debug("reset_xmit_timer: sk=%p %d when=0x%lx, caller=%p\n",
                         sk, what, when, (void *)_THIS_IP_);
                when = max_when;
        }

        if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0 ||
            what == ICSK_TIME_LOSS_PROBE || what == ICSK_TIME_REO_TIMEOUT) {
                smp_store_release(&icsk->icsk_pending, what);
                icsk->icsk_timeout = jiffies + when;
                sk_reset_timer(sk, &icsk->icsk_retransmit_timer, icsk->icsk_timeout);
        } else if (what == ICSK_TIME_DACK) {
                smp_store_release(&icsk->icsk_ack.pending,
                                  icsk->icsk_ack.pending | ICSK_ACK_TIMER);
                icsk->icsk_ack.timeout = jiffies + when;
                sk_reset_timer(sk, &icsk->icsk_delack_timer, icsk->icsk_ack.timeout);
        } else {
                pr_debug("inet_csk BUG: unknown timer value\n");
        }
}

static inline unsigned long
inet_csk_rto_backoff(const struct inet_connection_sock *icsk,
                     unsigned long max_when)
{
        u64 when = (u64)icsk->icsk_rto << icsk->icsk_backoff;

        return (unsigned long)min_t(u64, when, max_when);
}

struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg);

int inet_csk_get_port(struct sock *sk, unsigned short snum);

struct dst_entry *inet_csk_route_req(const struct sock *sk, struct flowi4 *fl4,
                                     const struct request_sock *req);
struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
                                            struct sock *newsk,
                                            const struct request_sock *req);

struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
                                      struct request_sock *req,
                                      struct sock *child);
bool inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
                                   unsigned long timeout);
struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
                                         struct request_sock *req,
                                         bool own_req);

static inline void inet_csk_reqsk_queue_added(struct sock *sk)
{
        reqsk_queue_added(&inet_csk(sk)->icsk_accept_queue);
}

static inline int inet_csk_reqsk_queue_len(const struct sock *sk)
{
        return reqsk_queue_len(&inet_csk(sk)->icsk_accept_queue);
}

static inline int inet_csk_reqsk_queue_is_full(const struct sock *sk)
{
        return inet_csk_reqsk_queue_len(sk) > READ_ONCE(sk->sk_max_ack_backlog);
}

bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req);
void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req);

static inline unsigned long
reqsk_timeout(struct request_sock *req, unsigned long max_timeout)
{
        u64 timeout = (u64)req->timeout << req->num_timeout;

        return (unsigned long)min_t(u64, timeout, max_timeout);
}

static inline void inet_csk_prepare_for_destroy_sock(struct sock *sk)
{
        /* The below has to be done to allow calling inet_csk_destroy_sock */
        sock_set_flag(sk, SOCK_DEAD);
        this_cpu_inc(*sk->sk_prot->orphan_count);
}

void inet_csk_destroy_sock(struct sock *sk);
void inet_csk_prepare_forced_close(struct sock *sk);

/*
 * LISTEN is a special case for poll..
 */
static inline __poll_t inet_csk_listen_poll(const struct sock *sk)
{
        return !reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue) ?
                        (EPOLLIN | EPOLLRDNORM) : 0;
}

int inet_csk_listen_start(struct sock *sk);
void inet_csk_listen_stop(struct sock *sk);

void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr);

/* update the fast reuse flag when adding a socket */
void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
                               struct sock *sk);

struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu);

static inline void inet_csk_enter_pingpong_mode(struct sock *sk)
{
        inet_csk(sk)->icsk_ack.pingpong =
                READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pingpong_thresh);
}

static inline void inet_csk_exit_pingpong_mode(struct sock *sk)
{
        inet_csk(sk)->icsk_ack.pingpong = 0;
}

static inline bool inet_csk_in_pingpong_mode(struct sock *sk)
{
        return inet_csk(sk)->icsk_ack.pingpong >=
               READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_pingpong_thresh);
}

static inline void inet_csk_inc_pingpong_cnt(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        if (icsk->icsk_ack.pingpong < U8_MAX)
                icsk->icsk_ack.pingpong++;
}

static inline bool inet_csk_has_ulp(const struct sock *sk)
{
        return inet_test_bit(IS_ICSK, sk) && !!inet_csk(sk)->icsk_ulp_ops;
}

static inline void inet_init_csk_locks(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        spin_lock_init(&icsk->icsk_accept_queue.rskq_lock);
        spin_lock_init(&icsk->icsk_accept_queue.fastopenq.lock);
}

#endif /* _INET_CONNECTION_SOCK_H */






























































 9094 












 8242 
































 4147 

















   23 














    6 














   15 


































  783 















  328 







   11 
  451 













































  532 














  185 




































 3267 


 3266 

















   28 


   28 
























   47 



   47 



   47 




   47 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
// SPDX-License-Identifier: GPL-2.0
/*
 *  Kernel internal schedule timeout and sleeping functions
 */

#include <linux/delay.h>
#include <linux/jiffies.h>
#include <linux/timer.h>
#include <linux/sched/signal.h>
#include <linux/sched/debug.h>

#include "tick-internal.h"

/*
 * Since schedule_timeout()'s timer is defined on the stack, it must store
 * the target task on the stack as well.
 */
struct process_timer {
        struct timer_list timer;
        struct task_struct *task;
};

static void process_timeout(struct timer_list *t)
{
        struct process_timer *timeout = from_timer(timeout, t, timer);

        wake_up_process(timeout->task);
}

/**
 * schedule_timeout - sleep until timeout
 * @timeout: timeout value in jiffies
 *
 * Make the current task sleep until @timeout jiffies have elapsed.
 * The function behavior depends on the current task state
 * (see also set_current_state() description):
 *
 * %TASK_RUNNING - the scheduler is called, but the task does not sleep
 * at all. That happens because sched_submit_work() does nothing for
 * tasks in %TASK_RUNNING state.
 *
 * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
 * pass before the routine returns unless the current task is explicitly
 * woken up, (e.g. by wake_up_process()).
 *
 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
 * delivered to the current task or the current task is explicitly woken
 * up.
 *
 * The current task state is guaranteed to be %TASK_RUNNING when this
 * routine returns.
 *
 * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
 * the CPU away without a bound on the timeout. In this case the return
 * value will be %MAX_SCHEDULE_TIMEOUT.
 *
 * Returns: 0 when the timer has expired otherwise the remaining time in
 * jiffies will be returned. In all cases the return value is guaranteed
 * to be non-negative.
 */
signed long __sched schedule_timeout(signed long timeout)
{
        struct process_timer timer;
        unsigned long expire;

        switch (timeout) {
        case MAX_SCHEDULE_TIMEOUT:
                /*
                 * These two special cases are useful to be comfortable
                 * in the caller. Nothing more. We could take
                 * MAX_SCHEDULE_TIMEOUT from one of the negative value
                 * but I' d like to return a valid offset (>=0) to allow
                 * the caller to do everything it want with the retval.
                 */
                schedule();
                goto out;
        default:
                /*
                 * Another bit of PARANOID. Note that the retval will be
                 * 0 since no piece of kernel is supposed to do a check
                 * for a negative retval of schedule_timeout() (since it
                 * should never happens anyway). You just have the printk()
                 * that will tell you if something is gone wrong and where.
                 */
                if (timeout < 0) {
                        pr_err("%s: wrong timeout value %lx\n", __func__, timeout);
                        dump_stack();
                        __set_current_state(TASK_RUNNING);
                        goto out;
                }
        }

        expire = timeout + jiffies;

        timer.task = current;
        timer_setup_on_stack(&timer.timer, process_timeout, 0);
        timer.timer.expires = expire;
        add_timer(&timer.timer);
        schedule();
        del_timer_sync(&timer.timer);

        /* Remove the timer from the object tracker */
        destroy_timer_on_stack(&timer.timer);

        timeout = expire - jiffies;

 out:
        return timeout < 0 ? 0 : timeout;
}
EXPORT_SYMBOL(schedule_timeout);

/*
 * __set_current_state() can be used in schedule_timeout_*() functions, because
 * schedule_timeout() calls schedule() unconditionally.
 */

/**
 * schedule_timeout_interruptible - sleep until timeout (interruptible)
 * @timeout: timeout value in jiffies
 *
 * See schedule_timeout() for details.
 *
 * Task state is set to TASK_INTERRUPTIBLE before starting the timeout.
 */
signed long __sched schedule_timeout_interruptible(signed long timeout)
{
        __set_current_state(TASK_INTERRUPTIBLE);
        return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_interruptible);

/**
 * schedule_timeout_killable - sleep until timeout (killable)
 * @timeout: timeout value in jiffies
 *
 * See schedule_timeout() for details.
 *
 * Task state is set to TASK_KILLABLE before starting the timeout.
 */
signed long __sched schedule_timeout_killable(signed long timeout)
{
        __set_current_state(TASK_KILLABLE);
        return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_killable);

/**
 * schedule_timeout_uninterruptible - sleep until timeout (uninterruptible)
 * @timeout: timeout value in jiffies
 *
 * See schedule_timeout() for details.
 *
 * Task state is set to TASK_UNINTERRUPTIBLE before starting the timeout.
 */
signed long __sched schedule_timeout_uninterruptible(signed long timeout)
{
        __set_current_state(TASK_UNINTERRUPTIBLE);
        return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_uninterruptible);

/**
 * schedule_timeout_idle - sleep until timeout (idle)
 * @timeout: timeout value in jiffies
 *
 * See schedule_timeout() for details.
 *
 * Task state is set to TASK_IDLE before starting the timeout. It is similar to
 * schedule_timeout_uninterruptible(), except this task will not contribute to
 * load average.
 */
signed long __sched schedule_timeout_idle(signed long timeout)
{
        __set_current_state(TASK_IDLE);
        return schedule_timeout(timeout);
}
EXPORT_SYMBOL(schedule_timeout_idle);

/**
 * schedule_hrtimeout_range_clock - sleep until timeout
 * @expires:        timeout value (ktime_t)
 * @delta:        slack in expires timeout (ktime_t)
 * @mode:        timer mode
 * @clock_id:        timer clock to be used
 *
 * Details are explained in schedule_hrtimeout_range() function description as
 * this function is commonly used.
 */
int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
                                           const enum hrtimer_mode mode, clockid_t clock_id)
{
        struct hrtimer_sleeper t;

        /*
         * Optimize when a zero timeout value is given. It does not
         * matter whether this is an absolute or a relative time.
         */
        if (expires && *expires == 0) {
                __set_current_state(TASK_RUNNING);
                return 0;
        }

        /*
         * A NULL parameter means "infinite"
         */
        if (!expires) {
                schedule();
                return -EINTR;
        }

        hrtimer_setup_sleeper_on_stack(&t, clock_id, mode);
        hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
        hrtimer_sleeper_start_expires(&t, mode);

        if (likely(t.task))
                schedule();

        hrtimer_cancel(&t.timer);
        destroy_hrtimer_on_stack(&t.timer);

        __set_current_state(TASK_RUNNING);

        return !t.task ? 0 : -EINTR;
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);

/**
 * schedule_hrtimeout_range - sleep until timeout
 * @expires:        timeout value (ktime_t)
 * @delta:        slack in expires timeout (ktime_t)
 * @mode:        timer mode
 *
 * Make the current task sleep until the given expiry time has
 * elapsed. The routine will return immediately unless
 * the current task state has been set (see set_current_state()).
 *
 * The @delta argument gives the kernel the freedom to schedule the
 * actual wakeup to a time that is both power and performance friendly
 * for regular (non RT/DL) tasks.
 * The kernel give the normal best effort behavior for "@expires+@delta",
 * but may decide to fire the timer earlier, but no earlier than @expires.
 *
 * You can set the task state as follows -
 *
 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
 * pass before the routine returns unless the current task is explicitly
 * woken up, (e.g. by wake_up_process()).
 *
 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
 * delivered to the current task or the current task is explicitly woken
 * up.
 *
 * The current task state is guaranteed to be TASK_RUNNING when this
 * routine returns.
 *
 * Returns: 0 when the timer has expired. If the task was woken before the
 * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
 * by an explicit wakeup, it returns -EINTR.
 */
int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
                                     const enum hrtimer_mode mode)
{
        return schedule_hrtimeout_range_clock(expires, delta, mode,
                                              CLOCK_MONOTONIC);
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);

/**
 * schedule_hrtimeout - sleep until timeout
 * @expires:        timeout value (ktime_t)
 * @mode:        timer mode
 *
 * See schedule_hrtimeout_range() for details. @delta argument of
 * schedule_hrtimeout_range() is set to 0 and has therefore no impact.
 */
int __sched schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode)
{
        return schedule_hrtimeout_range(expires, 0, mode);
}
EXPORT_SYMBOL_GPL(schedule_hrtimeout);

/**
 * msleep - sleep safely even with waitqueue interruptions
 * @msecs:        Requested sleep duration in milliseconds
 *
 * msleep() uses jiffy based timeouts for the sleep duration. Because of the
 * design of the timer wheel, the maximum additional percentage delay (slack) is
 * 12.5%. This is only valid for timers which will end up in level 1 or a higher
 * level of the timer wheel. For explanation of those 12.5% please check the
 * detailed description about the basics of the timer wheel.
 *
 * The slack of timers which will end up in level 0 depends on sleep duration
 * (msecs) and HZ configuration and can be calculated in the following way (with
 * the timer wheel design restriction that the slack is not less than 12.5%):
 *
 *   ``slack = MSECS_PER_TICK / msecs``
 *
 * When the allowed slack of the callsite is known, the calculation could be
 * turned around to find the minimal allowed sleep duration to meet the
 * constraints. For example:
 *
 * * ``HZ=1000`` with ``slack=25%``: ``MSECS_PER_TICK / slack = 1 / (1/4) = 4``:
 *   all sleep durations greater or equal 4ms will meet the constraints.
 * * ``HZ=1000`` with ``slack=12.5%``: ``MSECS_PER_TICK / slack = 1 / (1/8) = 8``:
 *   all sleep durations greater or equal 8ms will meet the constraints.
 * * ``HZ=250`` with ``slack=25%``: ``MSECS_PER_TICK / slack = 4 / (1/4) = 16``:
 *   all sleep durations greater or equal 16ms will meet the constraints.
 * * ``HZ=250`` with ``slack=12.5%``: ``MSECS_PER_TICK / slack = 4 / (1/8) = 32``:
 *   all sleep durations greater or equal 32ms will meet the constraints.
 *
 * See also the signal aware variant msleep_interruptible().
 */
void msleep(unsigned int msecs)
{
        unsigned long timeout = msecs_to_jiffies(msecs);

        while (timeout)
                timeout = schedule_timeout_uninterruptible(timeout);
}
EXPORT_SYMBOL(msleep);

/**
 * msleep_interruptible - sleep waiting for signals
 * @msecs:        Requested sleep duration in milliseconds
 *
 * See msleep() for some basic information.
 *
 * The difference between msleep() and msleep_interruptible() is that the sleep
 * could be interrupted by a signal delivery and then returns early.
 *
 * Returns: The remaining time of the sleep duration transformed to msecs (see
 * schedule_timeout() for details).
 */
unsigned long msleep_interruptible(unsigned int msecs)
{
        unsigned long timeout = msecs_to_jiffies(msecs);

        while (timeout && !signal_pending(current))
                timeout = schedule_timeout_interruptible(timeout);
        return jiffies_to_msecs(timeout);
}
EXPORT_SYMBOL(msleep_interruptible);

/**
 * usleep_range_state - Sleep for an approximate time in a given state
 * @min:        Minimum time in usecs to sleep
 * @max:        Maximum time in usecs to sleep
 * @state:        State of the current task that will be while sleeping
 *
 * usleep_range_state() sleeps at least for the minimum specified time but not
 * longer than the maximum specified amount of time. The range might reduce
 * power usage by allowing hrtimers to coalesce an already scheduled interrupt
 * with this hrtimer. In the worst case, an interrupt is scheduled for the upper
 * bound.
 *
 * The sleeping task is set to the specified state before starting the sleep.
 *
 * In non-atomic context where the exact wakeup time is flexible, use
 * usleep_range() or its variants instead of udelay(). The sleep improves
 * responsiveness by avoiding the CPU-hogging busy-wait of udelay().
 */
void __sched usleep_range_state(unsigned long min, unsigned long max, unsigned int state)
{
        ktime_t exp = ktime_add_us(ktime_get(), min);
        u64 delta = (u64)(max - min) * NSEC_PER_USEC;

        if (WARN_ON_ONCE(max < min))
                delta = 0;

        for (;;) {
                __set_current_state(state);
                /* Do not return before the requested sleep time has elapsed */
                if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
                        break;
        }
}
EXPORT_SYMBOL(usleep_range_state);























 2304 







  431 






  430 





  341 
   20 
  344 


   21 
    4 




















 2428 





 2172 




 2172 




 1407 
 1409 





  285 
  286 





  147 
  149 




 2282 
 2289 
 2287 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
// SPDX-License-Identifier: GPL-2.0-only
/*
 * mm/interval_tree.c - interval tree for mapping->i_mmap
 *
 * Copyright (C) 2012, Michel Lespinasse <walken@google.com>
 */

#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/rmap.h>
#include <linux/interval_tree_generic.h>

static inline unsigned long vma_start_pgoff(struct vm_area_struct *v)
{
        return v->vm_pgoff;
}

static inline unsigned long vma_last_pgoff(struct vm_area_struct *v)
{
        return v->vm_pgoff + vma_pages(v) - 1;
}

INTERVAL_TREE_DEFINE(struct vm_area_struct, shared.rb,
                     unsigned long, shared.rb_subtree_last,
                     vma_start_pgoff, vma_last_pgoff, /* empty */, vma_interval_tree)

/* Insert node immediately after prev in the interval tree */
void vma_interval_tree_insert_after(struct vm_area_struct *node,
                                    struct vm_area_struct *prev,
                                    struct rb_root_cached *root)
{
        struct rb_node **link;
        struct vm_area_struct *parent;
        unsigned long last = vma_last_pgoff(node);

        VM_BUG_ON_VMA(vma_start_pgoff(node) != vma_start_pgoff(prev), node);

        if (!prev->shared.rb.rb_right) {
                parent = prev;
                link = &prev->shared.rb.rb_right;
        } else {
                parent = rb_entry(prev->shared.rb.rb_right,
                                  struct vm_area_struct, shared.rb);
                if (parent->shared.rb_subtree_last < last)
                        parent->shared.rb_subtree_last = last;
                while (parent->shared.rb.rb_left) {
                        parent = rb_entry(parent->shared.rb.rb_left,
                                struct vm_area_struct, shared.rb);
                        if (parent->shared.rb_subtree_last < last)
                                parent->shared.rb_subtree_last = last;
                }
                link = &parent->shared.rb.rb_left;
        }

        node->shared.rb_subtree_last = last;
        rb_link_node(&node->shared.rb, &parent->shared.rb, link);
        rb_insert_augmented(&node->shared.rb, &root->rb_root,
                            &vma_interval_tree_augment);
}

static inline unsigned long avc_start_pgoff(struct anon_vma_chain *avc)
{
        return vma_start_pgoff(avc->vma);
}

static inline unsigned long avc_last_pgoff(struct anon_vma_chain *avc)
{
        return vma_last_pgoff(avc->vma);
}

INTERVAL_TREE_DEFINE(struct anon_vma_chain, rb, unsigned long, rb_subtree_last,
                     avc_start_pgoff, avc_last_pgoff,
                     static inline, __anon_vma_interval_tree)

void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
                                   struct rb_root_cached *root)
{
#ifdef CONFIG_DEBUG_VM_RB
        node->cached_vma_start = avc_start_pgoff(node);
        node->cached_vma_last = avc_last_pgoff(node);
#endif
        __anon_vma_interval_tree_insert(node, root);
}

void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
                                   struct rb_root_cached *root)
{
        __anon_vma_interval_tree_remove(node, root);
}

struct anon_vma_chain *
anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
                                  unsigned long first, unsigned long last)
{
        return __anon_vma_interval_tree_iter_first(root, first, last);
}

struct anon_vma_chain *
anon_vma_interval_tree_iter_next(struct anon_vma_chain *node,
                                 unsigned long first, unsigned long last)
{
        return __anon_vma_interval_tree_iter_next(node, first, last);
}

#ifdef CONFIG_DEBUG_VM_RB
void anon_vma_interval_tree_verify(struct anon_vma_chain *node)
{
        WARN_ON_ONCE(node->cached_vma_start != avc_start_pgoff(node));
        WARN_ON_ONCE(node->cached_vma_last != avc_last_pgoff(node));
}
#endif


























   65 






   65 





















   48 
    5 
   46 










   62 















    7 




    4 


    1 
    5 
















    5 
    2 


    4 









    1 















    1 












    3 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * OSS compatible sequencer driver
 *
 * seq_oss_writeq.c - write queue and sync
 *
 * Copyright (C) 1998,99 Takashi Iwai <tiwai@suse.de>
 */

#include "seq_oss_writeq.h"
#include "seq_oss_event.h"
#include "seq_oss_timer.h"
#include <sound/seq_oss_legacy.h>
#include "../seq_lock.h"
#include "../seq_clientmgr.h"
#include <linux/wait.h>
#include <linux/slab.h>
#include <linux/sched/signal.h>


/*
 * create a write queue record
 */
struct seq_oss_writeq *
snd_seq_oss_writeq_new(struct seq_oss_devinfo *dp, int maxlen)
{
        struct seq_oss_writeq *q;
        struct snd_seq_client_pool pool;

        q = kzalloc(sizeof(*q), GFP_KERNEL);
        if (!q)
                return NULL;
        q->dp = dp;
        q->maxlen = maxlen;
        spin_lock_init(&q->sync_lock);
        q->sync_event_put = 0;
        q->sync_time = 0;
        init_waitqueue_head(&q->sync_sleep);

        memset(&pool, 0, sizeof(pool));
        pool.client = dp->cseq;
        pool.output_pool = maxlen;
        pool.output_room = maxlen / 2;

        snd_seq_oss_control(dp, SNDRV_SEQ_IOCTL_SET_CLIENT_POOL, &pool);

        return q;
}

/*
 * delete the write queue
 */
void
snd_seq_oss_writeq_delete(struct seq_oss_writeq *q)
{
        if (q) {
                snd_seq_oss_writeq_clear(q);        /* to be sure */
                kfree(q);
        }
}


/*
 * reset the write queue
 */
void
snd_seq_oss_writeq_clear(struct seq_oss_writeq *q)
{
        struct snd_seq_remove_events reset;

        memset(&reset, 0, sizeof(reset));
        reset.remove_mode = SNDRV_SEQ_REMOVE_OUTPUT; /* remove all */
        snd_seq_oss_control(q->dp, SNDRV_SEQ_IOCTL_REMOVE_EVENTS, &reset);

        /* wake up sleepers if any */
        snd_seq_oss_writeq_wakeup(q, 0);
}

/*
 * wait until the write buffer has enough room
 */
int
snd_seq_oss_writeq_sync(struct seq_oss_writeq *q)
{
        struct seq_oss_devinfo *dp = q->dp;
        abstime_t time;

        time = snd_seq_oss_timer_cur_tick(dp->timer);
        if (q->sync_time >= time)
                return 0; /* already finished */

        if (! q->sync_event_put) {
                struct snd_seq_event ev;
                union evrec *rec;

                /* put echoback event */
                memset(&ev, 0, sizeof(ev));
                ev.flags = 0;
                ev.type = SNDRV_SEQ_EVENT_ECHO;
                ev.time.tick = time;
                /* echo back to itself */
                snd_seq_oss_fill_addr(dp, &ev, dp->addr.client, dp->addr.port);
                rec = (union evrec *)&ev.data;
                rec->t.code = SEQ_SYNCTIMER;
                rec->t.time = time;
                q->sync_event_put = 1;
                snd_seq_kernel_client_enqueue(dp->cseq, &ev, NULL, true);
        }

        wait_event_interruptible_timeout(q->sync_sleep, ! q->sync_event_put, HZ);
        if (signal_pending(current))
                /* interrupted - return 0 to finish sync */
                q->sync_event_put = 0;
        if (! q->sync_event_put || q->sync_time >= time)
                return 0;
        return 1;
}

/*
 * wake up sync - echo event was catched
 */
void
snd_seq_oss_writeq_wakeup(struct seq_oss_writeq *q, abstime_t time)
{
        unsigned long flags;

        spin_lock_irqsave(&q->sync_lock, flags);
        q->sync_time = time;
        q->sync_event_put = 0;
        wake_up(&q->sync_sleep);
        spin_unlock_irqrestore(&q->sync_lock, flags);
}


/*
 * return the unused pool size
 */
int
snd_seq_oss_writeq_get_free_size(struct seq_oss_writeq *q)
{
        struct snd_seq_client_pool pool;
        pool.client = q->dp->cseq;
        snd_seq_oss_control(q->dp, SNDRV_SEQ_IOCTL_GET_CLIENT_POOL, &pool);
        return pool.output_free;
}


/*
 * set output threshold size from ioctl
 */
void
snd_seq_oss_writeq_set_output(struct seq_oss_writeq *q, int val)
{
        struct snd_seq_client_pool pool;
        pool.client = q->dp->cseq;
        snd_seq_oss_control(q->dp, SNDRV_SEQ_IOCTL_GET_CLIENT_POOL, &pool);
        pool.output_room = val;
        snd_seq_oss_control(q->dp, SNDRV_SEQ_IOCTL_SET_CLIENT_POOL, &pool);
}




























  223 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2008 IBM Corporation
 *
 * Authors:
 * Mimi Zohar <zohar@us.ibm.com>
 *
 * File: integrity_iint.c
 *        - initialize the integrity directory in securityfs
 *        - load IMA and EVM keys
 */
#include <linux/security.h>
#include "integrity.h"

struct dentry *integrity_dir;

/*
 * integrity_kernel_read - read data from the file
 *
 * This is a function for reading file content instead of kernel_read().
 * It does not perform locking checks to ensure it cannot be blocked.
 * It does not perform security checks because it is irrelevant for IMA.
 *
 */
int integrity_kernel_read(struct file *file, loff_t offset,
                          void *addr, unsigned long count)
{
        return __kernel_read(file, addr, count, &offset);
}

/*
 * integrity_load_keys - load integrity keys hook
 *
 * Hooks is called from init/main.c:kernel_init_freeable()
 * when rootfs is ready
 */
void __init integrity_load_keys(void)
{
        ima_load_x509();

        if (!IS_ENABLED(CONFIG_IMA_LOAD_X509))
                evm_load_x509();
}

static int __init integrity_fs_init(void)
{
        integrity_dir = securityfs_create_dir("integrity", NULL);
        if (IS_ERR(integrity_dir)) {
                int ret = PTR_ERR(integrity_dir);

                if (ret != -ENODEV)
                        pr_err("Unable to create integrity sysfs dir: %d\n",
                               ret);
                integrity_dir = NULL;
                return ret;
        }

        return 0;
}

late_initcall(integrity_fs_init)














































   34 



  230 
















   29 





  208 









  208 

























  232 





  119 
  130 
   23 
    1 



  130 




  133 
  147 
   23 
    1 



  147 










   92 










  232 
















  230 


  232 









  231 





  118 


  118 














  333 




  167 

   26 














  365 

  188 

   56 











  335 




  190 

  235 







  333 




  188 

  234 























  100 
  138 
  222 

























  331 


  331 

  223 







   53 


   53 

   53 











   53 






    2 














  330 







  296 
   41 



  186 


  187 







  231 


  231 






















  330 














   14 




  323 










   12 




  330 
   26 












   19 














   43 




  140 







    3 















  219 















   17 




  313 





















   92 









   57 


   38 











   92 











   57 






   57 





   56 

   13 





   57 











    8 




   55 



   38 





   38 





   36 











   11 









    9 



































    9 










   17 













   15 
    2 





































   17 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
/*
 *  PCM Plug-In shared (kernel/library) code
 *  Copyright (c) 1999 by Jaroslav Kysela <perex@perex.cz>
 *  Copyright (c) 2000 by Abramo Bagnara <abramo@alsa-project.org>
 *
 *
 *   This library is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU Library General Public License as
 *   published by the Free Software Foundation; either version 2 of
 *   the License, or (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU Library General Public License for more details.
 *
 *   You should have received a copy of the GNU Library General Public
 *   License along with this library; if not, write to the Free Software
 *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 *
 */
  
#if 0
#define PLUGIN_DEBUG
#endif

#include <linux/slab.h>
#include <linux/time.h>
#include <linux/vmalloc.h>
#include <sound/core.h>
#include <sound/pcm.h>
#include <sound/pcm_params.h>
#include "pcm_plugin.h"

#define snd_pcm_plug_first(plug) ((plug)->runtime->oss.plugin_first)
#define snd_pcm_plug_last(plug) ((plug)->runtime->oss.plugin_last)

/*
 *  because some cards might have rates "very close", we ignore
 *  all "resampling" requests within +-5%
 */
static int rate_match(unsigned int src_rate, unsigned int dst_rate)
{
        unsigned int low = (src_rate * 95) / 100;
        unsigned int high = (src_rate * 105) / 100;
        return dst_rate >= low && dst_rate <= high;
}

static int snd_pcm_plugin_alloc(struct snd_pcm_plugin *plugin, snd_pcm_uframes_t frames)
{
        struct snd_pcm_plugin_format *format;
        ssize_t width;
        size_t size;
        unsigned int channel;
        struct snd_pcm_plugin_channel *c;

        if (plugin->stream == SNDRV_PCM_STREAM_PLAYBACK) {
                format = &plugin->src_format;
        } else {
                format = &plugin->dst_format;
        }
        width = snd_pcm_format_physical_width(format->format);
        if (width < 0)
                return width;
        size = array3_size(frames, format->channels, width);
        /* check for too large period size once again */
        if (size > 1024 * 1024)
                return -ENOMEM;
        if (snd_BUG_ON(size % 8))
                return -ENXIO;
        size /= 8;
        if (plugin->buf_frames < frames) {
                kvfree(plugin->buf);
                plugin->buf = kvzalloc(size, GFP_KERNEL);
                plugin->buf_frames = frames;
        }
        if (!plugin->buf) {
                plugin->buf_frames = 0;
                return -ENOMEM;
        }
        c = plugin->buf_channels;
        if (plugin->access == SNDRV_PCM_ACCESS_RW_INTERLEAVED) {
                for (channel = 0; channel < format->channels; channel++, c++) {
                        c->frames = frames;
                        c->enabled = 1;
                        c->wanted = 0;
                        c->area.addr = plugin->buf;
                        c->area.first = channel * width;
                        c->area.step = format->channels * width;
                }
        } else if (plugin->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) {
                if (snd_BUG_ON(size % format->channels))
                        return -EINVAL;
                size /= format->channels;
                for (channel = 0; channel < format->channels; channel++, c++) {
                        c->frames = frames;
                        c->enabled = 1;
                        c->wanted = 0;
                        c->area.addr = plugin->buf + (channel * size);
                        c->area.first = 0;
                        c->area.step = width;
                }
        } else
                return -EINVAL;
        return 0;
}

int snd_pcm_plug_alloc(struct snd_pcm_substream *plug, snd_pcm_uframes_t frames)
{
        int err;
        if (snd_BUG_ON(!snd_pcm_plug_first(plug)))
                return -ENXIO;
        if (snd_pcm_plug_stream(plug) == SNDRV_PCM_STREAM_PLAYBACK) {
                struct snd_pcm_plugin *plugin = snd_pcm_plug_first(plug);
                while (plugin->next) {
                        if (plugin->dst_frames)
                                frames = plugin->dst_frames(plugin, frames);
                        if ((snd_pcm_sframes_t)frames <= 0)
                                return -ENXIO;
                        plugin = plugin->next;
                        err = snd_pcm_plugin_alloc(plugin, frames);
                        if (err < 0)
                                return err;
                }
        } else {
                struct snd_pcm_plugin *plugin = snd_pcm_plug_last(plug);
                while (plugin->prev) {
                        if (plugin->src_frames)
                                frames = plugin->src_frames(plugin, frames);
                        if ((snd_pcm_sframes_t)frames <= 0)
                                return -ENXIO;
                        plugin = plugin->prev;
                        err = snd_pcm_plugin_alloc(plugin, frames);
                        if (err < 0)
                                return err;
                }
        }
        return 0;
}


snd_pcm_sframes_t snd_pcm_plugin_client_channels(struct snd_pcm_plugin *plugin,
                                       snd_pcm_uframes_t frames,
                                       struct snd_pcm_plugin_channel **channels)
{
        *channels = plugin->buf_channels;
        return frames;
}

int snd_pcm_plugin_build(struct snd_pcm_substream *plug,
                         const char *name,
                         struct snd_pcm_plugin_format *src_format,
                         struct snd_pcm_plugin_format *dst_format,
                         size_t extra,
                         struct snd_pcm_plugin **ret)
{
        struct snd_pcm_plugin *plugin;
        unsigned int channels;
        
        if (snd_BUG_ON(!plug))
                return -ENXIO;
        if (snd_BUG_ON(!src_format || !dst_format))
                return -ENXIO;
        plugin = kzalloc(sizeof(*plugin) + extra, GFP_KERNEL);
        if (plugin == NULL)
                return -ENOMEM;
        plugin->name = name;
        plugin->plug = plug;
        plugin->stream = snd_pcm_plug_stream(plug);
        plugin->access = SNDRV_PCM_ACCESS_RW_INTERLEAVED;
        plugin->src_format = *src_format;
        plugin->src_width = snd_pcm_format_physical_width(src_format->format);
        snd_BUG_ON(plugin->src_width <= 0);
        plugin->dst_format = *dst_format;
        plugin->dst_width = snd_pcm_format_physical_width(dst_format->format);
        snd_BUG_ON(plugin->dst_width <= 0);
        if (plugin->stream == SNDRV_PCM_STREAM_PLAYBACK)
                channels = src_format->channels;
        else
                channels = dst_format->channels;
        plugin->buf_channels = kcalloc(channels, sizeof(*plugin->buf_channels), GFP_KERNEL);
        if (plugin->buf_channels == NULL) {
                snd_pcm_plugin_free(plugin);
                return -ENOMEM;
        }
        plugin->client_channels = snd_pcm_plugin_client_channels;
        *ret = plugin;
        return 0;
}

int snd_pcm_plugin_free(struct snd_pcm_plugin *plugin)
{
        if (! plugin)
                return 0;
        if (plugin->private_free)
                plugin->private_free(plugin);
        kfree(plugin->buf_channels);
        kvfree(plugin->buf);
        kfree(plugin);
        return 0;
}

static snd_pcm_sframes_t calc_dst_frames(struct snd_pcm_substream *plug,
                                         snd_pcm_sframes_t frames,
                                         bool check_size)
{
        struct snd_pcm_plugin *plugin, *plugin_next;

        plugin = snd_pcm_plug_first(plug);
        while (plugin && frames > 0) {
                plugin_next = plugin->next;
                if (check_size && plugin->buf_frames &&
                    frames > plugin->buf_frames)
                        frames = plugin->buf_frames;
                if (plugin->dst_frames) {
                        frames = plugin->dst_frames(plugin, frames);
                        if (frames < 0)
                                return frames;
                }
                plugin = plugin_next;
        }
        return frames;
}

static snd_pcm_sframes_t calc_src_frames(struct snd_pcm_substream *plug,
                                         snd_pcm_sframes_t frames,
                                         bool check_size)
{
        struct snd_pcm_plugin *plugin, *plugin_prev;

        plugin = snd_pcm_plug_last(plug);
        while (plugin && frames > 0) {
                plugin_prev = plugin->prev;
                if (plugin->src_frames) {
                        frames = plugin->src_frames(plugin, frames);
                        if (frames < 0)
                                return frames;
                }
                if (check_size && plugin->buf_frames &&
                    frames > plugin->buf_frames)
                        frames = plugin->buf_frames;
                plugin = plugin_prev;
        }
        return frames;
}

snd_pcm_sframes_t snd_pcm_plug_client_size(struct snd_pcm_substream *plug, snd_pcm_uframes_t drv_frames)
{
        if (snd_BUG_ON(!plug))
                return -ENXIO;
        switch (snd_pcm_plug_stream(plug)) {
        case SNDRV_PCM_STREAM_PLAYBACK:
                return calc_src_frames(plug, drv_frames, false);
        case SNDRV_PCM_STREAM_CAPTURE:
                return calc_dst_frames(plug, drv_frames, false);
        default:
                snd_BUG();
                return -EINVAL;
        }
}

snd_pcm_sframes_t snd_pcm_plug_slave_size(struct snd_pcm_substream *plug, snd_pcm_uframes_t clt_frames)
{
        if (snd_BUG_ON(!plug))
                return -ENXIO;
        switch (snd_pcm_plug_stream(plug)) {
        case SNDRV_PCM_STREAM_PLAYBACK:
                return calc_dst_frames(plug, clt_frames, false);
        case SNDRV_PCM_STREAM_CAPTURE:
                return calc_src_frames(plug, clt_frames, false);
        default:
                snd_BUG();
                return -EINVAL;
        }
}

static int snd_pcm_plug_formats(const struct snd_mask *mask,
                                snd_pcm_format_t format)
{
        struct snd_mask formats = *mask;
        u64 linfmts = (SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
                       SNDRV_PCM_FMTBIT_U16_LE | SNDRV_PCM_FMTBIT_S16_LE |
                       SNDRV_PCM_FMTBIT_U16_BE | SNDRV_PCM_FMTBIT_S16_BE |
                       SNDRV_PCM_FMTBIT_U24_LE | SNDRV_PCM_FMTBIT_S24_LE |
                       SNDRV_PCM_FMTBIT_U24_BE | SNDRV_PCM_FMTBIT_S24_BE |
                       SNDRV_PCM_FMTBIT_U24_3LE | SNDRV_PCM_FMTBIT_S24_3LE |
                       SNDRV_PCM_FMTBIT_U24_3BE | SNDRV_PCM_FMTBIT_S24_3BE |
                       SNDRV_PCM_FMTBIT_U32_LE | SNDRV_PCM_FMTBIT_S32_LE |
                       SNDRV_PCM_FMTBIT_U32_BE | SNDRV_PCM_FMTBIT_S32_BE);
        snd_mask_set(&formats, (__force int)SNDRV_PCM_FORMAT_MU_LAW);
        
        if (formats.bits[0] & lower_32_bits(linfmts))
                formats.bits[0] |= lower_32_bits(linfmts);
        if (formats.bits[1] & upper_32_bits(linfmts))
                formats.bits[1] |= upper_32_bits(linfmts);
        return snd_mask_test(&formats, (__force int)format);
}

static const snd_pcm_format_t preferred_formats[] = {
        SNDRV_PCM_FORMAT_S16_LE,
        SNDRV_PCM_FORMAT_S16_BE,
        SNDRV_PCM_FORMAT_U16_LE,
        SNDRV_PCM_FORMAT_U16_BE,
        SNDRV_PCM_FORMAT_S24_3LE,
        SNDRV_PCM_FORMAT_S24_3BE,
        SNDRV_PCM_FORMAT_U24_3LE,
        SNDRV_PCM_FORMAT_U24_3BE,
        SNDRV_PCM_FORMAT_S24_LE,
        SNDRV_PCM_FORMAT_S24_BE,
        SNDRV_PCM_FORMAT_U24_LE,
        SNDRV_PCM_FORMAT_U24_BE,
        SNDRV_PCM_FORMAT_S32_LE,
        SNDRV_PCM_FORMAT_S32_BE,
        SNDRV_PCM_FORMAT_U32_LE,
        SNDRV_PCM_FORMAT_U32_BE,
        SNDRV_PCM_FORMAT_S8,
        SNDRV_PCM_FORMAT_U8
};

snd_pcm_format_t snd_pcm_plug_slave_format(snd_pcm_format_t format,
                                           const struct snd_mask *format_mask)
{
        int i;

        if (snd_mask_test(format_mask, (__force int)format))
                return format;
        if (!snd_pcm_plug_formats(format_mask, format))
                return (__force snd_pcm_format_t)-EINVAL;
        if (snd_pcm_format_linear(format)) {
                unsigned int width = snd_pcm_format_width(format);
                int unsignd = snd_pcm_format_unsigned(format) > 0;
                int big = snd_pcm_format_big_endian(format) > 0;
                unsigned int badness, best = -1;
                snd_pcm_format_t best_format = (__force snd_pcm_format_t)-1;
                for (i = 0; i < ARRAY_SIZE(preferred_formats); i++) {
                        snd_pcm_format_t f = preferred_formats[i];
                        unsigned int w;
                        if (!snd_mask_test(format_mask, (__force int)f))
                                continue;
                        w = snd_pcm_format_width(f);
                        if (w >= width)
                                badness = w - width;
                        else
                                badness = width - w + 32;
                        badness += snd_pcm_format_unsigned(f) != unsignd;
                        badness += snd_pcm_format_big_endian(f) != big;
                        if (badness < best) {
                                best_format = f;
                                best = badness;
                        }
                }
                if ((__force int)best_format >= 0)
                        return best_format;
                else
                        return (__force snd_pcm_format_t)-EINVAL;
        } else {
                switch (format) {
                case SNDRV_PCM_FORMAT_MU_LAW:
                        for (i = 0; i < ARRAY_SIZE(preferred_formats); ++i) {
                                snd_pcm_format_t format1 = preferred_formats[i];
                                if (snd_mask_test(format_mask, (__force int)format1))
                                        return format1;
                        }
                        fallthrough;
                default:
                        return (__force snd_pcm_format_t)-EINVAL;
                }
        }
}

int snd_pcm_plug_format_plugins(struct snd_pcm_substream *plug,
                                struct snd_pcm_hw_params *params,
                                struct snd_pcm_hw_params *slave_params)
{
        struct snd_pcm_plugin_format tmpformat;
        struct snd_pcm_plugin_format dstformat;
        struct snd_pcm_plugin_format srcformat;
        snd_pcm_access_t src_access, dst_access;
        struct snd_pcm_plugin *plugin = NULL;
        int err;
        int stream = snd_pcm_plug_stream(plug);
        int slave_interleaved = (params_channels(slave_params) == 1 ||
                                 params_access(slave_params) == SNDRV_PCM_ACCESS_RW_INTERLEAVED);

        switch (stream) {
        case SNDRV_PCM_STREAM_PLAYBACK:
                dstformat.format = params_format(slave_params);
                dstformat.rate = params_rate(slave_params);
                dstformat.channels = params_channels(slave_params);
                srcformat.format = params_format(params);
                srcformat.rate = params_rate(params);
                srcformat.channels = params_channels(params);
                src_access = SNDRV_PCM_ACCESS_RW_INTERLEAVED;
                dst_access = (slave_interleaved ? SNDRV_PCM_ACCESS_RW_INTERLEAVED :
                                                  SNDRV_PCM_ACCESS_RW_NONINTERLEAVED);
                break;
        case SNDRV_PCM_STREAM_CAPTURE:
                dstformat.format = params_format(params);
                dstformat.rate = params_rate(params);
                dstformat.channels = params_channels(params);
                srcformat.format = params_format(slave_params);
                srcformat.rate = params_rate(slave_params);
                srcformat.channels = params_channels(slave_params);
                src_access = (slave_interleaved ? SNDRV_PCM_ACCESS_RW_INTERLEAVED :
                                                  SNDRV_PCM_ACCESS_RW_NONINTERLEAVED);
                dst_access = SNDRV_PCM_ACCESS_RW_INTERLEAVED;
                break;
        default:
                snd_BUG();
                return -EINVAL;
        }
        tmpformat = srcformat;
                
        pdprintf("srcformat: format=%i, rate=%i, channels=%i\n", 
                 srcformat.format,
                 srcformat.rate,
                 srcformat.channels);
        pdprintf("dstformat: format=%i, rate=%i, channels=%i\n", 
                 dstformat.format,
                 dstformat.rate,
                 dstformat.channels);

        /* Format change (linearization) */
        if (! rate_match(srcformat.rate, dstformat.rate) &&
            ! snd_pcm_format_linear(srcformat.format)) {
                if (srcformat.format != SNDRV_PCM_FORMAT_MU_LAW)
                        return -EINVAL;
                tmpformat.format = SNDRV_PCM_FORMAT_S16;
                err = snd_pcm_plugin_build_mulaw(plug,
                                                 &srcformat, &tmpformat,
                                                 &plugin);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
                srcformat = tmpformat;
                src_access = dst_access;
        }

        /* channels reduction */
        if (srcformat.channels > dstformat.channels) {
                tmpformat.channels = dstformat.channels;
                err = snd_pcm_plugin_build_route(plug, &srcformat, &tmpformat, &plugin);
                pdprintf("channels reduction: src=%i, dst=%i returns %i\n", srcformat.channels, tmpformat.channels, err);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
                srcformat = tmpformat;
                src_access = dst_access;
        }

        /* rate resampling */
        if (!rate_match(srcformat.rate, dstformat.rate)) {
                if (srcformat.format != SNDRV_PCM_FORMAT_S16) {
                        /* convert to S16 for resampling */
                        tmpformat.format = SNDRV_PCM_FORMAT_S16;
                        err = snd_pcm_plugin_build_linear(plug,
                                                          &srcformat, &tmpformat,
                                                          &plugin);
                        if (err < 0)
                                return err;
                        err = snd_pcm_plugin_append(plugin);
                        if (err < 0) {
                                snd_pcm_plugin_free(plugin);
                                return err;
                        }
                        srcformat = tmpformat;
                        src_access = dst_access;
                }
                tmpformat.rate = dstformat.rate;
                err = snd_pcm_plugin_build_rate(plug,
                                                &srcformat, &tmpformat,
                                                &plugin);
                pdprintf("rate down resampling: src=%i, dst=%i returns %i\n", srcformat.rate, tmpformat.rate, err);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
                srcformat = tmpformat;
                src_access = dst_access;
        }

        /* format change */
        if (srcformat.format != dstformat.format) {
                tmpformat.format = dstformat.format;
                if (srcformat.format == SNDRV_PCM_FORMAT_MU_LAW ||
                    tmpformat.format == SNDRV_PCM_FORMAT_MU_LAW) {
                        err = snd_pcm_plugin_build_mulaw(plug,
                                                         &srcformat, &tmpformat,
                                                         &plugin);
                }
                else if (snd_pcm_format_linear(srcformat.format) &&
                         snd_pcm_format_linear(tmpformat.format)) {
                        err = snd_pcm_plugin_build_linear(plug,
                                                          &srcformat, &tmpformat,
                                                          &plugin);
                }
                else
                        return -EINVAL;
                pdprintf("format change: src=%i, dst=%i returns %i\n", srcformat.format, tmpformat.format, err);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
                srcformat = tmpformat;
                src_access = dst_access;
        }

        /* channels extension */
        if (srcformat.channels < dstformat.channels) {
                tmpformat.channels = dstformat.channels;
                err = snd_pcm_plugin_build_route(plug, &srcformat, &tmpformat, &plugin);
                pdprintf("channels extension: src=%i, dst=%i returns %i\n", srcformat.channels, tmpformat.channels, err);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
                srcformat = tmpformat;
                src_access = dst_access;
        }

        /* de-interleave */
        if (src_access != dst_access) {
                err = snd_pcm_plugin_build_copy(plug,
                                                &srcformat,
                                                &tmpformat,
                                                &plugin);
                pdprintf("interleave change (copy: returns %i)\n", err);
                if (err < 0)
                        return err;
                err = snd_pcm_plugin_append(plugin);
                if (err < 0) {
                        snd_pcm_plugin_free(plugin);
                        return err;
                }
        }

        return 0;
}

snd_pcm_sframes_t snd_pcm_plug_client_channels_buf(struct snd_pcm_substream *plug,
                                         char *buf,
                                         snd_pcm_uframes_t count,
                                         struct snd_pcm_plugin_channel **channels)
{
        struct snd_pcm_plugin *plugin;
        struct snd_pcm_plugin_channel *v;
        struct snd_pcm_plugin_format *format;
        int width, nchannels, channel;
        int stream = snd_pcm_plug_stream(plug);

        if (snd_BUG_ON(!buf))
                return -ENXIO;
        if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
                plugin = snd_pcm_plug_first(plug);
                format = &plugin->src_format;
        } else {
                plugin = snd_pcm_plug_last(plug);
                format = &plugin->dst_format;
        }
        v = plugin->buf_channels;
        *channels = v;
        width = snd_pcm_format_physical_width(format->format);
        if (width < 0)
                return width;
        nchannels = format->channels;
        if (snd_BUG_ON(plugin->access != SNDRV_PCM_ACCESS_RW_INTERLEAVED &&
                       format->channels > 1))
                return -ENXIO;
        for (channel = 0; channel < nchannels; channel++, v++) {
                v->frames = count;
                v->enabled = 1;
                v->wanted = (stream == SNDRV_PCM_STREAM_CAPTURE);
                v->area.addr = buf;
                v->area.first = channel * width;
                v->area.step = nchannels * width;
        }
        return count;
}

snd_pcm_sframes_t snd_pcm_plug_write_transfer(struct snd_pcm_substream *plug, struct snd_pcm_plugin_channel *src_channels, snd_pcm_uframes_t size)
{
        struct snd_pcm_plugin *plugin, *next;
        struct snd_pcm_plugin_channel *dst_channels;
        int err;
        snd_pcm_sframes_t frames = size;

        plugin = snd_pcm_plug_first(plug);
        while (plugin) {
                if (frames <= 0)
                        return frames;
                next = plugin->next;
                if (next) {
                        snd_pcm_sframes_t frames1 = frames;
                        if (plugin->dst_frames) {
                                frames1 = plugin->dst_frames(plugin, frames);
                                if (frames1 <= 0)
                                        return frames1;
                        }
                        err = next->client_channels(next, frames1, &dst_channels);
                        if (err < 0)
                                return err;
                        if (err != frames1) {
                                frames = err;
                                if (plugin->src_frames) {
                                        frames = plugin->src_frames(plugin, frames1);
                                        if (frames <= 0)
                                                return frames;
                                }
                        }
                } else
                        dst_channels = NULL;
                pdprintf("write plugin: %s, %li\n", plugin->name, frames);
                frames = plugin->transfer(plugin, src_channels, dst_channels, frames);
                if (frames < 0)
                        return frames;
                src_channels = dst_channels;
                plugin = next;
        }
        return calc_src_frames(plug, frames, true);
}

snd_pcm_sframes_t snd_pcm_plug_read_transfer(struct snd_pcm_substream *plug, struct snd_pcm_plugin_channel *dst_channels_final, snd_pcm_uframes_t size)
{
        struct snd_pcm_plugin *plugin, *next;
        struct snd_pcm_plugin_channel *src_channels, *dst_channels;
        snd_pcm_sframes_t frames = size;
        int err;

        frames = calc_src_frames(plug, frames, true);
        if (frames < 0)
                return frames;

        src_channels = NULL;
        plugin = snd_pcm_plug_first(plug);
        while (plugin && frames > 0) {
                next = plugin->next;
                if (next) {
                        err = plugin->client_channels(plugin, frames, &dst_channels);
                        if (err < 0)
                                return err;
                        frames = err;
                } else {
                        dst_channels = dst_channels_final;
                }
                pdprintf("read plugin: %s, %li\n", plugin->name, frames);
                frames = plugin->transfer(plugin, src_channels, dst_channels, frames);
                if (frames < 0)
                        return frames;
                plugin = next;
                src_channels = dst_channels;
        }
        return frames;
}

int snd_pcm_area_silence(const struct snd_pcm_channel_area *dst_area, size_t dst_offset,
                         size_t samples, snd_pcm_format_t format)
{
        /* FIXME: sub byte resolution and odd dst_offset */
        unsigned char *dst;
        unsigned int dst_step;
        int width;
        const unsigned char *silence;
        if (!dst_area->addr)
                return 0;
        dst = dst_area->addr + (dst_area->first + dst_area->step * dst_offset) / 8;
        width = snd_pcm_format_physical_width(format);
        if (width <= 0)
                return -EINVAL;
        if (dst_area->step == (unsigned int) width && width >= 8)
                return snd_pcm_format_set_silence(format, dst, samples);
        silence = snd_pcm_format_silence_64(format);
        if (! silence)
                return -EINVAL;
        dst_step = dst_area->step / 8;
        if (width == 4) {
                /* Ima ADPCM */
                int dstbit = dst_area->first % 8;
                int dstbit_step = dst_area->step % 8;
                while (samples-- > 0) {
                        if (dstbit)
                                *dst &= 0xf0;
                        else
                                *dst &= 0x0f;
                        dst += dst_step;
                        dstbit += dstbit_step;
                        if (dstbit == 8) {
                                dst++;
                                dstbit = 0;
                        }
                }
        } else {
                width /= 8;
                while (samples-- > 0) {
                        memcpy(dst, silence, width);
                        dst += dst_step;
                }
        }
        return 0;
}

int snd_pcm_area_copy(const struct snd_pcm_channel_area *src_area, size_t src_offset,
                      const struct snd_pcm_channel_area *dst_area, size_t dst_offset,
                      size_t samples, snd_pcm_format_t format)
{
        /* FIXME: sub byte resolution and odd dst_offset */
        char *src, *dst;
        int width;
        int src_step, dst_step;
        src = src_area->addr + (src_area->first + src_area->step * src_offset) / 8;
        if (!src_area->addr)
                return snd_pcm_area_silence(dst_area, dst_offset, samples, format);
        dst = dst_area->addr + (dst_area->first + dst_area->step * dst_offset) / 8;
        if (!dst_area->addr)
                return 0;
        width = snd_pcm_format_physical_width(format);
        if (width <= 0)
                return -EINVAL;
        if (src_area->step == (unsigned int) width &&
            dst_area->step == (unsigned int) width && width >= 8) {
                size_t bytes = samples * width / 8;
                memcpy(dst, src, bytes);
                return 0;
        }
        src_step = src_area->step / 8;
        dst_step = dst_area->step / 8;
        if (width == 4) {
                /* Ima ADPCM */
                int srcbit = src_area->first % 8;
                int srcbit_step = src_area->step % 8;
                int dstbit = dst_area->first % 8;
                int dstbit_step = dst_area->step % 8;
                while (samples-- > 0) {
                        unsigned char srcval;
                        if (srcbit)
                                srcval = *src & 0x0f;
                        else
                                srcval = (*src & 0xf0) >> 4;
                        if (dstbit)
                                *dst = (*dst & 0xf0) | srcval;
                        else
                                *dst = (*dst & 0x0f) | (srcval << 4);
                        src += src_step;
                        srcbit += srcbit_step;
                        if (srcbit == 8) {
                                src++;
                                srcbit = 0;
                        }
                        dst += dst_step;
                        dstbit += dstbit_step;
                        if (dstbit == 8) {
                                dst++;
                                dstbit = 0;
                        }
                }
        } else {
                width /= 8;
                while (samples-- > 0) {
                        memcpy(dst, src, width);
                        src += src_step;
                        dst += dst_step;
                }
        }
        return 0;
}

























































    5 




    5 






   12 



    6 













   12 










 2162 




















 2118 

 2118 

 1078 
 1077 

 1077 

    3 


  173 
   23 





  342 





  343 



































    2 









    5 

    3 











































  490 

  318 











  341 



  319 





   85 
   15 








  502 

    6 

  499 





  342 




    3 



  339 










  338 











  504 



  504 
  147 







  503 

   41 


   48 



  504 





  505 


  505 
   24 


  504 
   11 









    1 



    1 










    1 
    1 


























   23 





    1 





   22 


   19 
    2 


   15 

    1 



    3 





   12 
    6 

   11 
    7 




   18 









    4 


    4 
    4 







    9 


    9 


    8 


    7 




    6 



    1 



    6 



    6 



    5 



    5 








   26 







    1 



   25 


   26 




   24 
    9 
   20 









   22 








   22 
   21 





















   85 















   85 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
// SPDX-License-Identifier: GPL-2.0-only
/*
 * net/ipv6/fib6_rules.c        IPv6 Routing Policy Rules
 *
 * Copyright (C)2003-2006 Helsinki University of Technology
 * Copyright (C)2003-2006 USAGI/WIDE Project
 *
 * Authors
 *        Thomas Graf                <tgraf@suug.ch>
 *        Ville Nuorvala                <vnuorval@tcs.hut.fi>
 */

#include <linux/netdevice.h>
#include <linux/notifier.h>
#include <linux/export.h>
#include <linux/indirect_call_wrapper.h>

#include <net/fib_rules.h>
#include <net/inet_dscp.h>
#include <net/ipv6.h>
#include <net/addrconf.h>
#include <net/ip6_route.h>
#include <net/netlink.h>

struct fib6_rule {
        struct fib_rule                common;
        struct rt6key                src;
        struct rt6key                dst;
        __be32                        flowlabel;
        __be32                        flowlabel_mask;
        dscp_t                        dscp;
        u8                        dscp_full:1;        /* DSCP or TOS selector */
};

static bool fib6_rule_matchall(const struct fib_rule *rule)
{
        struct fib6_rule *r = container_of(rule, struct fib6_rule, common);

        if (r->dst.plen || r->src.plen || r->dscp || r->flowlabel_mask)
                return false;
        return fib_rule_matchall(rule);
}

bool fib6_rule_default(const struct fib_rule *rule)
{
        if (!fib6_rule_matchall(rule) || rule->action != FR_ACT_TO_TBL ||
            rule->l3mdev)
                return false;
        if (rule->table != RT6_TABLE_LOCAL && rule->table != RT6_TABLE_MAIN)
                return false;
        return true;
}
EXPORT_SYMBOL_GPL(fib6_rule_default);

int fib6_rules_dump(struct net *net, struct notifier_block *nb,
                    struct netlink_ext_ack *extack)
{
        return fib_rules_dump(net, nb, AF_INET6, extack);
}

unsigned int fib6_rules_seq_read(const struct net *net)
{
        return fib_rules_seq_read(net, AF_INET6);
}

/* called with rcu lock held; no reference taken on fib6_info */
int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6,
                struct fib6_result *res, int flags)
{
        int err;

        if (net->ipv6.fib6_has_custom_rules) {
                struct fib_lookup_arg arg = {
                        .lookup_ptr = fib6_table_lookup,
                        .lookup_data = &oif,
                        .result = res,
                        .flags = FIB_LOOKUP_NOREF,
                };

                l3mdev_update_flow(net, flowi6_to_flowi(fl6));

                err = fib_rules_lookup(net->ipv6.fib6_rules_ops,
                                       flowi6_to_flowi(fl6), flags, &arg);
        } else {
                err = fib6_table_lookup(net, net->ipv6.fib6_local_tbl, oif,
                                        fl6, res, flags);
                if (err || res->f6i == net->ipv6.fib6_null_entry)
                        err = fib6_table_lookup(net, net->ipv6.fib6_main_tbl,
                                                oif, fl6, res, flags);
        }

        return err;
}

struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,
                                   const struct sk_buff *skb,
                                   int flags, pol_lookup_t lookup)
{
        if (net->ipv6.fib6_has_custom_rules) {
                struct fib6_result res = {};
                struct fib_lookup_arg arg = {
                        .lookup_ptr = lookup,
                        .lookup_data = skb,
                        .result = &res,
                        .flags = FIB_LOOKUP_NOREF,
                };

                /* update flow if oif or iif point to device enslaved to l3mdev */
                l3mdev_update_flow(net, flowi6_to_flowi(fl6));

                fib_rules_lookup(net->ipv6.fib6_rules_ops,
                                 flowi6_to_flowi(fl6), flags, &arg);

                if (res.rt6)
                        return &res.rt6->dst;
        } else {
                struct rt6_info *rt;

                rt = pol_lookup_func(lookup,
                             net, net->ipv6.fib6_local_tbl, fl6, skb, flags);
                if (rt != net->ipv6.ip6_null_entry && rt->dst.error != -EAGAIN)
                        return &rt->dst;
                ip6_rt_put_flags(rt, flags);
                rt = pol_lookup_func(lookup,
                             net, net->ipv6.fib6_main_tbl, fl6, skb, flags);
                if (rt->dst.error != -EAGAIN)
                        return &rt->dst;
                ip6_rt_put_flags(rt, flags);
        }

        if (!(flags & RT6_LOOKUP_F_DST_NOREF))
                dst_hold(&net->ipv6.ip6_null_entry->dst);
        return &net->ipv6.ip6_null_entry->dst;
}

static int fib6_rule_saddr(struct net *net, struct fib_rule *rule, int flags,
                           struct flowi6 *flp6, const struct net_device *dev)
{
        struct fib6_rule *r = (struct fib6_rule *)rule;

        /* If we need to find a source address for this traffic,
         * we check the result if it meets requirement of the rule.
         */
        if ((rule->flags & FIB_RULE_FIND_SADDR) &&
            r->src.plen && !(flags & RT6_LOOKUP_F_HAS_SADDR)) {
                struct in6_addr saddr;

                if (ipv6_dev_get_saddr(net, dev, &flp6->daddr,
                                       rt6_flags2srcprefs(flags), &saddr))
                        return -EAGAIN;

                if (!ipv6_prefix_equal(&saddr, &r->src.addr, r->src.plen))
                        return -EAGAIN;

                flp6->saddr = saddr;
        }

        return 0;
}

static int fib6_rule_action_alt(struct fib_rule *rule, struct flowi *flp,
                                int flags, struct fib_lookup_arg *arg)
{
        struct fib6_result *res = arg->result;
        struct flowi6 *flp6 = &flp->u.ip6;
        struct net *net = rule->fr_net;
        struct fib6_table *table;
        int err, *oif;
        u32 tb_id;

        switch (rule->action) {
        case FR_ACT_TO_TBL:
                break;
        case FR_ACT_UNREACHABLE:
                return -ENETUNREACH;
        case FR_ACT_PROHIBIT:
                return -EACCES;
        case FR_ACT_BLACKHOLE:
        default:
                return -EINVAL;
        }

        tb_id = fib_rule_get_table(rule, arg);
        table = fib6_get_table(net, tb_id);
        if (!table)
                return -EAGAIN;

        oif = (int *)arg->lookup_data;
        err = fib6_table_lookup(net, table, *oif, flp6, res, flags);
        if (!err && res->f6i != net->ipv6.fib6_null_entry)
                err = fib6_rule_saddr(net, rule, flags, flp6,
                                      res->nh->fib_nh_dev);
        else
                err = -EAGAIN;

        return err;
}

static int __fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
                              int flags, struct fib_lookup_arg *arg)
{
        struct fib6_result *res = arg->result;
        struct flowi6 *flp6 = &flp->u.ip6;
        struct rt6_info *rt = NULL;
        struct fib6_table *table;
        struct net *net = rule->fr_net;
        pol_lookup_t lookup = arg->lookup_ptr;
        int err = 0;
        u32 tb_id;

        switch (rule->action) {
        case FR_ACT_TO_TBL:
                break;
        case FR_ACT_UNREACHABLE:
                err = -ENETUNREACH;
                rt = net->ipv6.ip6_null_entry;
                goto discard_pkt;
        default:
        case FR_ACT_BLACKHOLE:
                err = -EINVAL;
                rt = net->ipv6.ip6_blk_hole_entry;
                goto discard_pkt;
        case FR_ACT_PROHIBIT:
                err = -EACCES;
                rt = net->ipv6.ip6_prohibit_entry;
                goto discard_pkt;
        }

        tb_id = fib_rule_get_table(rule, arg);
        table = fib6_get_table(net, tb_id);
        if (!table) {
                err = -EAGAIN;
                goto out;
        }

        rt = pol_lookup_func(lookup,
                             net, table, flp6, arg->lookup_data, flags);
        if (rt != net->ipv6.ip6_null_entry) {
                struct inet6_dev *idev = ip6_dst_idev(&rt->dst);

                if (!idev)
                        goto again;
                err = fib6_rule_saddr(net, rule, flags, flp6,
                                      idev->dev);

                if (err == -EAGAIN)
                        goto again;

                err = rt->dst.error;
                if (err != -EAGAIN)
                        goto out;
        }
again:
        ip6_rt_put_flags(rt, flags);
        err = -EAGAIN;
        rt = NULL;
        goto out;

discard_pkt:
        if (!(flags & RT6_LOOKUP_F_DST_NOREF))
                dst_hold(&rt->dst);
out:
        res->rt6 = rt;
        return err;
}

INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
                                             struct flowi *flp, int flags,
                                             struct fib_lookup_arg *arg)
{
        if (arg->lookup_ptr == fib6_table_lookup)
                return fib6_rule_action_alt(rule, flp, flags, arg);

        return __fib6_rule_action(rule, flp, flags, arg);
}

INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
                                                int flags,
                                                struct fib_lookup_arg *arg)
{
        struct fib6_result *res = arg->result;
        struct rt6_info *rt = res->rt6;
        struct net_device *dev = NULL;

        if (!rt)
                return false;

        if (rt->rt6i_idev)
                dev = rt->rt6i_idev->dev;

        /* do not accept result if the route does
         * not meet the required prefix length
         */
        if (rt->rt6i_dst.plen <= rule->suppress_prefixlen)
                goto suppress_route;

        /* do not accept result if the route uses a device
         * belonging to a forbidden interface group
         */
        if (rule->suppress_ifgroup != -1 && dev && dev->group == rule->suppress_ifgroup)
                goto suppress_route;

        return false;

suppress_route:
        ip6_rt_put_flags(rt, flags);
        return true;
}

INDIRECT_CALLABLE_SCOPE int fib6_rule_match(struct fib_rule *rule,
                                            struct flowi *fl, int flags)
{
        struct fib6_rule *r = (struct fib6_rule *) rule;
        struct flowi6 *fl6 = &fl->u.ip6;

        if (r->dst.plen &&
            !ipv6_prefix_equal(&fl6->daddr, &r->dst.addr, r->dst.plen))
                return 0;

        /*
         * If FIB_RULE_FIND_SADDR is set and we do not have a
         * source address for the traffic, we defer check for
         * source address.
         */
        if (r->src.plen) {
                if (flags & RT6_LOOKUP_F_HAS_SADDR) {
                        if (!ipv6_prefix_equal(&fl6->saddr, &r->src.addr,
                                               r->src.plen))
                                return 0;
                } else if (!(r->common.flags & FIB_RULE_FIND_SADDR))
                        return 0;
        }

        if (r->dscp && r->dscp != ip6_dscp(fl6->flowlabel))
                return 0;

        if ((r->flowlabel ^ flowi6_get_flowlabel(fl6)) & r->flowlabel_mask)
                return 0;

        if (rule->ip_proto && (rule->ip_proto != fl6->flowi6_proto))
                return 0;

        if (fib_rule_port_range_set(&rule->sport_range) &&
            !fib_rule_port_inrange(&rule->sport_range, fl6->fl6_sport))
                return 0;

        if (fib_rule_port_range_set(&rule->dport_range) &&
            !fib_rule_port_inrange(&rule->dport_range, fl6->fl6_dport))
                return 0;

        return 1;
}

static int fib6_nl2rule_dscp(const struct nlattr *nla, struct fib6_rule *rule6,
                             struct netlink_ext_ack *extack)
{
        if (rule6->dscp) {
                NL_SET_ERR_MSG(extack, "Cannot specify both TOS and DSCP");
                return -EINVAL;
        }

        rule6->dscp = inet_dsfield_to_dscp(nla_get_u8(nla) << 2);
        rule6->dscp_full = true;

        return 0;
}

static int fib6_nl2rule_flowlabel(struct nlattr **tb, struct fib6_rule *rule6,
                                  struct netlink_ext_ack *extack)
{
        __be32 flowlabel, flowlabel_mask;

        if (NL_REQ_ATTR_CHECK(extack, NULL, tb, FRA_FLOWLABEL) ||
            NL_REQ_ATTR_CHECK(extack, NULL, tb, FRA_FLOWLABEL_MASK))
                return -EINVAL;

        flowlabel = nla_get_be32(tb[FRA_FLOWLABEL]);
        flowlabel_mask = nla_get_be32(tb[FRA_FLOWLABEL_MASK]);

        if (flowlabel_mask & ~IPV6_FLOWLABEL_MASK) {
                NL_SET_ERR_MSG_ATTR(extack, tb[FRA_FLOWLABEL_MASK],
                                    "Invalid flow label mask");
                return -EINVAL;
        }

        if (flowlabel & ~flowlabel_mask) {
                NL_SET_ERR_MSG(extack, "Flow label and mask do not match");
                return -EINVAL;
        }

        rule6->flowlabel = flowlabel;
        rule6->flowlabel_mask = flowlabel_mask;

        return 0;
}

static int fib6_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
                               struct fib_rule_hdr *frh,
                               struct nlattr **tb,
                               struct netlink_ext_ack *extack)
{
        int err = -EINVAL;
        struct net *net = sock_net(skb->sk);
        struct fib6_rule *rule6 = (struct fib6_rule *) rule;

        if (!inet_validate_dscp(frh->tos)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid dsfield (tos): ECN bits must be 0");
                goto errout;
        }
        rule6->dscp = inet_dsfield_to_dscp(frh->tos);

        if (tb[FRA_DSCP] && fib6_nl2rule_dscp(tb[FRA_DSCP], rule6, extack) < 0)
                goto errout;

        if ((tb[FRA_FLOWLABEL] || tb[FRA_FLOWLABEL_MASK]) &&
            fib6_nl2rule_flowlabel(tb, rule6, extack) < 0)
                goto errout;

        if (rule->action == FR_ACT_TO_TBL && !rule->l3mdev) {
                if (rule->table == RT6_TABLE_UNSPEC) {
                        NL_SET_ERR_MSG(extack, "Invalid table");
                        goto errout;
                }

                if (fib6_new_table(net, rule->table) == NULL) {
                        err = -ENOBUFS;
                        goto errout;
                }
        }

        if (frh->src_len)
                rule6->src.addr = nla_get_in6_addr(tb[FRA_SRC]);

        if (frh->dst_len)
                rule6->dst.addr = nla_get_in6_addr(tb[FRA_DST]);

        rule6->src.plen = frh->src_len;
        rule6->dst.plen = frh->dst_len;

        if (fib_rule_requires_fldissect(rule))
                net->ipv6.fib6_rules_require_fldissect++;

        net->ipv6.fib6_has_custom_rules = true;
        err = 0;
errout:
        return err;
}

static int fib6_rule_delete(struct fib_rule *rule)
{
        struct net *net = rule->fr_net;

        if (net->ipv6.fib6_rules_require_fldissect &&
            fib_rule_requires_fldissect(rule))
                net->ipv6.fib6_rules_require_fldissect--;

        return 0;
}

static int fib6_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh,
                             struct nlattr **tb)
{
        struct fib6_rule *rule6 = (struct fib6_rule *) rule;

        if (frh->src_len && (rule6->src.plen != frh->src_len))
                return 0;

        if (frh->dst_len && (rule6->dst.plen != frh->dst_len))
                return 0;

        if (frh->tos &&
            (rule6->dscp_full ||
             inet_dscp_to_dsfield(rule6->dscp) != frh->tos))
                return 0;

        if (tb[FRA_DSCP]) {
                dscp_t dscp;

                dscp = inet_dsfield_to_dscp(nla_get_u8(tb[FRA_DSCP]) << 2);
                if (!rule6->dscp_full || rule6->dscp != dscp)
                        return 0;
        }

        if (tb[FRA_FLOWLABEL] &&
            nla_get_be32(tb[FRA_FLOWLABEL]) != rule6->flowlabel)
                return 0;

        if (tb[FRA_FLOWLABEL_MASK] &&
            nla_get_be32(tb[FRA_FLOWLABEL_MASK]) != rule6->flowlabel_mask)
                return 0;

        if (frh->src_len &&
            nla_memcmp(tb[FRA_SRC], &rule6->src.addr, sizeof(struct in6_addr)))
                return 0;

        if (frh->dst_len &&
            nla_memcmp(tb[FRA_DST], &rule6->dst.addr, sizeof(struct in6_addr)))
                return 0;

        return 1;
}

static int fib6_rule_fill(struct fib_rule *rule, struct sk_buff *skb,
                          struct fib_rule_hdr *frh)
{
        struct fib6_rule *rule6 = (struct fib6_rule *) rule;

        frh->dst_len = rule6->dst.plen;
        frh->src_len = rule6->src.plen;

        if (rule6->dscp_full) {
                frh->tos = 0;
                if (nla_put_u8(skb, FRA_DSCP,
                               inet_dscp_to_dsfield(rule6->dscp) >> 2))
                        goto nla_put_failure;
        } else {
                frh->tos = inet_dscp_to_dsfield(rule6->dscp);
        }

        if (rule6->flowlabel_mask &&
            (nla_put_be32(skb, FRA_FLOWLABEL, rule6->flowlabel) ||
             nla_put_be32(skb, FRA_FLOWLABEL_MASK, rule6->flowlabel_mask)))
                goto nla_put_failure;

        if ((rule6->dst.plen &&
             nla_put_in6_addr(skb, FRA_DST, &rule6->dst.addr)) ||
            (rule6->src.plen &&
             nla_put_in6_addr(skb, FRA_SRC, &rule6->src.addr)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -ENOBUFS;
}

static size_t fib6_rule_nlmsg_payload(struct fib_rule *rule)
{
        return nla_total_size(16) /* dst */
               + nla_total_size(16) /* src */
               + nla_total_size(1) /* dscp */
               + nla_total_size(4) /* flowlabel */
               + nla_total_size(4); /* flowlabel mask */
}

static void fib6_rule_flush_cache(struct fib_rules_ops *ops)
{
        rt_genid_bump_ipv6(ops->fro_net);
}

static const struct fib_rules_ops __net_initconst fib6_rules_ops_template = {
        .family                        = AF_INET6,
        .rule_size                = sizeof(struct fib6_rule),
        .addr_size                = sizeof(struct in6_addr),
        .action                        = fib6_rule_action,
        .match                        = fib6_rule_match,
        .suppress                = fib6_rule_suppress,
        .configure                = fib6_rule_configure,
        .delete                        = fib6_rule_delete,
        .compare                = fib6_rule_compare,
        .fill                        = fib6_rule_fill,
        .nlmsg_payload                = fib6_rule_nlmsg_payload,
        .flush_cache                = fib6_rule_flush_cache,
        .nlgroup                = RTNLGRP_IPV6_RULE,
        .owner                        = THIS_MODULE,
        .fro_net                = &init_net,
};

static int __net_init fib6_rules_net_init(struct net *net)
{
        struct fib_rules_ops *ops;
        int err;

        ops = fib_rules_register(&fib6_rules_ops_template, net);
        if (IS_ERR(ops))
                return PTR_ERR(ops);

        err = fib_default_rule_add(ops, 0, RT6_TABLE_LOCAL);
        if (err)
                goto out_fib6_rules_ops;

        err = fib_default_rule_add(ops, 0x7FFE, RT6_TABLE_MAIN);
        if (err)
                goto out_fib6_rules_ops;

        net->ipv6.fib6_rules_ops = ops;
        net->ipv6.fib6_rules_require_fldissect = 0;
out:
        return err;

out_fib6_rules_ops:
        fib_rules_unregister(ops);
        goto out;
}

static void __net_exit fib6_rules_net_exit_batch(struct list_head *net_list)
{
        struct net *net;

        rtnl_lock();
        list_for_each_entry(net, net_list, exit_list) {
                fib_rules_unregister(net->ipv6.fib6_rules_ops);
                cond_resched();
        }
        rtnl_unlock();
}

static struct pernet_operations fib6_rules_net_ops = {
        .init = fib6_rules_net_init,
        .exit_batch = fib6_rules_net_exit_batch,
};

int __init fib6_rules_init(void)
{
        return register_pernet_subsys(&fib6_rules_net_ops);
}


void fib6_rules_cleanup(void)
{
        unregister_pernet_subsys(&fib6_rules_net_ops);
}






















































































































































    2 




    2 




















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
 */

#include <linux/types.h>
#include <linux/bpf.h>
#include <linux/bpf_local_storage.h>
#include <uapi/linux/btf.h>
#include <linux/btf_ids.h>

DEFINE_BPF_STORAGE_CACHE(cgroup_cache);

static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy);

static void bpf_cgrp_storage_lock(void)
{
        cant_migrate();
        this_cpu_inc(bpf_cgrp_storage_busy);
}

static void bpf_cgrp_storage_unlock(void)
{
        this_cpu_dec(bpf_cgrp_storage_busy);
}

static bool bpf_cgrp_storage_trylock(void)
{
        cant_migrate();
        if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) {
                this_cpu_dec(bpf_cgrp_storage_busy);
                return false;
        }
        return true;
}

static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
{
        struct cgroup *cg = owner;

        return &cg->bpf_cgrp_storage;
}

void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
        struct bpf_local_storage *local_storage;

        migrate_disable();
        rcu_read_lock();
        local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
        if (!local_storage)
                goto out;

        bpf_cgrp_storage_lock();
        bpf_local_storage_destroy(local_storage);
        bpf_cgrp_storage_unlock();
out:
        rcu_read_unlock();
        migrate_enable();
}

static struct bpf_local_storage_data *
cgroup_storage_lookup(struct cgroup *cgroup, struct bpf_map *map, bool cacheit_lockit)
{
        struct bpf_local_storage *cgroup_storage;
        struct bpf_local_storage_map *smap;

        cgroup_storage = rcu_dereference_check(cgroup->bpf_cgrp_storage,
                                               bpf_rcu_lock_held());
        if (!cgroup_storage)
                return NULL;

        smap = (struct bpf_local_storage_map *)map;
        return bpf_local_storage_lookup(cgroup_storage, smap, cacheit_lockit);
}

static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key)
{
        struct bpf_local_storage_data *sdata;
        struct cgroup *cgroup;
        int fd;

        fd = *(int *)key;
        cgroup = cgroup_v1v2_get_from_fd(fd);
        if (IS_ERR(cgroup))
                return ERR_CAST(cgroup);

        bpf_cgrp_storage_lock();
        sdata = cgroup_storage_lookup(cgroup, map, true);
        bpf_cgrp_storage_unlock();
        cgroup_put(cgroup);
        return sdata ? sdata->data : NULL;
}

static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
                                         void *value, u64 map_flags)
{
        struct bpf_local_storage_data *sdata;
        struct cgroup *cgroup;
        int fd;

        fd = *(int *)key;
        cgroup = cgroup_v1v2_get_from_fd(fd);
        if (IS_ERR(cgroup))
                return PTR_ERR(cgroup);

        bpf_cgrp_storage_lock();
        sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
                                         value, map_flags, false, GFP_ATOMIC);
        bpf_cgrp_storage_unlock();
        cgroup_put(cgroup);
        return PTR_ERR_OR_ZERO(sdata);
}

static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
{
        struct bpf_local_storage_data *sdata;

        sdata = cgroup_storage_lookup(cgroup, map, false);
        if (!sdata)
                return -ENOENT;

        bpf_selem_unlink(SELEM(sdata), false);
        return 0;
}

static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
{
        struct cgroup *cgroup;
        int err, fd;

        fd = *(int *)key;
        cgroup = cgroup_v1v2_get_from_fd(fd);
        if (IS_ERR(cgroup))
                return PTR_ERR(cgroup);

        bpf_cgrp_storage_lock();
        err = cgroup_storage_delete(cgroup, map);
        bpf_cgrp_storage_unlock();
        cgroup_put(cgroup);
        return err;
}

static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
        return -ENOTSUPP;
}

static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
{
        return bpf_local_storage_map_alloc(attr, &cgroup_cache, true);
}

static void cgroup_storage_map_free(struct bpf_map *map)
{
        bpf_local_storage_map_free(map, &cgroup_cache, NULL);
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
           void *, value, u64, flags, gfp_t, gfp_flags)
{
        struct bpf_local_storage_data *sdata;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
                return (unsigned long)NULL;

        if (!cgroup)
                return (unsigned long)NULL;

        if (!bpf_cgrp_storage_trylock())
                return (unsigned long)NULL;

        sdata = cgroup_storage_lookup(cgroup, map, true);
        if (sdata)
                goto unlock;

        /* only allocate new storage, when the cgroup is refcounted */
        if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
            (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
                sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
                                                 value, BPF_NOEXIST, false, gfp_flags);

unlock:
        bpf_cgrp_storage_unlock();
        return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
}

BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup)
{
        int ret;

        WARN_ON_ONCE(!bpf_rcu_lock_held());
        if (!cgroup)
                return -EINVAL;

        if (!bpf_cgrp_storage_trylock())
                return -EBUSY;

        ret = cgroup_storage_delete(cgroup, map);
        bpf_cgrp_storage_unlock();
        return ret;
}

const struct bpf_map_ops cgrp_storage_map_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = bpf_local_storage_map_alloc_check,
        .map_alloc = cgroup_storage_map_alloc,
        .map_free = cgroup_storage_map_free,
        .map_get_next_key = notsupp_get_next_key,
        .map_lookup_elem = bpf_cgrp_storage_lookup_elem,
        .map_update_elem = bpf_cgrp_storage_update_elem,
        .map_delete_elem = bpf_cgrp_storage_delete_elem,
        .map_check_btf = bpf_local_storage_map_check_btf,
        .map_mem_usage = bpf_local_storage_map_mem_usage,
        .map_btf_id = &bpf_local_storage_map_btf_id[0],
        .map_owner_storage_ptr = cgroup_storage_ptr,
};

const struct bpf_func_proto bpf_cgrp_storage_get_proto = {
        .func                = bpf_cgrp_storage_get,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_MAP_VALUE_OR_NULL,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id        = &bpf_cgroup_btf_id[0],
        .arg3_type        = ARG_PTR_TO_MAP_VALUE_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};

const struct bpf_func_proto bpf_cgrp_storage_delete_proto = {
        .func                = bpf_cgrp_storage_delete,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_BTF_ID_OR_NULL,
        .arg2_btf_id        = &bpf_cgroup_btf_id[0],
};











































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 



    5 


    5 





    5 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
// SPDX-License-Identifier: GPL-2.0+
/*
 * Driver for Realtek RTS51xx USB card reader
 *
 * Copyright(c) 2009 Realtek Semiconductor Corp. All rights reserved.
 *
 * Author:
 *   wwang (wei_wang@realsil.com.cn)
 *   No. 450, Shenhu Road, Suzhou Industry Park, Suzhou, China
 */

#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/kernel.h>

#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <linux/cdrom.h>

#include <linux/usb.h>
#include <linux/slab.h>
#include <linux/usb_usual.h>

#include "usb.h"
#include "transport.h"
#include "protocol.h"
#include "debug.h"
#include "scsiglue.h"

#define DRV_NAME "ums-realtek"

MODULE_DESCRIPTION("Driver for Realtek USB Card Reader");
MODULE_AUTHOR("wwang <wei_wang@realsil.com.cn>");
MODULE_LICENSE("GPL");
MODULE_IMPORT_NS("USB_STORAGE");

static int auto_delink_en = 1;
module_param(auto_delink_en, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(auto_delink_en, "auto delink mode (0=firmware, 1=software [default])");

#ifdef CONFIG_REALTEK_AUTOPM
static int ss_en = 1;
module_param(ss_en, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(ss_en, "enable selective suspend");

static int ss_delay = 50;
module_param(ss_delay, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(ss_delay,
                 "seconds to delay before entering selective suspend");

enum RTS51X_STAT {
        RTS51X_STAT_INIT,
        RTS51X_STAT_IDLE,
        RTS51X_STAT_RUN,
        RTS51X_STAT_SS
};

#define POLLING_INTERVAL        50

#define rts51x_set_stat(chip, stat)        \
        ((chip)->state = (enum RTS51X_STAT)(stat))
#define rts51x_get_stat(chip)                ((chip)->state)

#define SET_LUN_READY(chip, lun)        ((chip)->lun_ready |= ((u8)1 << (lun)))
#define CLR_LUN_READY(chip, lun)        ((chip)->lun_ready &= ~((u8)1 << (lun)))
#define TST_LUN_READY(chip, lun)        ((chip)->lun_ready & ((u8)1 << (lun)))

#endif

struct rts51x_status {
        u16 vid;
        u16 pid;
        u8 cur_lun;
        u8 card_type;
        u8 total_lun;
        u16 fw_ver;
        u8 phy_exist;
        u8 multi_flag;
        u8 multi_card;
        u8 log_exist;
        union {
                u8 detailed_type1;
                u8 detailed_type2;
        } detailed_type;
        u8 function[2];
};

struct rts51x_chip {
        u16 vendor_id;
        u16 product_id;
        char max_lun;

        struct rts51x_status *status;
        int status_len;

        u32 flag;
        struct us_data *us;

#ifdef CONFIG_REALTEK_AUTOPM
        struct timer_list rts51x_suspend_timer;
        unsigned long timer_expires;
        int pwr_state;
        u8 lun_ready;
        enum RTS51X_STAT state;
        int support_auto_delink;
#endif
        /* used to back up the protocol chosen in probe1 phase */
        proto_cmnd proto_handler_backup;
};

/* flag definition */
#define FLIDX_AUTO_DELINK                0x01

#define SCSI_LUN(srb)                        ((srb)->device->lun)

/* Bit Operation */
#define SET_BIT(data, idx)                ((data) |= 1 << (idx))
#define CLR_BIT(data, idx)                ((data) &= ~(1 << (idx)))
#define CHK_BIT(data, idx)                ((data) & (1 << (idx)))

#define SET_AUTO_DELINK(chip)                ((chip)->flag |= FLIDX_AUTO_DELINK)
#define CLR_AUTO_DELINK(chip)                ((chip)->flag &= ~FLIDX_AUTO_DELINK)
#define CHK_AUTO_DELINK(chip)                ((chip)->flag & FLIDX_AUTO_DELINK)

#define RTS51X_GET_VID(chip)                ((chip)->vendor_id)
#define RTS51X_GET_PID(chip)                ((chip)->product_id)

#define VENDOR_ID(chip)                        ((chip)->status[0].vid)
#define PRODUCT_ID(chip)                ((chip)->status[0].pid)
#define FW_VERSION(chip)                ((chip)->status[0].fw_ver)
#define STATUS_LEN(chip)                ((chip)->status_len)

#define STATUS_SUCCESS                0
#define STATUS_FAIL                1

/* Check card reader function */
#define SUPPORT_DETAILED_TYPE1(chip)        \
                CHK_BIT((chip)->status[0].function[0], 1)
#define SUPPORT_OT(chip)                \
                CHK_BIT((chip)->status[0].function[0], 2)
#define SUPPORT_OC(chip)                \
                CHK_BIT((chip)->status[0].function[0], 3)
#define SUPPORT_AUTO_DELINK(chip)        \
                CHK_BIT((chip)->status[0].function[0], 4)
#define SUPPORT_SDIO(chip)                \
                CHK_BIT((chip)->status[0].function[1], 0)
#define SUPPORT_DETAILED_TYPE2(chip)        \
                CHK_BIT((chip)->status[0].function[1], 1)

#define CHECK_PID(chip, pid)                (RTS51X_GET_PID(chip) == (pid))
#define CHECK_FW_VER(chip, fw_ver)        (FW_VERSION(chip) == (fw_ver))
#define CHECK_ID(chip, pid, fw_ver)        \
                (CHECK_PID((chip), (pid)) && CHECK_FW_VER((chip), (fw_ver)))

static int init_realtek_cr(struct us_data *us);

/*
 * The table of devices
 */
#define UNUSUAL_DEV(id_vendor, id_product, bcdDeviceMin, bcdDeviceMax, \
                    vendorName, productName, useProtocol, useTransport, \
                    initFunction, flags) \
{\
        USB_DEVICE_VER(id_vendor, id_product, bcdDeviceMin, bcdDeviceMax), \
        .driver_info = (flags) \
}

static const struct usb_device_id realtek_cr_ids[] = {
#        include "unusual_realtek.h"
        {}                        /* Terminating entry */
};

MODULE_DEVICE_TABLE(usb, realtek_cr_ids);

#undef UNUSUAL_DEV

/*
 * The flags table
 */
#define UNUSUAL_DEV(idVendor, idProduct, bcdDeviceMin, bcdDeviceMax, \
                    vendor_name, product_name, use_protocol, use_transport, \
                    init_function, Flags) \
{ \
        .vendorName = vendor_name,        \
        .productName = product_name,        \
        .useProtocol = use_protocol,        \
        .useTransport = use_transport,        \
        .initFunction = init_function,        \
}

static struct us_unusual_dev realtek_cr_unusual_dev_list[] = {
#        include "unusual_realtek.h"
        {}                        /* Terminating entry */
};

#undef UNUSUAL_DEV

static int rts51x_bulk_transport(struct us_data *us, u8 lun,
                                 u8 *cmd, int cmd_len, u8 *buf, int buf_len,
                                 enum dma_data_direction dir, int *act_len)
{
        struct bulk_cb_wrap *bcb = (struct bulk_cb_wrap *)us->iobuf;
        struct bulk_cs_wrap *bcs = (struct bulk_cs_wrap *)us->iobuf;
        int result;
        unsigned int residue;
        unsigned int cswlen;
        unsigned int cbwlen = US_BULK_CB_WRAP_LEN;

        /* set up the command wrapper */
        bcb->Signature = cpu_to_le32(US_BULK_CB_SIGN);
        bcb->DataTransferLength = cpu_to_le32(buf_len);
        bcb->Flags = (dir == DMA_FROM_DEVICE) ? US_BULK_FLAG_IN : US_BULK_FLAG_OUT;
        bcb->Tag = ++us->tag;
        bcb->Lun = lun;
        bcb->Length = cmd_len;

        /* copy the command payload */
        memset(bcb->CDB, 0, sizeof(bcb->CDB));
        memcpy(bcb->CDB, cmd, bcb->Length);

        /* send it to out endpoint */
        result = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
                                            bcb, cbwlen, NULL);
        if (result != USB_STOR_XFER_GOOD)
                return USB_STOR_TRANSPORT_ERROR;

        /* DATA STAGE */
        /* send/receive data payload, if there is any */

        if (buf && buf_len) {
                unsigned int pipe = (dir == DMA_FROM_DEVICE) ?
                    us->recv_bulk_pipe : us->send_bulk_pipe;
                result = usb_stor_bulk_transfer_buf(us, pipe,
                                                    buf, buf_len, NULL);
                if (result == USB_STOR_XFER_ERROR)
                        return USB_STOR_TRANSPORT_ERROR;
        }

        /* get CSW for device status */
        result = usb_stor_bulk_transfer_buf(us, us->recv_bulk_pipe,
                                            bcs, US_BULK_CS_WRAP_LEN, &cswlen);
        if (result != USB_STOR_XFER_GOOD)
                return USB_STOR_TRANSPORT_ERROR;

        /* check bulk status */
        if (bcs->Signature != cpu_to_le32(US_BULK_CS_SIGN)) {
                usb_stor_dbg(us, "Signature mismatch: got %08X, expecting %08X\n",
                             le32_to_cpu(bcs->Signature), US_BULK_CS_SIGN);
                return USB_STOR_TRANSPORT_ERROR;
        }

        residue = bcs->Residue;
        if (bcs->Tag != us->tag)
                return USB_STOR_TRANSPORT_ERROR;

        /*
         * try to compute the actual residue, based on how much data
         * was really transferred and what the device tells us
         */
        if (residue)
                residue = residue < buf_len ? residue : buf_len;

        if (act_len)
                *act_len = buf_len - residue;

        /* based on the status code, we report good or bad */
        switch (bcs->Status) {
        case US_BULK_STAT_OK:
                /* command good -- note that data could be short */
                return USB_STOR_TRANSPORT_GOOD;

        case US_BULK_STAT_FAIL:
                /* command failed */
                return USB_STOR_TRANSPORT_FAILED;

        case US_BULK_STAT_PHASE:
                /*
                 * phase error -- note that a transport reset will be
                 * invoked by the invoke_transport() function
                 */
                return USB_STOR_TRANSPORT_ERROR;
        }

        /* we should never get here, but if we do, we're in trouble */
        return USB_STOR_TRANSPORT_ERROR;
}

static int rts51x_bulk_transport_special(struct us_data *us, u8 lun,
                                 u8 *cmd, int cmd_len, u8 *buf, int buf_len,
                                 enum dma_data_direction dir, int *act_len)
{
        struct bulk_cb_wrap *bcb = (struct bulk_cb_wrap *) us->iobuf;
        struct bulk_cs_wrap *bcs = (struct bulk_cs_wrap *) us->iobuf;
        int result;
        unsigned int cswlen;
        unsigned int cbwlen = US_BULK_CB_WRAP_LEN;

        /* set up the command wrapper */
        bcb->Signature = cpu_to_le32(US_BULK_CB_SIGN);
        bcb->DataTransferLength = cpu_to_le32(buf_len);
        bcb->Flags = (dir == DMA_FROM_DEVICE) ? US_BULK_FLAG_IN : US_BULK_FLAG_OUT;
        bcb->Tag = ++us->tag;
        bcb->Lun = lun;
        bcb->Length = cmd_len;

        /* copy the command payload */
        memset(bcb->CDB, 0, sizeof(bcb->CDB));
        memcpy(bcb->CDB, cmd, bcb->Length);

        /* send it to out endpoint */
        result = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
                                bcb, cbwlen, NULL);
        if (result != USB_STOR_XFER_GOOD)
                return USB_STOR_TRANSPORT_ERROR;

        /* DATA STAGE */
        /* send/receive data payload, if there is any */

        if (buf && buf_len) {
                unsigned int pipe = (dir == DMA_FROM_DEVICE) ?
                                us->recv_bulk_pipe : us->send_bulk_pipe;
                result = usb_stor_bulk_transfer_buf(us, pipe,
                                buf, buf_len, NULL);
                if (result == USB_STOR_XFER_ERROR)
                        return USB_STOR_TRANSPORT_ERROR;
        }

        /* get CSW for device status */
        result = usb_bulk_msg(us->pusb_dev, us->recv_bulk_pipe, bcs,
                        US_BULK_CS_WRAP_LEN, &cswlen, 250);
        return result;
}

/* Determine what the maximum LUN supported is */
static int rts51x_get_max_lun(struct us_data *us)
{
        int result;

        /* issue the command */
        us->iobuf[0] = 0;
        result = usb_stor_control_msg(us, us->recv_ctrl_pipe,
                                      US_BULK_GET_MAX_LUN,
                                      USB_DIR_IN | USB_TYPE_CLASS |
                                      USB_RECIP_INTERFACE,
                                      0, us->ifnum, us->iobuf, 1, 10 * HZ);

        usb_stor_dbg(us, "GetMaxLUN command result is %d, data is %d\n",
                     result, us->iobuf[0]);

        /* if we have a successful request, return the result */
        if (result > 0)
                return us->iobuf[0];

        return 0;
}

static int rts51x_read_mem(struct us_data *us, u16 addr, u8 *data, u16 len)
{
        int retval;
        u8 cmnd[12] = { 0 };
        u8 *buf;

        buf = kmalloc(len, GFP_NOIO);
        if (buf == NULL)
                return -ENOMEM;

        usb_stor_dbg(us, "addr = 0x%x, len = %d\n", addr, len);

        cmnd[0] = 0xF0;
        cmnd[1] = 0x0D;
        cmnd[2] = (u8) (addr >> 8);
        cmnd[3] = (u8) addr;
        cmnd[4] = (u8) (len >> 8);
        cmnd[5] = (u8) len;

        retval = rts51x_bulk_transport(us, 0, cmnd, 12,
                                       buf, len, DMA_FROM_DEVICE, NULL);
        if (retval != USB_STOR_TRANSPORT_GOOD) {
                kfree(buf);
                return -EIO;
        }

        memcpy(data, buf, len);
        kfree(buf);
        return 0;
}

static int rts51x_write_mem(struct us_data *us, u16 addr, u8 *data, u16 len)
{
        int retval;
        u8 cmnd[12] = { 0 };
        u8 *buf;

        buf = kmemdup(data, len, GFP_NOIO);
        if (buf == NULL)
                return USB_STOR_TRANSPORT_ERROR;

        usb_stor_dbg(us, "addr = 0x%x, len = %d\n", addr, len);

        cmnd[0] = 0xF0;
        cmnd[1] = 0x0E;
        cmnd[2] = (u8) (addr >> 8);
        cmnd[3] = (u8) addr;
        cmnd[4] = (u8) (len >> 8);
        cmnd[5] = (u8) len;

        retval = rts51x_bulk_transport(us, 0, cmnd, 12,
                                       buf, len, DMA_TO_DEVICE, NULL);
        kfree(buf);
        if (retval != USB_STOR_TRANSPORT_GOOD)
                return -EIO;

        return 0;
}

static int rts51x_read_status(struct us_data *us,
                              u8 lun, u8 *status, int len, int *actlen)
{
        int retval;
        u8 cmnd[12] = { 0 };
        u8 *buf;

        buf = kmalloc(len, GFP_NOIO);
        if (buf == NULL)
                return USB_STOR_TRANSPORT_ERROR;

        usb_stor_dbg(us, "lun = %d\n", lun);

        cmnd[0] = 0xF0;
        cmnd[1] = 0x09;

        retval = rts51x_bulk_transport(us, lun, cmnd, 12,
                                       buf, len, DMA_FROM_DEVICE, actlen);
        if (retval != USB_STOR_TRANSPORT_GOOD) {
                kfree(buf);
                return -EIO;
        }

        memcpy(status, buf, len);
        kfree(buf);
        return 0;
}

static int rts51x_check_status(struct us_data *us, u8 lun)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        int retval;
        u8 buf[16];

        retval = rts51x_read_status(us, lun, buf, 16, &(chip->status_len));
        if (retval != STATUS_SUCCESS)
                return -EIO;

        usb_stor_dbg(us, "chip->status_len = %d\n", chip->status_len);

        chip->status[lun].vid = ((u16) buf[0] << 8) | buf[1];
        chip->status[lun].pid = ((u16) buf[2] << 8) | buf[3];
        chip->status[lun].cur_lun = buf[4];
        chip->status[lun].card_type = buf[5];
        chip->status[lun].total_lun = buf[6];
        chip->status[lun].fw_ver = ((u16) buf[7] << 8) | buf[8];
        chip->status[lun].phy_exist = buf[9];
        chip->status[lun].multi_flag = buf[10];
        chip->status[lun].multi_card = buf[11];
        chip->status[lun].log_exist = buf[12];
        if (chip->status_len == 16) {
                chip->status[lun].detailed_type.detailed_type1 = buf[13];
                chip->status[lun].function[0] = buf[14];
                chip->status[lun].function[1] = buf[15];
        }

        return 0;
}

static int enable_oscillator(struct us_data *us)
{
        int retval;
        u8 value;

        retval = rts51x_read_mem(us, 0xFE77, &value, 1);
        if (retval < 0)
                return -EIO;

        value |= 0x04;
        retval = rts51x_write_mem(us, 0xFE77, &value, 1);
        if (retval < 0)
                return -EIO;

        retval = rts51x_read_mem(us, 0xFE77, &value, 1);
        if (retval < 0)
                return -EIO;

        if (!(value & 0x04))
                return -EIO;

        return 0;
}

static int __do_config_autodelink(struct us_data *us, u8 *data, u16 len)
{
        int retval;
        u8 cmnd[12] = {0};
        u8 *buf;

        usb_stor_dbg(us, "addr = 0xfe47, len = %d\n", len);

        buf = kmemdup(data, len, GFP_NOIO);
        if (!buf)
                return USB_STOR_TRANSPORT_ERROR;

        cmnd[0] = 0xF0;
        cmnd[1] = 0x0E;
        cmnd[2] = 0xfe;
        cmnd[3] = 0x47;
        cmnd[4] = (u8)(len >> 8);
        cmnd[5] = (u8)len;

        retval = rts51x_bulk_transport_special(us, 0, cmnd, 12, buf, len, DMA_TO_DEVICE, NULL);
        kfree(buf);
        if (retval != USB_STOR_TRANSPORT_GOOD) {
                return -EIO;
        }

        return 0;
}

static int do_config_autodelink(struct us_data *us, int enable, int force)
{
        int retval;
        u8 value;

        retval = rts51x_read_mem(us, 0xFE47, &value, 1);
        if (retval < 0)
                return -EIO;

        if (enable) {
                if (force)
                        value |= 0x03;
                else
                        value |= 0x01;
        } else {
                value &= ~0x03;
        }

        usb_stor_dbg(us, "set 0xfe47 to 0x%x\n", value);

        /* retval = rts51x_write_mem(us, 0xFE47, &value, 1); */
        retval = __do_config_autodelink(us, &value, 1);
        if (retval < 0)
                return -EIO;

        return 0;
}

static int config_autodelink_after_power_on(struct us_data *us)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        int retval;
        u8 value;

        if (!CHK_AUTO_DELINK(chip))
                return 0;

        retval = rts51x_read_mem(us, 0xFE47, &value, 1);
        if (retval < 0)
                return -EIO;

        if (auto_delink_en) {
                CLR_BIT(value, 0);
                CLR_BIT(value, 1);
                SET_BIT(value, 2);

                if (CHECK_ID(chip, 0x0138, 0x3882))
                        CLR_BIT(value, 2);

                SET_BIT(value, 7);

                /* retval = rts51x_write_mem(us, 0xFE47, &value, 1); */
                retval = __do_config_autodelink(us, &value, 1);
                if (retval < 0)
                        return -EIO;

                retval = enable_oscillator(us);
                if (retval == 0)
                        (void)do_config_autodelink(us, 1, 0);
        } else {
                /* Autodelink controlled by firmware */

                SET_BIT(value, 2);

                if (CHECK_ID(chip, 0x0138, 0x3882))
                        CLR_BIT(value, 2);

                if (CHECK_ID(chip, 0x0159, 0x5889) ||
                    CHECK_ID(chip, 0x0138, 0x3880)) {
                        CLR_BIT(value, 0);
                        CLR_BIT(value, 7);
                }

                /* retval = rts51x_write_mem(us, 0xFE47, &value, 1); */
                retval = __do_config_autodelink(us, &value, 1);
                if (retval < 0)
                        return -EIO;

                if (CHECK_ID(chip, 0x0159, 0x5888)) {
                        value = 0xFF;
                        retval = rts51x_write_mem(us, 0xFE79, &value, 1);
                        if (retval < 0)
                                return -EIO;

                        value = 0x01;
                        retval = rts51x_write_mem(us, 0x48, &value, 1);
                        if (retval < 0)
                                return -EIO;
                }
        }

        return 0;
}

#ifdef CONFIG_PM
static int config_autodelink_before_power_down(struct us_data *us)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        int retval;
        u8 value;

        if (!CHK_AUTO_DELINK(chip))
                return 0;

        if (auto_delink_en) {
                retval = rts51x_read_mem(us, 0xFE77, &value, 1);
                if (retval < 0)
                        return -EIO;

                SET_BIT(value, 2);
                retval = rts51x_write_mem(us, 0xFE77, &value, 1);
                if (retval < 0)
                        return -EIO;

                if (CHECK_ID(chip, 0x0159, 0x5888)) {
                        value = 0x01;
                        retval = rts51x_write_mem(us, 0x48, &value, 1);
                        if (retval < 0)
                                return -EIO;
                }

                retval = rts51x_read_mem(us, 0xFE47, &value, 1);
                if (retval < 0)
                        return -EIO;

                SET_BIT(value, 0);
                if (CHECK_ID(chip, 0x0138, 0x3882))
                        SET_BIT(value, 2);
                retval = rts51x_write_mem(us, 0xFE77, &value, 1);
                if (retval < 0)
                        return -EIO;
        } else {
                if (CHECK_ID(chip, 0x0159, 0x5889) ||
                    CHECK_ID(chip, 0x0138, 0x3880) ||
                    CHECK_ID(chip, 0x0138, 0x3882)) {
                        retval = rts51x_read_mem(us, 0xFE47, &value, 1);
                        if (retval < 0)
                                return -EIO;

                        if (CHECK_ID(chip, 0x0159, 0x5889) ||
                            CHECK_ID(chip, 0x0138, 0x3880)) {
                                SET_BIT(value, 0);
                                SET_BIT(value, 7);
                        }

                        if (CHECK_ID(chip, 0x0138, 0x3882))
                                SET_BIT(value, 2);

                        /* retval = rts51x_write_mem(us, 0xFE47, &value, 1); */
                        retval = __do_config_autodelink(us, &value, 1);
                        if (retval < 0)
                                return -EIO;
                }

                if (CHECK_ID(chip, 0x0159, 0x5888)) {
                        value = 0x01;
                        retval = rts51x_write_mem(us, 0x48, &value, 1);
                        if (retval < 0)
                                return -EIO;
                }
        }

        return 0;
}

static void fw5895_init(struct us_data *us)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        int retval;
        u8 val;

        if ((PRODUCT_ID(chip) != 0x0158) || (FW_VERSION(chip) != 0x5895)) {
                usb_stor_dbg(us, "Not the specified device, return immediately!\n");
        } else {
                retval = rts51x_read_mem(us, 0xFD6F, &val, 1);
                if (retval == STATUS_SUCCESS && (val & 0x1F) == 0) {
                        val = 0x1F;
                        retval = rts51x_write_mem(us, 0xFD70, &val, 1);
                        if (retval != STATUS_SUCCESS)
                                usb_stor_dbg(us, "Write memory fail\n");
                } else {
                        usb_stor_dbg(us, "Read memory fail, OR (val & 0x1F) != 0\n");
                }
        }
}
#endif

#ifdef CONFIG_REALTEK_AUTOPM
static void fw5895_set_mmc_wp(struct us_data *us)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        int retval;
        u8 buf[13];

        if ((PRODUCT_ID(chip) != 0x0158) || (FW_VERSION(chip) != 0x5895)) {
                usb_stor_dbg(us, "Not the specified device, return immediately!\n");
        } else {
                retval = rts51x_read_mem(us, 0xFD6F, buf, 1);
                if (retval == STATUS_SUCCESS && (buf[0] & 0x24) == 0x24) {
                        /* SD Exist and SD WP */
                        retval = rts51x_read_mem(us, 0xD04E, buf, 1);
                        if (retval == STATUS_SUCCESS) {
                                buf[0] |= 0x04;
                                retval = rts51x_write_mem(us, 0xFD70, buf, 1);
                                if (retval != STATUS_SUCCESS)
                                        usb_stor_dbg(us, "Write memory fail\n");
                        } else {
                                usb_stor_dbg(us, "Read memory fail\n");
                        }
                } else {
                        usb_stor_dbg(us, "Read memory fail, OR (buf[0]&0x24)!=0x24\n");
                }
        }
}

static void rts51x_modi_suspend_timer(struct rts51x_chip *chip)
{
        struct us_data *us = chip->us;

        usb_stor_dbg(us, "state:%d\n", rts51x_get_stat(chip));

        chip->timer_expires = jiffies + msecs_to_jiffies(1000*ss_delay);
        mod_timer(&chip->rts51x_suspend_timer, chip->timer_expires);
}

static void rts51x_suspend_timer_fn(struct timer_list *t)
{
        struct rts51x_chip *chip = from_timer(chip, t, rts51x_suspend_timer);
        struct us_data *us = chip->us;

        switch (rts51x_get_stat(chip)) {
        case RTS51X_STAT_INIT:
        case RTS51X_STAT_RUN:
                rts51x_modi_suspend_timer(chip);
                break;
        case RTS51X_STAT_IDLE:
        case RTS51X_STAT_SS:
                usb_stor_dbg(us, "RTS51X_STAT_SS, power.usage:%d\n",
                             atomic_read(&us->pusb_intf->dev.power.usage_count));

                if (atomic_read(&us->pusb_intf->dev.power.usage_count) > 0) {
                        usb_stor_dbg(us, "Ready to enter SS state\n");
                        rts51x_set_stat(chip, RTS51X_STAT_SS);
                        /* ignore mass storage interface's children */
                        pm_suspend_ignore_children(&us->pusb_intf->dev, true);
                        usb_autopm_put_interface_async(us->pusb_intf);
                        usb_stor_dbg(us, "RTS51X_STAT_SS 01, power.usage:%d\n",
                                     atomic_read(&us->pusb_intf->dev.power.usage_count));
                }
                break;
        default:
                usb_stor_dbg(us, "Unknown state !!!\n");
                break;
        }
}

static inline int working_scsi(struct scsi_cmnd *srb)
{
        if ((srb->cmnd[0] == TEST_UNIT_READY) ||
            (srb->cmnd[0] == ALLOW_MEDIUM_REMOVAL)) {
                return 0;
        }

        return 1;
}

static void rts51x_invoke_transport(struct scsi_cmnd *srb, struct us_data *us)
{
        struct rts51x_chip *chip = (struct rts51x_chip *)(us->extra);
        static int card_first_show = 1;
        static u8 media_not_present[] = { 0x70, 0, 0x02, 0, 0, 0, 0,
                10, 0, 0, 0, 0, 0x3A, 0, 0, 0, 0, 0
        };
        static u8 invalid_cmd_field[] = { 0x70, 0, 0x05, 0, 0, 0, 0,
                10, 0, 0, 0, 0, 0x24, 0, 0, 0, 0, 0
        };
        int ret;

        if (working_scsi(srb)) {
                usb_stor_dbg(us, "working scsi, power.usage:%d\n",
                             atomic_read(&us->pusb_intf->dev.power.usage_count));

                if (atomic_read(&us->pusb_intf->dev.power.usage_count) <= 0) {
                        ret = usb_autopm_get_interface(us->pusb_intf);
                        usb_stor_dbg(us, "working scsi, ret=%d\n", ret);
                }
                if (rts51x_get_stat(chip) != RTS51X_STAT_RUN)
                        rts51x_set_stat(chip, RTS51X_STAT_RUN);
                chip->proto_handler_backup(srb, us);
        } else {
                if (rts51x_get_stat(chip) == RTS51X_STAT_SS) {
                        usb_stor_dbg(us, "NOT working scsi\n");
                        if ((srb->cmnd[0] == TEST_UNIT_READY) &&
                            (chip->pwr_state == US_SUSPEND)) {
                                if (TST_LUN_READY(chip, srb->device->lun)) {
                                        srb->result = SAM_STAT_GOOD;
                                } else {
                                        srb->result = SAM_STAT_CHECK_CONDITION;
                                        memcpy(srb->sense_buffer,
                                               media_not_present,
                                               US_SENSE_SIZE);
                                }
                                usb_stor_dbg(us, "TEST_UNIT_READY\n");
                                goto out;
                        }
                        if (srb->cmnd[0] == ALLOW_MEDIUM_REMOVAL) {
                                int prevent = srb->cmnd[4] & 0x1;
                                if (prevent) {
                                        srb->result = SAM_STAT_CHECK_CONDITION;
                                        memcpy(srb->sense_buffer,
                                               invalid_cmd_field,
                                               US_SENSE_SIZE);
                                } else {
                                        srb->result = SAM_STAT_GOOD;
                                }
                                usb_stor_dbg(us, "ALLOW_MEDIUM_REMOVAL\n");
                                goto out;
                        }
                } else {
                        usb_stor_dbg(us, "NOT working scsi, not SS\n");
                        chip->proto_handler_backup(srb, us);
                        /* Check whether card is plugged in */
                        if (srb->cmnd[0] == TEST_UNIT_READY) {
                                if (srb->result == SAM_STAT_GOOD) {
                                        SET_LUN_READY(chip, srb->device->lun);
                                        if (card_first_show) {
                                                card_first_show = 0;
                                                fw5895_set_mmc_wp(us);
                                        }
                                } else {
                                        CLR_LUN_READY(chip, srb->device->lun);
                                        card_first_show = 1;
                                }
                        }
                        if (rts51x_get_stat(chip) != RTS51X_STAT_IDLE)
                                rts51x_set_stat(chip, RTS51X_STAT_IDLE);
                }
        }
out:
        usb_stor_dbg(us, "state:%d\n", rts51x_get_stat(chip));
        if (rts51x_get_stat(chip) == RTS51X_STAT_RUN)
                rts51x_modi_suspend_timer(chip);
}

static int realtek_cr_autosuspend_setup(struct us_data *us)
{
        struct rts51x_chip *chip;
        struct rts51x_status *status = NULL;
        u8 buf[16];
        int retval;

        chip = (struct rts51x_chip *)us->extra;
        chip->support_auto_delink = 0;
        chip->pwr_state = US_RESUME;
        chip->lun_ready = 0;
        rts51x_set_stat(chip, RTS51X_STAT_INIT);

        retval = rts51x_read_status(us, 0, buf, 16, &(chip->status_len));
        if (retval != STATUS_SUCCESS) {
                usb_stor_dbg(us, "Read status fail\n");
                return -EIO;
        }
        status = chip->status;
        status->vid = ((u16) buf[0] << 8) | buf[1];
        status->pid = ((u16) buf[2] << 8) | buf[3];
        status->cur_lun = buf[4];
        status->card_type = buf[5];
        status->total_lun = buf[6];
        status->fw_ver = ((u16) buf[7] << 8) | buf[8];
        status->phy_exist = buf[9];
        status->multi_flag = buf[10];
        status->multi_card = buf[11];
        status->log_exist = buf[12];
        if (chip->status_len == 16) {
                status->detailed_type.detailed_type1 = buf[13];
                status->function[0] = buf[14];
                status->function[1] = buf[15];
        }

        /* back up the proto_handler in us->extra */
        chip = (struct rts51x_chip *)(us->extra);
        chip->proto_handler_backup = us->proto_handler;
        /* Set the autosuspend_delay to 0 */
        pm_runtime_set_autosuspend_delay(&us->pusb_dev->dev, 0);
        /* override us->proto_handler setted in get_protocol() */
        us->proto_handler = rts51x_invoke_transport;

        chip->timer_expires = 0;
        timer_setup(&chip->rts51x_suspend_timer, rts51x_suspend_timer_fn, 0);
        fw5895_init(us);

        /* enable autosuspend function of the usb device */
        usb_enable_autosuspend(us->pusb_dev);

        return 0;
}
#endif

static void realtek_cr_destructor(void *extra)
{
        struct rts51x_chip *chip = extra;

        if (!chip)
                return;

#ifdef CONFIG_REALTEK_AUTOPM
        if (ss_en) {
                del_timer(&chip->rts51x_suspend_timer);
                chip->timer_expires = 0;
        }
#endif
        kfree(chip->status);
}

#ifdef CONFIG_PM
static int realtek_cr_suspend(struct usb_interface *iface, pm_message_t message)
{
        struct us_data *us = usb_get_intfdata(iface);

        /* wait until no command is running */
        mutex_lock(&us->dev_mutex);

        config_autodelink_before_power_down(us);

        mutex_unlock(&us->dev_mutex);

        return 0;
}

static int realtek_cr_resume(struct usb_interface *iface)
{
        struct us_data *us = usb_get_intfdata(iface);

        fw5895_init(us);
        config_autodelink_after_power_on(us);

        return 0;
}
#else
#define realtek_cr_suspend        NULL
#define realtek_cr_resume        NULL
#endif

static int init_realtek_cr(struct us_data *us)
{
        struct rts51x_chip *chip;
        int size, i, retval;

        chip = kzalloc(sizeof(struct rts51x_chip), GFP_KERNEL);
        if (!chip)
                return -ENOMEM;

        us->extra = chip;
        us->extra_destructor = realtek_cr_destructor;
        us->max_lun = chip->max_lun = rts51x_get_max_lun(us);
        chip->us = us;

        usb_stor_dbg(us, "chip->max_lun = %d\n", chip->max_lun);

        size = (chip->max_lun + 1) * sizeof(struct rts51x_status);
        chip->status = kzalloc(size, GFP_KERNEL);
        if (!chip->status)
                goto INIT_FAIL;

        for (i = 0; i <= (int)(chip->max_lun); i++) {
                retval = rts51x_check_status(us, (u8) i);
                if (retval < 0)
                        goto INIT_FAIL;
        }

        if (CHECK_PID(chip, 0x0138) || CHECK_PID(chip, 0x0158) ||
            CHECK_PID(chip, 0x0159)) {
                if (CHECK_FW_VER(chip, 0x5888) || CHECK_FW_VER(chip, 0x5889) ||
                                CHECK_FW_VER(chip, 0x5901))
                        SET_AUTO_DELINK(chip);
                if (STATUS_LEN(chip) == 16) {
                        if (SUPPORT_AUTO_DELINK(chip))
                                SET_AUTO_DELINK(chip);
                }
        }
#ifdef CONFIG_REALTEK_AUTOPM
        if (ss_en)
                realtek_cr_autosuspend_setup(us);
#endif

        usb_stor_dbg(us, "chip->flag = 0x%x\n", chip->flag);

        (void)config_autodelink_after_power_on(us);

        return 0;

INIT_FAIL:
        if (us->extra) {
                kfree(chip->status);
                kfree(us->extra);
                us->extra = NULL;
        }

        return -EIO;
}

static struct scsi_host_template realtek_cr_host_template;

static int realtek_cr_probe(struct usb_interface *intf,
                            const struct usb_device_id *id)
{
        struct us_data *us;
        int result;

        dev_dbg(&intf->dev, "Probe Realtek Card Reader!\n");

        result = usb_stor_probe1(&us, intf, id,
                                 (id - realtek_cr_ids) +
                                 realtek_cr_unusual_dev_list,
                                 &realtek_cr_host_template);
        if (result)
                return result;

        result = usb_stor_probe2(us);

        return result;
}

static struct usb_driver realtek_cr_driver = {
        .name = DRV_NAME,
        .probe = realtek_cr_probe,
        .disconnect = usb_stor_disconnect,
        /* .suspend =      usb_stor_suspend, */
        /* .resume =       usb_stor_resume, */
        .reset_resume = usb_stor_reset_resume,
        .suspend = realtek_cr_suspend,
        .resume = realtek_cr_resume,
        .pre_reset = usb_stor_pre_reset,
        .post_reset = usb_stor_post_reset,
        .id_table = realtek_cr_ids,
        .soft_unbind = 1,
        .supports_autosuspend = 1,
        .no_dynamic_id = 1,
};

module_usb_stor_driver(realtek_cr_driver, realtek_cr_host_template, DRV_NAME);







































































   12 























   12 
   10 
    3 







    3 



    3 









    8 





    8 

















    4 

    4 












































   31 



























  246 













  225 








   31 







   31 

















    2 
    8 






    8 
    8 
    8 
    1 





















    1 

    1 


    1 





    8 



   14 







   11 


    4 







   15 

   11 
   10 




   10 

    5 
    9 





    4 







    4 

    4 






   13 




    3 








   10 






   27 


   27 

   19 


   10 





   20 





























































   21 


   18 


    4 





   10 










   16 













    5 
   10 













   22 




   22 

    2 














   20 

    2 























    6 
   14 

























    5 






   15 


    4 
   11 
    2 
   13 

















   24 




    2 





    9 


   14 
    1 














































    9 

    3 



    4 

    5 



    9 



    9 




















   22 




    4 

   16 















    6 














   13 
   13 

    2 




  166 





  329 

















   56 


  323 




















  378 


  378 
  379 

  269 








  240 





   71 

   71 



  354 

  355 



   11 

   11 



    1 

    1 




  176 

  176 















  273 


  271 
  273 

  171 



  167 












    9 





   27 











  273 




  264 




  263 


  262 
  264 
  165 
  164 
  262 




  261 


  259 
  259 
  162 
  161 









  261 



   85 



   85 
   85 
   41 






   62 

   84 













   38 




   38 
   38 

   36 



   19 














































































   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 

   10 




    9 



















































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/blkdev.h>
#include <linux/blktrace_api.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/debugfs.h>
#include <linux/export.h>
#include <linux/time.h>
#include <linux/uaccess.h>
#include <linux/list.h>
#include <linux/blk-cgroup.h>

#include "../../block/blk.h"

#include <trace/events/block.h>

#include "trace_output.h"

#ifdef CONFIG_BLK_DEV_IO_TRACE

static unsigned int blktrace_seq __read_mostly = 1;

static struct trace_array *blk_tr;
static bool blk_tracer_enabled __read_mostly;

static LIST_HEAD(running_trace_list);
static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock);

/* Select an alternative, minimalistic output than the original one */
#define TRACE_BLK_OPT_CLASSIC        0x1
#define TRACE_BLK_OPT_CGROUP        0x2
#define TRACE_BLK_OPT_CGNAME        0x4

static struct tracer_opt blk_tracer_opts[] = {
        /* Default disable the minimalistic output */
        { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
#ifdef CONFIG_BLK_CGROUP
        { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) },
        { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) },
#endif
        { }
};

static struct tracer_flags blk_tracer_flags = {
        .val  = 0,
        .opts = blk_tracer_opts,
};

/* Global reference count of probes */
static DEFINE_MUTEX(blk_probe_mutex);
static int blk_probes_ref;

static void blk_register_tracepoints(void);
static void blk_unregister_tracepoints(void);

/*
 * Send out a notify message.
 */
static void trace_note(struct blk_trace *bt, pid_t pid, int action,
                       const void *data, size_t len, u64 cgid)
{
        struct blk_io_trace *t;
        struct ring_buffer_event *event = NULL;
        struct trace_buffer *buffer = NULL;
        unsigned int trace_ctx = 0;
        int cpu = smp_processor_id();
        bool blk_tracer = blk_tracer_enabled;
        ssize_t cgid_len = cgid ? sizeof(cgid) : 0;

        if (blk_tracer) {
                buffer = blk_tr->array_buffer.buffer;
                trace_ctx = tracing_gen_ctx_flags(0);
                event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
                                                  sizeof(*t) + len + cgid_len,
                                                  trace_ctx);
                if (!event)
                        return;
                t = ring_buffer_event_data(event);
                goto record_it;
        }

        if (!bt->rchan)
                return;

        t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len);
        if (t) {
                t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
                t->time = ktime_to_ns(ktime_get());
record_it:
                t->device = bt->dev;
                t->action = action | (cgid ? __BLK_TN_CGROUP : 0);
                t->pid = pid;
                t->cpu = cpu;
                t->pdu_len = len + cgid_len;
                if (cgid_len)
                        memcpy((void *)t + sizeof(*t), &cgid, cgid_len);
                memcpy((void *) t + sizeof(*t) + cgid_len, data, len);

                if (blk_tracer)
                        trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx);
        }
}

/*
 * Send out a notify for this process, if we haven't done so since a trace
 * started
 */
static void trace_note_tsk(struct task_struct *tsk)
{
        unsigned long flags;
        struct blk_trace *bt;

        tsk->btrace_seq = blktrace_seq;
        raw_spin_lock_irqsave(&running_trace_lock, flags);
        list_for_each_entry(bt, &running_trace_list, running_list) {
                trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,
                           sizeof(tsk->comm), 0);
        }
        raw_spin_unlock_irqrestore(&running_trace_lock, flags);
}

static void trace_note_time(struct blk_trace *bt)
{
        struct timespec64 now;
        unsigned long flags;
        u32 words[2];

        /* need to check user space to see if this breaks in y2038 or y2106 */
        ktime_get_real_ts64(&now);
        words[0] = (u32)now.tv_sec;
        words[1] = now.tv_nsec;

        local_irq_save(flags);
        trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0);
        local_irq_restore(flags);
}

void __blk_trace_note_message(struct blk_trace *bt,
                struct cgroup_subsys_state *css, const char *fmt, ...)
{
        int n;
        va_list args;
        unsigned long flags;
        char *buf;
        u64 cgid = 0;

        if (unlikely(bt->trace_state != Blktrace_running &&
                     !blk_tracer_enabled))
                return;

        /*
         * If the BLK_TC_NOTIFY action mask isn't set, don't send any note
         * message to the trace.
         */
        if (!(bt->act_mask & BLK_TC_NOTIFY))
                return;

        local_irq_save(flags);
        buf = this_cpu_ptr(bt->msg_data);
        va_start(args, fmt);
        n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
        va_end(args);

#ifdef CONFIG_BLK_CGROUP
        if (css && (blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
                cgid = cgroup_id(css->cgroup);
        else
                cgid = 1;
#endif
        trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, cgid);
        local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(__blk_trace_note_message);

static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
                         pid_t pid)
{
        if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
                return 1;
        if (sector && (sector < bt->start_lba || sector > bt->end_lba))
                return 1;
        if (bt->pid && pid != bt->pid)
                return 1;

        return 0;
}

/*
 * Data direction bit lookup
 */
static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
                                 BLK_TC_ACT(BLK_TC_WRITE) };

#define BLK_TC_RAHEAD                BLK_TC_AHEAD
#define BLK_TC_PREFLUSH                BLK_TC_FLUSH

/* The ilog2() calls fall out because they're constant */
#define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) <<        \
          (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))

/*
 * The worker for the various blk_add_trace*() types. Fills out a
 * blk_io_trace structure and places it in a per-cpu subbuffer.
 */
static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
                            const blk_opf_t opf, u32 what, int error,
                            int pdu_len, void *pdu_data, u64 cgid)
{
        struct task_struct *tsk = current;
        struct ring_buffer_event *event = NULL;
        struct trace_buffer *buffer = NULL;
        struct blk_io_trace *t;
        unsigned long flags = 0;
        unsigned long *sequence;
        unsigned int trace_ctx = 0;
        pid_t pid;
        int cpu;
        bool blk_tracer = blk_tracer_enabled;
        ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
        const enum req_op op = opf & REQ_OP_MASK;

        if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
                return;

        what |= ddir_act[op_is_write(op) ? WRITE : READ];
        what |= MASK_TC_BIT(opf, SYNC);
        what |= MASK_TC_BIT(opf, RAHEAD);
        what |= MASK_TC_BIT(opf, META);
        what |= MASK_TC_BIT(opf, PREFLUSH);
        what |= MASK_TC_BIT(opf, FUA);
        if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE)
                what |= BLK_TC_ACT(BLK_TC_DISCARD);
        if (op == REQ_OP_FLUSH)
                what |= BLK_TC_ACT(BLK_TC_FLUSH);
        if (cgid)
                what |= __BLK_TA_CGROUP;

        pid = tsk->pid;
        if (act_log_check(bt, what, sector, pid))
                return;
        cpu = raw_smp_processor_id();

        if (blk_tracer) {
                tracing_record_cmdline(current);

                buffer = blk_tr->array_buffer.buffer;
                trace_ctx = tracing_gen_ctx_flags(0);
                event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
                                                  sizeof(*t) + pdu_len + cgid_len,
                                                  trace_ctx);
                if (!event)
                        return;
                t = ring_buffer_event_data(event);
                goto record_it;
        }

        if (unlikely(tsk->btrace_seq != blktrace_seq))
                trace_note_tsk(tsk);

        /*
         * A word about the locking here - we disable interrupts to reserve
         * some space in the relay per-cpu buffer, to prevent an irq
         * from coming in and stepping on our toes.
         */
        local_irq_save(flags);
        t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len);
        if (t) {
                sequence = per_cpu_ptr(bt->sequence, cpu);

                t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
                t->sequence = ++(*sequence);
                t->time = ktime_to_ns(ktime_get());
record_it:
                /*
                 * These two are not needed in ftrace as they are in the
                 * generic trace_entry, filled by tracing_generic_entry_update,
                 * but for the trace_event->bin() synthesizer benefit we do it
                 * here too.
                 */
                t->cpu = cpu;
                t->pid = pid;

                t->sector = sector;
                t->bytes = bytes;
                t->action = what;
                t->device = bt->dev;
                t->error = error;
                t->pdu_len = pdu_len + cgid_len;

                if (cgid_len)
                        memcpy((void *)t + sizeof(*t), &cgid, cgid_len);
                if (pdu_len)
                        memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len);

                if (blk_tracer) {
                        trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx);
                        return;
                }
        }

        local_irq_restore(flags);
}

static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)
{
        relay_close(bt->rchan);

        /*
         * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created
         * under 'q->debugfs_dir', thus lookup and remove them.
         */
        if (!bt->dir) {
                debugfs_lookup_and_remove("dropped", q->debugfs_dir);
                debugfs_lookup_and_remove("msg", q->debugfs_dir);
        } else {
                debugfs_remove(bt->dir);
        }
        free_percpu(bt->sequence);
        free_percpu(bt->msg_data);
        kfree(bt);
}

static void get_probe_ref(void)
{
        mutex_lock(&blk_probe_mutex);
        if (++blk_probes_ref == 1)
                blk_register_tracepoints();
        mutex_unlock(&blk_probe_mutex);
}

static void put_probe_ref(void)
{
        mutex_lock(&blk_probe_mutex);
        if (!--blk_probes_ref)
                blk_unregister_tracepoints();
        mutex_unlock(&blk_probe_mutex);
}

static int blk_trace_start(struct blk_trace *bt)
{
        if (bt->trace_state != Blktrace_setup &&
            bt->trace_state != Blktrace_stopped)
                return -EINVAL;

        blktrace_seq++;
        smp_mb();
        bt->trace_state = Blktrace_running;
        raw_spin_lock_irq(&running_trace_lock);
        list_add(&bt->running_list, &running_trace_list);
        raw_spin_unlock_irq(&running_trace_lock);
        trace_note_time(bt);

        return 0;
}

static int blk_trace_stop(struct blk_trace *bt)
{
        if (bt->trace_state != Blktrace_running)
                return -EINVAL;

        bt->trace_state = Blktrace_stopped;
        raw_spin_lock_irq(&running_trace_lock);
        list_del_init(&bt->running_list);
        raw_spin_unlock_irq(&running_trace_lock);
        relay_flush(bt->rchan);

        return 0;
}

static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)
{
        blk_trace_stop(bt);
        synchronize_rcu();
        blk_trace_free(q, bt);
        put_probe_ref();
}

static int __blk_trace_remove(struct request_queue *q)
{
        struct blk_trace *bt;

        bt = rcu_replace_pointer(q->blk_trace, NULL,
                                 lockdep_is_held(&q->debugfs_mutex));
        if (!bt)
                return -EINVAL;

        blk_trace_cleanup(q, bt);

        return 0;
}

int blk_trace_remove(struct request_queue *q)
{
        int ret;

        mutex_lock(&q->debugfs_mutex);
        ret = __blk_trace_remove(q);
        mutex_unlock(&q->debugfs_mutex);

        return ret;
}
EXPORT_SYMBOL_GPL(blk_trace_remove);

static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
                                size_t count, loff_t *ppos)
{
        struct blk_trace *bt = filp->private_data;
        char buf[16];

        snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));

        return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
}

static const struct file_operations blk_dropped_fops = {
        .owner =        THIS_MODULE,
        .open =                simple_open,
        .read =                blk_dropped_read,
        .llseek =        default_llseek,
};

static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
                                size_t count, loff_t *ppos)
{
        char *msg;
        struct blk_trace *bt;

        if (count >= BLK_TN_MAX_MSG)
                return -EINVAL;

        msg = memdup_user_nul(buffer, count);
        if (IS_ERR(msg))
                return PTR_ERR(msg);

        bt = filp->private_data;
        __blk_trace_note_message(bt, NULL, "%s", msg);
        kfree(msg);

        return count;
}

static const struct file_operations blk_msg_fops = {
        .owner =        THIS_MODULE,
        .open =                simple_open,
        .write =        blk_msg_write,
        .llseek =        noop_llseek,
};

/*
 * Keep track of how many times we encountered a full subbuffer, to aid
 * the user space app in telling how many lost events there were.
 */
static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
                                     void *prev_subbuf, size_t prev_padding)
{
        struct blk_trace *bt;

        if (!relay_buf_full(buf))
                return 1;

        bt = buf->chan->private_data;
        atomic_inc(&bt->dropped);
        return 0;
}

static int blk_remove_buf_file_callback(struct dentry *dentry)
{
        debugfs_remove(dentry);

        return 0;
}

static struct dentry *blk_create_buf_file_callback(const char *filename,
                                                   struct dentry *parent,
                                                   umode_t mode,
                                                   struct rchan_buf *buf,
                                                   int *is_global)
{
        return debugfs_create_file(filename, mode, parent, buf,
                                        &relay_file_operations);
}

static const struct rchan_callbacks blk_relay_callbacks = {
        .subbuf_start                = blk_subbuf_start_callback,
        .create_buf_file        = blk_create_buf_file_callback,
        .remove_buf_file        = blk_remove_buf_file_callback,
};

static void blk_trace_setup_lba(struct blk_trace *bt,
                                struct block_device *bdev)
{
        if (bdev) {
                bt->start_lba = bdev->bd_start_sect;
                bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev);
        } else {
                bt->start_lba = 0;
                bt->end_lba = -1ULL;
        }
}

/*
 * Setup everything required to start tracing
 */
static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
                              struct block_device *bdev,
                              struct blk_user_trace_setup *buts)
{
        struct blk_trace *bt = NULL;
        struct dentry *dir = NULL;
        int ret;

        lockdep_assert_held(&q->debugfs_mutex);

        if (!buts->buf_size || !buts->buf_nr)
                return -EINVAL;

        strscpy_pad(buts->name, name, BLKTRACE_BDEV_SIZE);

        /*
         * some device names have larger paths - convert the slashes
         * to underscores for this to work as expected
         */
        strreplace(buts->name, '/', '_');

        /*
         * bdev can be NULL, as with scsi-generic, this is a helpful as
         * we can be.
         */
        if (rcu_dereference_protected(q->blk_trace,
                                      lockdep_is_held(&q->debugfs_mutex))) {
                pr_warn("Concurrent blktraces are not allowed on %s\n",
                        buts->name);
                return -EBUSY;
        }

        bt = kzalloc(sizeof(*bt), GFP_KERNEL);
        if (!bt)
                return -ENOMEM;

        ret = -ENOMEM;
        bt->sequence = alloc_percpu(unsigned long);
        if (!bt->sequence)
                goto err;

        bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
        if (!bt->msg_data)
                goto err;

        /*
         * When tracing the whole disk reuse the existing debugfs directory
         * created by the block layer on init. For partitions block devices,
         * and scsi-generic block devices we create a temporary new debugfs
         * directory that will be removed once the trace ends.
         */
        if (bdev && !bdev_is_partition(bdev))
                dir = q->debugfs_dir;
        else
                bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);

        /*
         * As blktrace relies on debugfs for its interface the debugfs directory
         * is required, contrary to the usual mantra of not checking for debugfs
         * files or directories.
         */
        if (IS_ERR_OR_NULL(dir)) {
                pr_warn("debugfs_dir not present for %s so skipping\n",
                        buts->name);
                ret = -ENOENT;
                goto err;
        }

        bt->dev = dev;
        atomic_set(&bt->dropped, 0);
        INIT_LIST_HEAD(&bt->running_list);

        ret = -EIO;
        debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
        debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);

        bt->rchan = relay_open("trace", dir, buts->buf_size,
                                buts->buf_nr, &blk_relay_callbacks, bt);
        if (!bt->rchan)
                goto err;

        bt->act_mask = buts->act_mask;
        if (!bt->act_mask)
                bt->act_mask = (u16) -1;

        blk_trace_setup_lba(bt, bdev);

        /* overwrite with user settings */
        if (buts->start_lba)
                bt->start_lba = buts->start_lba;
        if (buts->end_lba)
                bt->end_lba = buts->end_lba;

        bt->pid = buts->pid;
        bt->trace_state = Blktrace_setup;

        rcu_assign_pointer(q->blk_trace, bt);
        get_probe_ref();

        ret = 0;
err:
        if (ret)
                blk_trace_free(q, bt);
        return ret;
}

int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
                    struct block_device *bdev,
                    char __user *arg)
{
        struct blk_user_trace_setup buts;
        int ret;

        ret = copy_from_user(&buts, arg, sizeof(buts));
        if (ret)
                return -EFAULT;

        mutex_lock(&q->debugfs_mutex);
        ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
        mutex_unlock(&q->debugfs_mutex);
        if (ret)
                return ret;

        if (copy_to_user(arg, &buts, sizeof(buts))) {
                blk_trace_remove(q);
                return -EFAULT;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(blk_trace_setup);

#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
static int compat_blk_trace_setup(struct request_queue *q, char *name,
                                  dev_t dev, struct block_device *bdev,
                                  char __user *arg)
{
        struct blk_user_trace_setup buts;
        struct compat_blk_user_trace_setup cbuts;
        int ret;

        if (copy_from_user(&cbuts, arg, sizeof(cbuts)))
                return -EFAULT;

        buts = (struct blk_user_trace_setup) {
                .act_mask = cbuts.act_mask,
                .buf_size = cbuts.buf_size,
                .buf_nr = cbuts.buf_nr,
                .start_lba = cbuts.start_lba,
                .end_lba = cbuts.end_lba,
                .pid = cbuts.pid,
        };

        mutex_lock(&q->debugfs_mutex);
        ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
        mutex_unlock(&q->debugfs_mutex);
        if (ret)
                return ret;

        if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) {
                blk_trace_remove(q);
                return -EFAULT;
        }

        return 0;
}
#endif

static int __blk_trace_startstop(struct request_queue *q, int start)
{
        struct blk_trace *bt;

        bt = rcu_dereference_protected(q->blk_trace,
                                       lockdep_is_held(&q->debugfs_mutex));
        if (bt == NULL)
                return -EINVAL;

        if (start)
                return blk_trace_start(bt);
        else
                return blk_trace_stop(bt);
}

int blk_trace_startstop(struct request_queue *q, int start)
{
        int ret;

        mutex_lock(&q->debugfs_mutex);
        ret = __blk_trace_startstop(q, start);
        mutex_unlock(&q->debugfs_mutex);

        return ret;
}
EXPORT_SYMBOL_GPL(blk_trace_startstop);

/*
 * When reading or writing the blktrace sysfs files, the references to the
 * opened sysfs or device files should prevent the underlying block device
 * from being removed. So no further delete protection is really needed.
 */

/**
 * blk_trace_ioctl - handle the ioctls associated with tracing
 * @bdev:        the block device
 * @cmd:        the ioctl cmd
 * @arg:        the argument data, if any
 *
 **/
int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
{
        struct request_queue *q = bdev_get_queue(bdev);
        int ret, start = 0;
        char b[BDEVNAME_SIZE];

        switch (cmd) {
        case BLKTRACESETUP:
                snprintf(b, sizeof(b), "%pg", bdev);
                ret = blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
                break;
#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
        case BLKTRACESETUP32:
                snprintf(b, sizeof(b), "%pg", bdev);
                ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
                break;
#endif
        case BLKTRACESTART:
                start = 1;
                fallthrough;
        case BLKTRACESTOP:
                ret = blk_trace_startstop(q, start);
                break;
        case BLKTRACETEARDOWN:
                ret = blk_trace_remove(q);
                break;
        default:
                ret = -ENOTTY;
                break;
        }
        return ret;
}

/**
 * blk_trace_shutdown - stop and cleanup trace structures
 * @q:    the request queue associated with the device
 *
 **/
void blk_trace_shutdown(struct request_queue *q)
{
        if (rcu_dereference_protected(q->blk_trace,
                                      lockdep_is_held(&q->debugfs_mutex)))
                __blk_trace_remove(q);
}

#ifdef CONFIG_BLK_CGROUP
static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
{
        struct cgroup_subsys_state *blkcg_css;
        struct blk_trace *bt;

        /* We don't use the 'bt' value here except as an optimization... */
        bt = rcu_dereference_protected(q->blk_trace, 1);
        if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
                return 0;

        blkcg_css = bio_blkcg_css(bio);
        if (!blkcg_css)
                return 0;
        return cgroup_id(blkcg_css->cgroup);
}
#else
static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
{
        return 0;
}
#endif

static u64
blk_trace_request_get_cgid(struct request *rq)
{
        if (!rq->bio)
                return 0;
        /* Use the first bio */
        return blk_trace_bio_get_cgid(rq->q, rq->bio);
}

/*
 * blktrace probes
 */

/**
 * blk_add_trace_rq - Add a trace for a request oriented action
 * @rq:                the source request
 * @error:        return status to log
 * @nr_bytes:        number of completed bytes
 * @what:        the action
 * @cgid:        the cgroup info
 *
 * Description:
 *     Records an action against a request. Will log the bio offset + size.
 *
 **/
static void blk_add_trace_rq(struct request *rq, blk_status_t error,
                             unsigned int nr_bytes, u32 what, u64 cgid)
{
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(rq->q->blk_trace);
        if (likely(!bt)) {
                rcu_read_unlock();
                return;
        }

        if (blk_rq_is_passthrough(rq))
                what |= BLK_TC_ACT(BLK_TC_PC);
        else
                what |= BLK_TC_ACT(BLK_TC_FS);

        __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags,
                        what, blk_status_to_errno(error), 0, NULL, cgid);
        rcu_read_unlock();
}

static void blk_add_trace_rq_insert(void *ignore, struct request *rq)
{
        blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT,
                         blk_trace_request_get_cgid(rq));
}

static void blk_add_trace_rq_issue(void *ignore, struct request *rq)
{
        blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE,
                         blk_trace_request_get_cgid(rq));
}

static void blk_add_trace_rq_merge(void *ignore, struct request *rq)
{
        blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE,
                         blk_trace_request_get_cgid(rq));
}

static void blk_add_trace_rq_requeue(void *ignore, struct request *rq)
{
        blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE,
                         blk_trace_request_get_cgid(rq));
}

static void blk_add_trace_rq_complete(void *ignore, struct request *rq,
                        blk_status_t error, unsigned int nr_bytes)
{
        blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE,
                         blk_trace_request_get_cgid(rq));
}

/**
 * blk_add_trace_bio - Add a trace for a bio oriented action
 * @q:                queue the io is for
 * @bio:        the source bio
 * @what:        the action
 * @error:        error, if any
 *
 * Description:
 *     Records an action against a bio. Will log the bio offset + size.
 *
 **/
static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
                              u32 what, int error)
{
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        if (likely(!bt)) {
                rcu_read_unlock();
                return;
        }

        __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
                        bio->bi_opf, what, error, 0, NULL,
                        blk_trace_bio_get_cgid(q, bio));
        rcu_read_unlock();
}

static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio)
{
        blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0);
}

static void blk_add_trace_bio_complete(void *ignore,
                                       struct request_queue *q, struct bio *bio)
{
        blk_add_trace_bio(q, bio, BLK_TA_COMPLETE,
                          blk_status_to_errno(bio->bi_status));
}

static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio)
{
        blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE,
                        0);
}

static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio)
{
        blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE,
                        0);
}

static void blk_add_trace_bio_queue(void *ignore, struct bio *bio)
{
        blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0);
}

static void blk_add_trace_getrq(void *ignore, struct bio *bio)
{
        blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0);
}

static void blk_add_trace_plug(void *ignore, struct request_queue *q)
{
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        if (bt)
                __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);
        rcu_read_unlock();
}

static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
                                    unsigned int depth, bool explicit)
{
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        if (bt) {
                __be64 rpdu = cpu_to_be64(depth);
                u32 what;

                if (explicit)
                        what = BLK_TA_UNPLUG_IO;
                else
                        what = BLK_TA_UNPLUG_TIMER;

                __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);
        }
        rcu_read_unlock();
}

static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu)
{
        struct request_queue *q = bio->bi_bdev->bd_disk->queue;
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        if (bt) {
                __be64 rpdu = cpu_to_be64(pdu);

                __blk_add_trace(bt, bio->bi_iter.bi_sector,
                                bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT,
                                blk_status_to_errno(bio->bi_status),
                                sizeof(rpdu), &rpdu,
                                blk_trace_bio_get_cgid(q, bio));
        }
        rcu_read_unlock();
}

/**
 * blk_add_trace_bio_remap - Add a trace for a bio-remap operation
 * @ignore:        trace callback data parameter (not used)
 * @bio:        the source bio
 * @dev:        source device
 * @from:        source sector
 *
 * Called after a bio is remapped to a different device and/or sector.
 **/
static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev,
                                    sector_t from)
{
        struct request_queue *q = bio->bi_bdev->bd_disk->queue;
        struct blk_trace *bt;
        struct blk_io_trace_remap r;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        if (likely(!bt)) {
                rcu_read_unlock();
                return;
        }

        r.device_from = cpu_to_be32(dev);
        r.device_to   = cpu_to_be32(bio_dev(bio));
        r.sector_from = cpu_to_be64(from);

        __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
                        bio->bi_opf, BLK_TA_REMAP,
                        blk_status_to_errno(bio->bi_status),
                        sizeof(r), &r, blk_trace_bio_get_cgid(q, bio));
        rcu_read_unlock();
}

/**
 * blk_add_trace_rq_remap - Add a trace for a request-remap operation
 * @ignore:        trace callback data parameter (not used)
 * @rq:                the source request
 * @dev:        target device
 * @from:        source sector
 *
 * Description:
 *     Device mapper remaps request to other devices.
 *     Add a trace for that action.
 *
 **/
static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev,
                                   sector_t from)
{
        struct blk_trace *bt;
        struct blk_io_trace_remap r;

        rcu_read_lock();
        bt = rcu_dereference(rq->q->blk_trace);
        if (likely(!bt)) {
                rcu_read_unlock();
                return;
        }

        r.device_from = cpu_to_be32(dev);
        r.device_to   = cpu_to_be32(disk_devt(rq->q->disk));
        r.sector_from = cpu_to_be64(from);

        __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
                        rq->cmd_flags, BLK_TA_REMAP, 0,
                        sizeof(r), &r, blk_trace_request_get_cgid(rq));
        rcu_read_unlock();
}

/**
 * blk_add_driver_data - Add binary message with driver-specific data
 * @rq:                io request
 * @data:        driver-specific data
 * @len:        length of driver-specific data
 *
 * Description:
 *     Some drivers might want to write driver-specific data per request.
 *
 **/
void blk_add_driver_data(struct request *rq, void *data, size_t len)
{
        struct blk_trace *bt;

        rcu_read_lock();
        bt = rcu_dereference(rq->q->blk_trace);
        if (likely(!bt)) {
                rcu_read_unlock();
                return;
        }

        __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0,
                                BLK_TA_DRV_DATA, 0, len, data,
                                blk_trace_request_get_cgid(rq));
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(blk_add_driver_data);

static void blk_register_tracepoints(void)
{
        int ret;

        ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);
        WARN_ON(ret);
        ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
        WARN_ON(ret);
        ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL);
        WARN_ON(ret);
        ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
        WARN_ON(ret);
        ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
        WARN_ON(ret);
        ret = register_trace_block_getrq(blk_add_trace_getrq, NULL);
        WARN_ON(ret);
        ret = register_trace_block_plug(blk_add_trace_plug, NULL);
        WARN_ON(ret);
        ret = register_trace_block_unplug(blk_add_trace_unplug, NULL);
        WARN_ON(ret);
        ret = register_trace_block_split(blk_add_trace_split, NULL);
        WARN_ON(ret);
        ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
        WARN_ON(ret);
        ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
        WARN_ON(ret);
}

static void blk_unregister_tracepoints(void)
{
        unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
        unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
        unregister_trace_block_split(blk_add_trace_split, NULL);
        unregister_trace_block_unplug(blk_add_trace_unplug, NULL);
        unregister_trace_block_plug(blk_add_trace_plug, NULL);
        unregister_trace_block_getrq(blk_add_trace_getrq, NULL);
        unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
        unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
        unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
        unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
        unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
        unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
        unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
        unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL);
        unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
        unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);

        tracepoint_synchronize_unregister();
}

/*
 * struct blk_io_tracer formatting routines
 */

static void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
{
        int i = 0;
        int tc = t->action >> BLK_TC_SHIFT;

        if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
                rwbs[i++] = 'N';
                goto out;
        }

        if (tc & BLK_TC_FLUSH)
                rwbs[i++] = 'F';

        if (tc & BLK_TC_DISCARD)
                rwbs[i++] = 'D';
        else if (tc & BLK_TC_WRITE)
                rwbs[i++] = 'W';
        else if (t->bytes)
                rwbs[i++] = 'R';
        else
                rwbs[i++] = 'N';

        if (tc & BLK_TC_FUA)
                rwbs[i++] = 'F';
        if (tc & BLK_TC_AHEAD)
                rwbs[i++] = 'A';
        if (tc & BLK_TC_SYNC)
                rwbs[i++] = 'S';
        if (tc & BLK_TC_META)
                rwbs[i++] = 'M';
out:
        rwbs[i] = '\0';
}

static inline
const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
{
        return (const struct blk_io_trace *)ent;
}

static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg)
{
        return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0);
}

static inline u64 t_cgid(const struct trace_entry *ent)
{
        return *(u64 *)(te_blk_io_trace(ent) + 1);
}

static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg)
{
        return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0);
}

static inline u32 t_action(const struct trace_entry *ent)
{
        return te_blk_io_trace(ent)->action;
}

static inline u32 t_bytes(const struct trace_entry *ent)
{
        return te_blk_io_trace(ent)->bytes;
}

static inline u32 t_sec(const struct trace_entry *ent)
{
        return te_blk_io_trace(ent)->bytes >> 9;
}

static inline unsigned long long t_sector(const struct trace_entry *ent)
{
        return te_blk_io_trace(ent)->sector;
}

static inline __u16 t_error(const struct trace_entry *ent)
{
        return te_blk_io_trace(ent)->error;
}

static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg)
{
        const __be64 *val = pdu_start(ent, has_cg);
        return be64_to_cpu(*val);
}

typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act,
        bool has_cg);

static void blk_log_action_classic(struct trace_iterator *iter, const char *act,
        bool has_cg)
{
        char rwbs[RWBS_LEN];
        unsigned long long ts  = iter->ts;
        unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC);
        unsigned secs               = (unsigned long)ts;
        const struct blk_io_trace *t = te_blk_io_trace(iter->ent);

        fill_rwbs(rwbs, t);

        trace_seq_printf(&iter->seq,
                         "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ",
                         MAJOR(t->device), MINOR(t->device), iter->cpu,
                         secs, nsec_rem, iter->ent->pid, act, rwbs);
}

static void blk_log_action(struct trace_iterator *iter, const char *act,
        bool has_cg)
{
        char rwbs[RWBS_LEN];
        const struct blk_io_trace *t = te_blk_io_trace(iter->ent);

        fill_rwbs(rwbs, t);
        if (has_cg) {
                u64 id = t_cgid(iter->ent);

                if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) {
                        char blkcg_name_buf[NAME_MAX + 1] = "<...>";

                        cgroup_path_from_kernfs_id(id, blkcg_name_buf,
                                sizeof(blkcg_name_buf));
                        trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ",
                                 MAJOR(t->device), MINOR(t->device),
                                 blkcg_name_buf, act, rwbs);
                } else {
                        /*
                         * The cgid portion used to be "INO,GEN".  Userland
                         * builds a FILEID_INO32_GEN fid out of them and
                         * opens the cgroup using open_by_handle_at(2).
                         * While 32bit ino setups are still the same, 64bit
                         * ones now use the 64bit ino as the whole ID and
                         * no longer use generation.
                         *
                         * Regardless of the content, always output
                         * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can
                         * be mapped back to @id on both 64 and 32bit ino
                         * setups.  See __kernfs_fh_to_dentry().
                         */
                        trace_seq_printf(&iter->seq,
                                 "%3d,%-3d %llx,%-llx %2s %3s ",
                                 MAJOR(t->device), MINOR(t->device),
                                 id & U32_MAX, id >> 32, act, rwbs);
                }
        } else
                trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ",
                                 MAJOR(t->device), MINOR(t->device), act, rwbs);
}

static void blk_log_dump_pdu(struct trace_seq *s,
        const struct trace_entry *ent, bool has_cg)
{
        const unsigned char *pdu_buf;
        int pdu_len;
        int i, end;

        pdu_buf = pdu_start(ent, has_cg);
        pdu_len = pdu_real_len(ent, has_cg);

        if (!pdu_len)
                return;

        /* find the last zero that needs to be printed */
        for (end = pdu_len - 1; end >= 0; end--)
                if (pdu_buf[end])
                        break;
        end++;

        trace_seq_putc(s, '(');

        for (i = 0; i < pdu_len; i++) {

                trace_seq_printf(s, "%s%02x",
                                 i == 0 ? "" : " ", pdu_buf[i]);

                /*
                 * stop when the rest is just zeros and indicate so
                 * with a ".." appended
                 */
                if (i == end && end != pdu_len - 1) {
                        trace_seq_puts(s, " ..) ");
                        return;
                }
        }

        trace_seq_puts(s, ") ");
}

static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
{
        char cmd[TASK_COMM_LEN];

        trace_find_cmdline(ent->pid, cmd);

        if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
                trace_seq_printf(s, "%u ", t_bytes(ent));
                blk_log_dump_pdu(s, ent, has_cg);
                trace_seq_printf(s, "[%s]\n", cmd);
        } else {
                if (t_sec(ent))
                        trace_seq_printf(s, "%llu + %u [%s]\n",
                                                t_sector(ent), t_sec(ent), cmd);
                else
                        trace_seq_printf(s, "[%s]\n", cmd);
        }
}

static void blk_log_with_error(struct trace_seq *s,
                              const struct trace_entry *ent, bool has_cg)
{
        if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
                blk_log_dump_pdu(s, ent, has_cg);
                trace_seq_printf(s, "[%d]\n", t_error(ent));
        } else {
                if (t_sec(ent))
                        trace_seq_printf(s, "%llu + %u [%d]\n",
                                         t_sector(ent),
                                         t_sec(ent), t_error(ent));
                else
                        trace_seq_printf(s, "%llu [%d]\n",
                                         t_sector(ent), t_error(ent));
        }
}

static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
{
        const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg);

        trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
                         t_sector(ent), t_sec(ent),
                         MAJOR(be32_to_cpu(__r->device_from)),
                         MINOR(be32_to_cpu(__r->device_from)),
                         be64_to_cpu(__r->sector_from));
}

static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
{
        char cmd[TASK_COMM_LEN];

        trace_find_cmdline(ent->pid, cmd);

        trace_seq_printf(s, "[%s]\n", cmd);
}

static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
{
        char cmd[TASK_COMM_LEN];

        trace_find_cmdline(ent->pid, cmd);

        trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg));
}

static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
{
        char cmd[TASK_COMM_LEN];

        trace_find_cmdline(ent->pid, cmd);

        trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
                         get_pdu_int(ent, has_cg), cmd);
}

static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent,
                        bool has_cg)
{

        trace_seq_putmem(s, pdu_start(ent, has_cg),
                pdu_real_len(ent, has_cg));
        trace_seq_putc(s, '\n');
}

/*
 * struct tracer operations
 */

static void blk_tracer_print_header(struct seq_file *m)
{
        if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
                return;
        seq_puts(m, "# DEV   CPU TIMESTAMP     PID ACT FLG\n"
                    "#  |     |     |           |   |   |\n");
}

static void blk_tracer_start(struct trace_array *tr)
{
        blk_tracer_enabled = true;
}

static int blk_tracer_init(struct trace_array *tr)
{
        blk_tr = tr;
        blk_tracer_start(tr);
        return 0;
}

static void blk_tracer_stop(struct trace_array *tr)
{
        blk_tracer_enabled = false;
}

static void blk_tracer_reset(struct trace_array *tr)
{
        blk_tracer_stop(tr);
}

static const struct {
        const char *act[2];
        void           (*print)(struct trace_seq *s, const struct trace_entry *ent,
                            bool has_cg);
} what2act[] = {
        [__BLK_TA_QUEUE]        = {{  "Q", "queue" },           blk_log_generic },
        [__BLK_TA_BACKMERGE]        = {{  "M", "backmerge" },  blk_log_generic },
        [__BLK_TA_FRONTMERGE]        = {{  "F", "frontmerge" }, blk_log_generic },
        [__BLK_TA_GETRQ]        = {{  "G", "getrq" },           blk_log_generic },
        [__BLK_TA_SLEEPRQ]        = {{  "S", "sleeprq" },           blk_log_generic },
        [__BLK_TA_REQUEUE]        = {{  "R", "requeue" },           blk_log_with_error },
        [__BLK_TA_ISSUE]        = {{  "D", "issue" },           blk_log_generic },
        [__BLK_TA_COMPLETE]        = {{  "C", "complete" },   blk_log_with_error },
        [__BLK_TA_PLUG]                = {{  "P", "plug" },           blk_log_plug },
        [__BLK_TA_UNPLUG_IO]        = {{  "U", "unplug_io" },  blk_log_unplug },
        [__BLK_TA_UNPLUG_TIMER]        = {{ "UT", "unplug_timer" }, blk_log_unplug },
        [__BLK_TA_INSERT]        = {{  "I", "insert" },           blk_log_generic },
        [__BLK_TA_SPLIT]        = {{  "X", "split" },           blk_log_split },
        [__BLK_TA_BOUNCE]        = {{  "B", "bounce" },           blk_log_generic },
        [__BLK_TA_REMAP]        = {{  "A", "remap" },           blk_log_remap },
};

static enum print_line_t print_one_line(struct trace_iterator *iter,
                                        bool classic)
{
        struct trace_array *tr = iter->tr;
        struct trace_seq *s = &iter->seq;
        const struct blk_io_trace *t;
        u16 what;
        bool long_act;
        blk_log_action_t *log_action;
        bool has_cg;

        t           = te_blk_io_trace(iter->ent);
        what           = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP;
        long_act   = !!(tr->trace_flags & TRACE_ITER_VERBOSE);
        log_action = classic ? &blk_log_action_classic : &blk_log_action;
        has_cg           = t->action & __BLK_TA_CGROUP;

        if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
                log_action(iter, long_act ? "message" : "m", has_cg);
                blk_log_msg(s, iter->ent, has_cg);
                return trace_handle_return(s);
        }

        if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act)))
                trace_seq_printf(s, "Unknown action %x\n", what);
        else {
                log_action(iter, what2act[what].act[long_act], has_cg);
                what2act[what].print(s, iter->ent, has_cg);
        }

        return trace_handle_return(s);
}

static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
                                               int flags, struct trace_event *event)
{
        return print_one_line(iter, false);
}

static void blk_trace_synthesize_old_trace(struct trace_iterator *iter)
{
        struct trace_seq *s = &iter->seq;
        struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
        const int offset = offsetof(struct blk_io_trace, sector);
        struct blk_io_trace old = {
                .magic          = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
                .time     = iter->ts,
        };

        trace_seq_putmem(s, &old, offset);
        trace_seq_putmem(s, &t->sector,
                         sizeof(old) - offset + t->pdu_len);
}

static enum print_line_t
blk_trace_event_print_binary(struct trace_iterator *iter, int flags,
                             struct trace_event *event)
{
        blk_trace_synthesize_old_trace(iter);

        return trace_handle_return(&iter->seq);
}

static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
{
        if ((iter->ent->type != TRACE_BLK) ||
            !(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
                return TRACE_TYPE_UNHANDLED;

        return print_one_line(iter, true);
}

static int
blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
        /* don't output context-info for blk_classic output */
        if (bit == TRACE_BLK_OPT_CLASSIC) {
                if (set)
                        tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
                else
                        tr->trace_flags |= TRACE_ITER_CONTEXT_INFO;
        }
        return 0;
}

static struct tracer blk_tracer __read_mostly = {
        .name                = "blk",
        .init                = blk_tracer_init,
        .reset                = blk_tracer_reset,
        .start                = blk_tracer_start,
        .stop                = blk_tracer_stop,
        .print_header        = blk_tracer_print_header,
        .print_line        = blk_tracer_print_line,
        .flags                = &blk_tracer_flags,
        .set_flag        = blk_tracer_set_flag,
};

static struct trace_event_functions trace_blk_event_funcs = {
        .trace                = blk_trace_event_print,
        .binary                = blk_trace_event_print_binary,
};

static struct trace_event trace_blk_event = {
        .type                = TRACE_BLK,
        .funcs                = &trace_blk_event_funcs,
};

static int __init init_blk_tracer(void)
{
        if (!register_trace_event(&trace_blk_event)) {
                pr_warn("Warning: could not register block events\n");
                return 1;
        }

        if (register_tracer(&blk_tracer) != 0) {
                pr_warn("Warning: could not register the block tracer\n");
                unregister_trace_event(&trace_blk_event);
                return 1;
        }

        return 0;
}

device_initcall(init_blk_tracer);

static int blk_trace_remove_queue(struct request_queue *q)
{
        struct blk_trace *bt;

        bt = rcu_replace_pointer(q->blk_trace, NULL,
                                 lockdep_is_held(&q->debugfs_mutex));
        if (bt == NULL)
                return -EINVAL;

        blk_trace_stop(bt);

        put_probe_ref();
        synchronize_rcu();
        blk_trace_free(q, bt);
        return 0;
}

/*
 * Setup everything required to start tracing
 */
static int blk_trace_setup_queue(struct request_queue *q,
                                 struct block_device *bdev)
{
        struct blk_trace *bt = NULL;
        int ret = -ENOMEM;

        bt = kzalloc(sizeof(*bt), GFP_KERNEL);
        if (!bt)
                return -ENOMEM;

        bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
        if (!bt->msg_data)
                goto free_bt;

        bt->dev = bdev->bd_dev;
        bt->act_mask = (u16)-1;

        blk_trace_setup_lba(bt, bdev);

        rcu_assign_pointer(q->blk_trace, bt);
        get_probe_ref();
        return 0;

free_bt:
        blk_trace_free(q, bt);
        return ret;
}

/*
 * sysfs interface to enable and configure tracing
 */

static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf);
static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
                                          struct device_attribute *attr,
                                          const char *buf, size_t count);
#define BLK_TRACE_DEVICE_ATTR(_name) \
        DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
                    sysfs_blk_trace_attr_show, \
                    sysfs_blk_trace_attr_store)

static BLK_TRACE_DEVICE_ATTR(enable);
static BLK_TRACE_DEVICE_ATTR(act_mask);
static BLK_TRACE_DEVICE_ATTR(pid);
static BLK_TRACE_DEVICE_ATTR(start_lba);
static BLK_TRACE_DEVICE_ATTR(end_lba);

static struct attribute *blk_trace_attrs[] = {
        &dev_attr_enable.attr,
        &dev_attr_act_mask.attr,
        &dev_attr_pid.attr,
        &dev_attr_start_lba.attr,
        &dev_attr_end_lba.attr,
        NULL
};

struct attribute_group blk_trace_attr_group = {
        .name  = "trace",
        .attrs = blk_trace_attrs,
};

static const struct {
        int mask;
        const char *str;
} mask_maps[] = {
        { BLK_TC_READ,                "read"                },
        { BLK_TC_WRITE,                "write"                },
        { BLK_TC_FLUSH,                "flush"                },
        { BLK_TC_SYNC,                "sync"                },
        { BLK_TC_QUEUE,                "queue"                },
        { BLK_TC_REQUEUE,        "requeue"        },
        { BLK_TC_ISSUE,                "issue"                },
        { BLK_TC_COMPLETE,        "complete"        },
        { BLK_TC_FS,                "fs"                },
        { BLK_TC_PC,                "pc"                },
        { BLK_TC_NOTIFY,        "notify"        },
        { BLK_TC_AHEAD,                "ahead"                },
        { BLK_TC_META,                "meta"                },
        { BLK_TC_DISCARD,        "discard"        },
        { BLK_TC_DRV_DATA,        "drv_data"        },
        { BLK_TC_FUA,                "fua"                },
};

static int blk_trace_str2mask(const char *str)
{
        int i;
        int mask = 0;
        char *buf, *s, *token;

        buf = kstrdup(str, GFP_KERNEL);
        if (buf == NULL)
                return -ENOMEM;
        s = strstrip(buf);

        while (1) {
                token = strsep(&s, ",");
                if (token == NULL)
                        break;

                if (*token == '\0')
                        continue;

                for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
                        if (strcasecmp(token, mask_maps[i].str) == 0) {
                                mask |= mask_maps[i].mask;
                                break;
                        }
                }
                if (i == ARRAY_SIZE(mask_maps)) {
                        mask = -EINVAL;
                        break;
                }
        }
        kfree(buf);

        return mask;
}

static ssize_t blk_trace_mask2str(char *buf, int mask)
{
        int i;
        char *p = buf;

        for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
                if (mask & mask_maps[i].mask) {
                        p += sprintf(p, "%s%s",
                                    (p == buf) ? "" : ",", mask_maps[i].str);
                }
        }
        *p++ = '\n';

        return p - buf;
}

static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
                                         struct device_attribute *attr,
                                         char *buf)
{
        struct block_device *bdev = dev_to_bdev(dev);
        struct request_queue *q = bdev_get_queue(bdev);
        struct blk_trace *bt;
        ssize_t ret = -ENXIO;

        mutex_lock(&q->debugfs_mutex);

        bt = rcu_dereference_protected(q->blk_trace,
                                       lockdep_is_held(&q->debugfs_mutex));
        if (attr == &dev_attr_enable) {
                ret = sprintf(buf, "%u\n", !!bt);
                goto out_unlock_bdev;
        }

        if (bt == NULL)
                ret = sprintf(buf, "disabled\n");
        else if (attr == &dev_attr_act_mask)
                ret = blk_trace_mask2str(buf, bt->act_mask);
        else if (attr == &dev_attr_pid)
                ret = sprintf(buf, "%u\n", bt->pid);
        else if (attr == &dev_attr_start_lba)
                ret = sprintf(buf, "%llu\n", bt->start_lba);
        else if (attr == &dev_attr_end_lba)
                ret = sprintf(buf, "%llu\n", bt->end_lba);

out_unlock_bdev:
        mutex_unlock(&q->debugfs_mutex);
        return ret;
}

static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
                                          struct device_attribute *attr,
                                          const char *buf, size_t count)
{
        struct block_device *bdev = dev_to_bdev(dev);
        struct request_queue *q = bdev_get_queue(bdev);
        struct blk_trace *bt;
        u64 value;
        ssize_t ret = -EINVAL;

        if (count == 0)
                goto out;

        if (attr == &dev_attr_act_mask) {
                if (kstrtoull(buf, 0, &value)) {
                        /* Assume it is a list of trace category names */
                        ret = blk_trace_str2mask(buf);
                        if (ret < 0)
                                goto out;
                        value = ret;
                }
        } else {
                if (kstrtoull(buf, 0, &value))
                        goto out;
        }

        mutex_lock(&q->debugfs_mutex);

        bt = rcu_dereference_protected(q->blk_trace,
                                       lockdep_is_held(&q->debugfs_mutex));
        if (attr == &dev_attr_enable) {
                if (!!value == !!bt) {
                        ret = 0;
                        goto out_unlock_bdev;
                }
                if (value)
                        ret = blk_trace_setup_queue(q, bdev);
                else
                        ret = blk_trace_remove_queue(q);
                goto out_unlock_bdev;
        }

        ret = 0;
        if (bt == NULL) {
                ret = blk_trace_setup_queue(q, bdev);
                bt = rcu_dereference_protected(q->blk_trace,
                                lockdep_is_held(&q->debugfs_mutex));
        }

        if (ret == 0) {
                if (attr == &dev_attr_act_mask)
                        bt->act_mask = value;
                else if (attr == &dev_attr_pid)
                        bt->pid = value;
                else if (attr == &dev_attr_start_lba)
                        bt->start_lba = value;
                else if (attr == &dev_attr_end_lba)
                        bt->end_lba = value;
        }

out_unlock_bdev:
        mutex_unlock(&q->debugfs_mutex);
out:
        return ret ? ret : count;
}
#endif /* CONFIG_BLK_DEV_IO_TRACE */

#ifdef CONFIG_EVENT_TRACING

/**
 * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string.
 * @rwbs:        buffer to be filled
 * @opf:        request operation type (REQ_OP_XXX) and flags for the tracepoint
 *
 * Description:
 *     Maps each request operation and flag to a single character and fills the
 *     buffer provided by the caller with resulting string.
 *
 **/
void blk_fill_rwbs(char *rwbs, blk_opf_t opf)
{
        int i = 0;

        if (opf & REQ_PREFLUSH)
                rwbs[i++] = 'F';

        switch (opf & REQ_OP_MASK) {
        case REQ_OP_WRITE:
                rwbs[i++] = 'W';
                break;
        case REQ_OP_DISCARD:
                rwbs[i++] = 'D';
                break;
        case REQ_OP_SECURE_ERASE:
                rwbs[i++] = 'D';
                rwbs[i++] = 'E';
                break;
        case REQ_OP_FLUSH:
                rwbs[i++] = 'F';
                break;
        case REQ_OP_READ:
                rwbs[i++] = 'R';
                break;
        default:
                rwbs[i++] = 'N';
        }

        if (opf & REQ_FUA)
                rwbs[i++] = 'F';
        if (opf & REQ_RAHEAD)
                rwbs[i++] = 'A';
        if (opf & REQ_SYNC)
                rwbs[i++] = 'S';
        if (opf & REQ_META)
                rwbs[i++] = 'M';

        rwbs[i] = '\0';
}
EXPORT_SYMBOL_GPL(blk_fill_rwbs);

#endif /* CONFIG_EVENT_TRACING */

































































    3 






    3 






    3 



































































































































































    3 













    3 













    3 





















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) International Business Machines Corp., 2006
 *
 * Author: Artem Bityutskiy (Битюцкий Артём)
 */

#ifndef __UBI_DEBUG_H__
#define __UBI_DEBUG_H__

void ubi_dump_flash(struct ubi_device *ubi, int pnum, int offset, int len);
void ubi_dump_ec_hdr(const struct ubi_ec_hdr *ec_hdr);
void ubi_dump_vid_hdr(const struct ubi_vid_hdr *vid_hdr);

#include <linux/random.h>

#define ubi_assert(expr)  do {                                               \
        if (unlikely(!(expr))) {                                             \
                pr_crit("UBI assert failed in %s at %u (pid %d)\n",          \
                       __func__, __LINE__, current->pid);                    \
                dump_stack();                                                \
        }                                                                    \
} while (0)

#define ubi_dbg_print_hex_dump(l, ps, pt, r, g, b, len, a)                   \
                print_hex_dump(l, ps, pt, r, g, b, len, a)

#define ubi_dbg_msg(type, fmt, ...) \
        pr_debug("UBI DBG " type " (pid %d): " fmt "\n", current->pid,       \
                 ##__VA_ARGS__)

/* General debugging messages */
#define dbg_gen(fmt, ...) ubi_dbg_msg("gen", fmt, ##__VA_ARGS__)
/* Messages from the eraseblock association sub-system */
#define dbg_eba(fmt, ...) ubi_dbg_msg("eba", fmt, ##__VA_ARGS__)
/* Messages from the wear-leveling sub-system */
#define dbg_wl(fmt, ...)  ubi_dbg_msg("wl", fmt, ##__VA_ARGS__)
/* Messages from the input/output sub-system */
#define dbg_io(fmt, ...)  ubi_dbg_msg("io", fmt, ##__VA_ARGS__)
/* Initialization and build messages */
#define dbg_bld(fmt, ...) ubi_dbg_msg("bld", fmt, ##__VA_ARGS__)

void ubi_dump_vol_info(const struct ubi_volume *vol);
void ubi_dump_vtbl_record(const struct ubi_vtbl_record *r, int idx);
void ubi_dump_av(const struct ubi_ainf_volume *av);
void ubi_dump_aeb(const struct ubi_ainf_peb *aeb, int type);
void ubi_dump_mkvol_req(const struct ubi_mkvol_req *req);
int ubi_self_check_all_ff(struct ubi_device *ubi, int pnum, int offset,
                          int len);
int ubi_debugfs_init(void);
void ubi_debugfs_exit(void);
int ubi_debugfs_init_dev(struct ubi_device *ubi);
void ubi_debugfs_exit_dev(struct ubi_device *ubi);

/**
 * The following function is a legacy implementation of UBI fault-injection
 * hook. When using more powerful fault injection capabilities, the legacy
 * fault injection interface should be retained.
 */
int ubi_dbg_power_cut(struct ubi_device *ubi, int caller);

static inline int ubi_dbg_bitflip(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_bitflips)
                return !get_random_u32_below(200);
        return 0;
}

static inline int ubi_dbg_write_failure(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_io_failures)
                return !get_random_u32_below(500);
        return 0;
}

static inline int ubi_dbg_erase_failure(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_io_failures)
                return !get_random_u32_below(400);
        return 0;
}

/**
 * MASK_XXX: Mask for emulate_failures in ubi_debug_info.The mask is used to
 * precisely control the type and process of fault injection.
 */
/* Emulate a power cut when writing EC/VID header */
#define MASK_POWER_CUT_EC                        (1 << 0)
#define MASK_POWER_CUT_VID                        (1 << 1)
/* Emulate a power cut when writing data*/
#define MASK_POWER_CUT_DATA                        (1 << 2)
/* Emulate bit-flips */
#define MASK_BITFLIPS                                (1 << 3)
/* Emulate ecc error */
#define MASK_ECCERR                                (1 << 4)
/* Emulates -EIO during data read */
#define MASK_READ_FAILURE                        (1 << 5)
#define MASK_READ_FAILURE_EC                        (1 << 6)
#define MASK_READ_FAILURE_VID                        (1 << 7)
/* Emulates -EIO during data write */
#define MASK_WRITE_FAILURE                        (1 << 8)
/* Emulates -EIO during erase a PEB*/
#define MASK_ERASE_FAILURE                        (1 << 9)
/* Return UBI_IO_FF when reading EC/VID header */
#define MASK_IO_FF_EC                                (1 << 10)
#define MASK_IO_FF_VID                                (1 << 11)
/* Return UBI_IO_FF_BITFLIPS when reading EC/VID header */
#define MASK_IO_FF_BITFLIPS_EC                        (1 << 12)
#define MASK_IO_FF_BITFLIPS_VID                        (1 << 13)
/* Return UBI_IO_BAD_HDR when reading EC/VID header */
#define MASK_BAD_HDR_EC                                (1 << 14)
#define MASK_BAD_HDR_VID                        (1 << 15)
/* Return UBI_IO_BAD_HDR_EBADMSG when reading EC/VID header */
#define MASK_BAD_HDR_EBADMSG_EC                        (1 << 16)
#define MASK_BAD_HDR_EBADMSG_VID                (1 << 17)

#ifdef CONFIG_MTD_UBI_FAULT_INJECTION

extern bool should_fail_eccerr(void);
extern bool should_fail_bitflips(void);
extern bool should_fail_read_failure(void);
extern bool should_fail_write_failure(void);
extern bool should_fail_erase_failure(void);
extern bool should_fail_power_cut(void);
extern bool should_fail_io_ff(void);
extern bool should_fail_io_ff_bitflips(void);
extern bool should_fail_bad_hdr(void);
extern bool should_fail_bad_hdr_ebadmsg(void);

static inline bool ubi_dbg_fail_bitflip(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_BITFLIPS)
                return should_fail_bitflips();
        return false;
}

static inline bool ubi_dbg_fail_write(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_WRITE_FAILURE)
                return should_fail_write_failure();
        return false;
}

static inline bool ubi_dbg_fail_erase(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_ERASE_FAILURE)
                return should_fail_erase_failure();
        return false;
}

static inline bool ubi_dbg_fail_power_cut(const struct ubi_device *ubi,
                                          unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_power_cut();
        return false;
}

static inline bool ubi_dbg_fail_read(const struct ubi_device *ubi,
                                     unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_read_failure();
        return false;
}

static inline bool ubi_dbg_fail_eccerr(const struct ubi_device *ubi)
{
        if (ubi->dbg.emulate_failures & MASK_ECCERR)
                return should_fail_eccerr();
        return false;
}

static inline bool ubi_dbg_fail_ff(const struct ubi_device *ubi,
                                   unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_io_ff();
        return false;
}

static inline bool ubi_dbg_fail_ff_bitflips(const struct ubi_device *ubi,
                                            unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_io_ff_bitflips();
        return false;
}

static inline bool ubi_dbg_fail_bad_hdr(const struct ubi_device *ubi,
                                         unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_bad_hdr();
        return false;
}

static inline bool ubi_dbg_fail_bad_hdr_ebadmsg(const struct ubi_device *ubi,
                                                 unsigned int caller)
{
        if (ubi->dbg.emulate_failures & caller)
                return should_fail_bad_hdr_ebadmsg();
        return false;
}
#else /* CONFIG_MTD_UBI_FAULT_INJECTION */

#define ubi_dbg_fail_bitflip(u)             false
#define ubi_dbg_fail_write(u)               false
#define ubi_dbg_fail_erase(u)               false
#define ubi_dbg_fail_power_cut(u, c)        false
#define ubi_dbg_fail_read(u, c)             false
#define ubi_dbg_fail_eccerr(u)              false
#define ubi_dbg_fail_ff(u, c)               false
#define ubi_dbg_fail_ff_bitflips(u, v)      false
#define ubi_dbg_fail_bad_hdr(u, c)          false
#define ubi_dbg_fail_bad_hdr_ebadmsg(u, c)  false

#endif

/**
 * ubi_dbg_is_power_cut - if it is time to emulate power cut.
 * @ubi: UBI device description object
 *
 * Returns true if power cut should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_power_cut(struct ubi_device *ubi,
                                        unsigned int caller)
{
        if (ubi_dbg_power_cut(ubi, caller))
                return true;
        return ubi_dbg_fail_power_cut(ubi, caller);
}

/**
 * ubi_dbg_is_bitflip - if it is time to emulate a bit-flip.
 * @ubi: UBI device description object
 *
 * Returns true if a bit-flip should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_bitflip(const struct ubi_device *ubi)
{
        if (ubi_dbg_bitflip(ubi))
                return true;
        return ubi_dbg_fail_bitflip(ubi);
}

/**
 * ubi_dbg_is_write_failure - if it is time to emulate a write failure.
 * @ubi: UBI device description object
 *
 * Returns true if a write failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_write_failure(const struct ubi_device *ubi)
{
        if (ubi_dbg_write_failure(ubi))
                return true;
        return ubi_dbg_fail_write(ubi);
}

/**
 * ubi_dbg_is_erase_failure - if its time to emulate an erase failure.
 * @ubi: UBI device description object
 *
 * Returns true if an erase failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_erase_failure(const struct ubi_device *ubi)
{
        if (ubi_dbg_erase_failure(ubi))
                return true;
        return ubi_dbg_fail_erase(ubi);
}

/**
 * ubi_dbg_is_eccerr - if it is time to emulate ECC error.
 * @ubi: UBI device description object
 *
 * Returns true if a ECC error should be emulated, otherwise returns false.
 */
static inline bool ubi_dbg_is_eccerr(const struct ubi_device *ubi)
{
        return ubi_dbg_fail_eccerr(ubi);
}

/**
 * ubi_dbg_is_read_failure - if it is time to emulate a read failure.
 * @ubi: UBI device description object
 *
 * Returns true if a read failure should be emulated, otherwise returns
 * false.
 */
static inline bool ubi_dbg_is_read_failure(const struct ubi_device *ubi,
                                           unsigned int caller)
{
        return ubi_dbg_fail_read(ubi, caller);
}

/**
 * ubi_dbg_is_ff - if it is time to emulate that read region is only 0xFF.
 * @ubi: UBI device description object
 *
 * Returns true if read region should be emulated 0xFF, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_ff(const struct ubi_device *ubi,
                                 unsigned int caller)
{
        return ubi_dbg_fail_ff(ubi, caller);
}

/**
 * ubi_dbg_is_ff_bitflips - if it is time to emulate that read region is only 0xFF
 * with error reported by the MTD driver
 *
 * @ubi: UBI device description object
 *
 * Returns true if read region should be emulated 0xFF and error
 * reported by the MTD driver, otherwise returns false.
 */
static inline bool ubi_dbg_is_ff_bitflips(const struct ubi_device *ubi,
                                          unsigned int caller)
{
        return ubi_dbg_fail_ff_bitflips(ubi, caller);
}

/**
 * ubi_dbg_is_bad_hdr - if it is time to emulate a bad header
 * @ubi: UBI device description object
 *
 * Returns true if a bad header error should be emulated, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_bad_hdr(const struct ubi_device *ubi,
                                      unsigned int caller)
{
        return ubi_dbg_fail_bad_hdr(ubi, caller);
}

/**
 * ubi_dbg_is_bad_hdr_ebadmsg - if it is time to emulate a bad header with
 * ECC error.
 *
 * @ubi: UBI device description object
 *
 * Returns true if a bad header with ECC error should be emulated, otherwise
 * returns false.
 */
static inline bool ubi_dbg_is_bad_hdr_ebadmsg(const struct ubi_device *ubi,
                                              unsigned int caller)
{
        return ubi_dbg_fail_bad_hdr_ebadmsg(ubi, caller);
}

/**
 * ubi_dbg_is_bgt_disabled - if the background thread is disabled.
 * @ubi: UBI device description object
 *
 * Returns non-zero if the UBI background thread is disabled for testing
 * purposes.
 */
static inline int ubi_dbg_is_bgt_disabled(const struct ubi_device *ubi)
{
        return ubi->dbg.disable_bgt;
}

static inline int ubi_dbg_chk_io(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_io;
}

static inline int ubi_dbg_chk_gen(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_gen;
}

static inline int ubi_dbg_chk_fastmap(const struct ubi_device *ubi)
{
        return ubi->dbg.chk_fastmap;
}

static inline void ubi_enable_dbg_chk_fastmap(struct ubi_device *ubi)
{
        ubi->dbg.chk_fastmap = 1;
}

#endif /* !__UBI_DEBUG_H__ */







































































































    5 
















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
#ifndef __DRM_VMA_MANAGER_H__
#define __DRM_VMA_MANAGER_H__

/*
 * Copyright (c) 2013 David Herrmann <dh.herrmann@gmail.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#include <drm/drm_mm.h>
#include <linux/mm.h>
#include <linux/rbtree.h>
#include <linux/spinlock.h>
#include <linux/types.h>

/* We make up offsets for buffer objects so we can recognize them at
 * mmap time. pgoff in mmap is an unsigned long, so we need to make sure
 * that the faked up offset will fit
 */
#if BITS_PER_LONG == 64
#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1)
#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 256)
#else
#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1)
#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16)
#endif

struct drm_file;

struct drm_vma_offset_file {
        struct rb_node vm_rb;
        struct drm_file *vm_tag;
        unsigned long vm_count;
};

struct drm_vma_offset_node {
        rwlock_t vm_lock;
        struct drm_mm_node vm_node;
        struct rb_root vm_files;
        void *driver_private;
};

struct drm_vma_offset_manager {
        rwlock_t vm_lock;
        struct drm_mm vm_addr_space_mm;
};

void drm_vma_offset_manager_init(struct drm_vma_offset_manager *mgr,
                                 unsigned long page_offset, unsigned long size);
void drm_vma_offset_manager_destroy(struct drm_vma_offset_manager *mgr);

struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_manager *mgr,
                                                           unsigned long start,
                                                           unsigned long pages);
int drm_vma_offset_add(struct drm_vma_offset_manager *mgr,
                       struct drm_vma_offset_node *node, unsigned long pages);
void drm_vma_offset_remove(struct drm_vma_offset_manager *mgr,
                           struct drm_vma_offset_node *node);

int drm_vma_node_allow(struct drm_vma_offset_node *node, struct drm_file *tag);
int drm_vma_node_allow_once(struct drm_vma_offset_node *node, struct drm_file *tag);
void drm_vma_node_revoke(struct drm_vma_offset_node *node,
                         struct drm_file *tag);
bool drm_vma_node_is_allowed(struct drm_vma_offset_node *node,
                             struct drm_file *tag);

/**
 * drm_vma_offset_exact_lookup_locked() - Look up node by exact address
 * @mgr: Manager object
 * @start: Start address (page-based, not byte-based)
 * @pages: Size of object (page-based)
 *
 * Same as drm_vma_offset_lookup_locked() but does not allow any offset into the node.
 * It only returns the exact object with the given start address.
 *
 * RETURNS:
 * Node at exact start address @start.
 */
static inline struct drm_vma_offset_node *
drm_vma_offset_exact_lookup_locked(struct drm_vma_offset_manager *mgr,
                                   unsigned long start,
                                   unsigned long pages)
{
        struct drm_vma_offset_node *node;

        node = drm_vma_offset_lookup_locked(mgr, start, pages);
        return (node && node->vm_node.start == start) ? node : NULL;
}

/**
 * drm_vma_offset_lock_lookup() - Lock lookup for extended private use
 * @mgr: Manager object
 *
 * Lock VMA manager for extended lookups. Only locked VMA function calls
 * are allowed while holding this lock. All other contexts are blocked from VMA
 * until the lock is released via drm_vma_offset_unlock_lookup().
 *
 * Use this if you need to take a reference to the objects returned by
 * drm_vma_offset_lookup_locked() before releasing this lock again.
 *
 * This lock must not be used for anything else than extended lookups. You must
 * not call any other VMA helpers while holding this lock.
 *
 * Note: You're in atomic-context while holding this lock!
 */
static inline void drm_vma_offset_lock_lookup(struct drm_vma_offset_manager *mgr)
{
        read_lock(&mgr->vm_lock);
}

/**
 * drm_vma_offset_unlock_lookup() - Unlock lookup for extended private use
 * @mgr: Manager object
 *
 * Release lookup-lock. See drm_vma_offset_lock_lookup() for more information.
 */
static inline void drm_vma_offset_unlock_lookup(struct drm_vma_offset_manager *mgr)
{
        read_unlock(&mgr->vm_lock);
}

/**
 * drm_vma_node_reset() - Initialize or reset node object
 * @node: Node to initialize or reset
 *
 * Reset a node to its initial state. This must be called before using it with
 * any VMA offset manager.
 *
 * This must not be called on an already allocated node, or you will leak
 * memory.
 */
static inline void drm_vma_node_reset(struct drm_vma_offset_node *node)
{
        memset(node, 0, sizeof(*node));
        node->vm_files = RB_ROOT;
        rwlock_init(&node->vm_lock);
}

/**
 * drm_vma_node_start() - Return start address for page-based addressing
 * @node: Node to inspect
 *
 * Return the start address of the given node. This can be used as offset into
 * the linear VM space that is provided by the VMA offset manager. Note that
 * this can only be used for page-based addressing. If you need a proper offset
 * for user-space mappings, you must apply "<< PAGE_SHIFT" or use the
 * drm_vma_node_offset_addr() helper instead.
 *
 * RETURNS:
 * Start address of @node for page-based addressing. 0 if the node does not
 * have an offset allocated.
 */
static inline unsigned long drm_vma_node_start(const struct drm_vma_offset_node *node)
{
        return node->vm_node.start;
}

/**
 * drm_vma_node_size() - Return size (page-based)
 * @node: Node to inspect
 *
 * Return the size as number of pages for the given node. This is the same size
 * that was passed to drm_vma_offset_add(). If no offset is allocated for the
 * node, this is 0.
 *
 * RETURNS:
 * Size of @node as number of pages. 0 if the node does not have an offset
 * allocated.
 */
static inline unsigned long drm_vma_node_size(struct drm_vma_offset_node *node)
{
        return node->vm_node.size;
}

/**
 * drm_vma_node_offset_addr() - Return sanitized offset for user-space mmaps
 * @node: Linked offset node
 *
 * Same as drm_vma_node_start() but returns the address as a valid offset that
 * can be used for user-space mappings during mmap().
 * This must not be called on unlinked nodes.
 *
 * RETURNS:
 * Offset of @node for byte-based addressing. 0 if the node does not have an
 * object allocated.
 */
static inline __u64 drm_vma_node_offset_addr(struct drm_vma_offset_node *node)
{
        return ((__u64)node->vm_node.start) << PAGE_SHIFT;
}

/**
 * drm_vma_node_unmap() - Unmap offset node
 * @node: Offset node
 * @file_mapping: Address space to unmap @node from
 *
 * Unmap all userspace mappings for a given offset node. The mappings must be
 * associated with the @file_mapping address-space. If no offset exists
 * nothing is done.
 *
 * This call is unlocked. The caller must guarantee that drm_vma_offset_remove()
 * is not called on this node concurrently.
 */
static inline void drm_vma_node_unmap(struct drm_vma_offset_node *node,
                                      struct address_space *file_mapping)
{
        if (drm_mm_node_allocated(&node->vm_node))
                unmap_mapping_range(file_mapping,
                                    drm_vma_node_offset_addr(node),
                                    drm_vma_node_size(node) << PAGE_SHIFT, 1);
}

/**
 * drm_vma_node_verify_access() - Access verification helper for TTM
 * @node: Offset node
 * @tag: Tag of file to check
 *
 * This checks whether @tag is granted access to @node. It is the same as
 * drm_vma_node_is_allowed() but suitable as drop-in helper for TTM
 * verify_access() callbacks.
 *
 * RETURNS:
 * 0 if access is granted, -EACCES otherwise.
 */
static inline int drm_vma_node_verify_access(struct drm_vma_offset_node *node,
                                             struct drm_file *tag)
{
        return drm_vma_node_is_allowed(node, tag) ? 0 : -EACCES;
}

#endif /* __DRM_VMA_MANAGER_H__ */









































    1 







    1 







    1 








    1 





    3 
    1 



    1 
    1 








    1 





    7 









    2 









    1 
    5 










    1 







    1 






    3 


    1 


    2 




    6 



    3 


    3 











    1 









    1 













    2 








    1 





    1 










    1 





    1 























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
// SPDX-License-Identifier: GPL-2.0-only
/*
 * kernel/ksysfs.c - sysfs attributes in /sys/kernel, which
 *                      are not related to any other subsystem
 *
 * Copyright (C) 2004 Kay Sievers <kay.sievers@vrfy.org>
 */

#include <asm/byteorder.h>
#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/sysfs.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/kexec.h>
#include <linux/profile.h>
#include <linux/stat.h>
#include <linux/sched.h>
#include <linux/capability.h>
#include <linux/compiler.h>

#include <linux/rcupdate.h>        /* rcu_expedited and rcu_normal */

#if defined(__LITTLE_ENDIAN)
#define CPU_BYTEORDER_STRING        "little"
#elif defined(__BIG_ENDIAN)
#define CPU_BYTEORDER_STRING        "big"
#else
#error Unknown byteorder
#endif

#define KERNEL_ATTR_RO(_name) \
static struct kobj_attribute _name##_attr = __ATTR_RO(_name)

#define KERNEL_ATTR_RW(_name) \
static struct kobj_attribute _name##_attr = __ATTR_RW(_name)

/* current uevent sequence number */
static ssize_t uevent_seqnum_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%llu\n", (u64)atomic64_read(&uevent_seqnum));
}
KERNEL_ATTR_RO(uevent_seqnum);

/* cpu byteorder */
static ssize_t cpu_byteorder_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%s\n", CPU_BYTEORDER_STRING);
}
KERNEL_ATTR_RO(cpu_byteorder);

/* address bits */
static ssize_t address_bits_show(struct kobject *kobj,
                                 struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%zu\n", sizeof(void *) * 8 /* CHAR_BIT */);
}
KERNEL_ATTR_RO(address_bits);

#ifdef CONFIG_UEVENT_HELPER
/* uevent helper program, used during early boot */
static ssize_t uevent_helper_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%s\n", uevent_helper);
}
static ssize_t uevent_helper_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count)
{
        if (count+1 > UEVENT_HELPER_PATH_LEN)
                return -ENOENT;
        memcpy(uevent_helper, buf, count);
        uevent_helper[count] = '\0';
        if (count && uevent_helper[count-1] == '\n')
                uevent_helper[count-1] = '\0';
        return count;
}
KERNEL_ATTR_RW(uevent_helper);
#endif

#ifdef CONFIG_PROFILING
static ssize_t profiling_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", prof_on);
}
static ssize_t profiling_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count)
{
        int ret;
        static DEFINE_MUTEX(lock);

        /*
         * We need serialization, for profile_setup() initializes prof_on
         * value and profile_init() must not reallocate prof_buffer after
         * once allocated.
         */
        guard(mutex)(&lock);
        if (prof_on)
                return -EEXIST;
        /*
         * This eventually calls into get_option() which
         * has a ton of callers and is not const.  It is
         * easiest to cast it away here.
         */
        profile_setup((char *)buf);
        ret = profile_init();
        if (ret)
                return ret;
        ret = create_proc_profile();
        if (ret)
                return ret;
        return count;
}
KERNEL_ATTR_RW(profiling);
#endif

#ifdef CONFIG_KEXEC_CORE
static ssize_t kexec_loaded_show(struct kobject *kobj,
                                 struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", !!kexec_image);
}
KERNEL_ATTR_RO(kexec_loaded);

#ifdef CONFIG_CRASH_DUMP
static ssize_t kexec_crash_loaded_show(struct kobject *kobj,
                                       struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", kexec_crash_loaded());
}
KERNEL_ATTR_RO(kexec_crash_loaded);

static ssize_t kexec_crash_size_show(struct kobject *kobj,
                                       struct kobj_attribute *attr, char *buf)
{
        ssize_t size = crash_get_memory_size();

        if (size < 0)
                return size;

        return sysfs_emit(buf, "%zd\n", size);
}
static ssize_t kexec_crash_size_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count)
{
        unsigned long cnt;
        int ret;

        if (kstrtoul(buf, 0, &cnt))
                return -EINVAL;

        ret = crash_shrink_memory(cnt);
        return ret < 0 ? ret : count;
}
KERNEL_ATTR_RW(kexec_crash_size);

#endif /* CONFIG_CRASH_DUMP*/
#endif /* CONFIG_KEXEC_CORE */

#ifdef CONFIG_VMCORE_INFO

static ssize_t vmcoreinfo_show(struct kobject *kobj,
                               struct kobj_attribute *attr, char *buf)
{
        phys_addr_t vmcore_base = paddr_vmcoreinfo_note();
        return sysfs_emit(buf, "%pa %x\n", &vmcore_base,
                          (unsigned int)VMCOREINFO_NOTE_SIZE);
}
KERNEL_ATTR_RO(vmcoreinfo);

#ifdef CONFIG_CRASH_HOTPLUG
static ssize_t crash_elfcorehdr_size_show(struct kobject *kobj,
                               struct kobj_attribute *attr, char *buf)
{
        unsigned int sz = crash_get_elfcorehdr_size();

        return sysfs_emit(buf, "%u\n", sz);
}
KERNEL_ATTR_RO(crash_elfcorehdr_size);

#endif

#endif /* CONFIG_VMCORE_INFO */

/* whether file capabilities are enabled */
static ssize_t fscaps_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", file_caps_enabled);
}
KERNEL_ATTR_RO(fscaps);

#ifndef CONFIG_TINY_RCU
int rcu_expedited;
static ssize_t rcu_expedited_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", READ_ONCE(rcu_expedited));
}
static ssize_t rcu_expedited_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count)
{
        if (kstrtoint(buf, 0, &rcu_expedited))
                return -EINVAL;

        return count;
}
KERNEL_ATTR_RW(rcu_expedited);

int rcu_normal;
static ssize_t rcu_normal_show(struct kobject *kobj,
                               struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%d\n", READ_ONCE(rcu_normal));
}
static ssize_t rcu_normal_store(struct kobject *kobj,
                                struct kobj_attribute *attr,
                                const char *buf, size_t count)
{
        if (kstrtoint(buf, 0, &rcu_normal))
                return -EINVAL;

        return count;
}
KERNEL_ATTR_RW(rcu_normal);
#endif /* #ifndef CONFIG_TINY_RCU */

/*
 * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
 */
extern const void __start_notes;
extern const void __stop_notes;
#define        notes_size (&__stop_notes - &__start_notes)

static __ro_after_init BIN_ATTR_SIMPLE_RO(notes);

struct kobject *kernel_kobj;
EXPORT_SYMBOL_GPL(kernel_kobj);

static struct attribute * kernel_attrs[] = {
        &fscaps_attr.attr,
        &uevent_seqnum_attr.attr,
        &cpu_byteorder_attr.attr,
        &address_bits_attr.attr,
#ifdef CONFIG_UEVENT_HELPER
        &uevent_helper_attr.attr,
#endif
#ifdef CONFIG_PROFILING
        &profiling_attr.attr,
#endif
#ifdef CONFIG_KEXEC_CORE
        &kexec_loaded_attr.attr,
#ifdef CONFIG_CRASH_DUMP
        &kexec_crash_loaded_attr.attr,
        &kexec_crash_size_attr.attr,
#endif
#endif
#ifdef CONFIG_VMCORE_INFO
        &vmcoreinfo_attr.attr,
#ifdef CONFIG_CRASH_HOTPLUG
        &crash_elfcorehdr_size_attr.attr,
#endif
#endif
#ifndef CONFIG_TINY_RCU
        &rcu_expedited_attr.attr,
        &rcu_normal_attr.attr,
#endif
        NULL
};

static const struct attribute_group kernel_attr_group = {
        .attrs = kernel_attrs,
};

static int __init ksysfs_init(void)
{
        int error;

        kernel_kobj = kobject_create_and_add("kernel", NULL);
        if (!kernel_kobj) {
                error = -ENOMEM;
                goto exit;
        }
        error = sysfs_create_group(kernel_kobj, &kernel_attr_group);
        if (error)
                goto kset_exit;

        if (notes_size > 0) {
                bin_attr_notes.private = (void *)&__start_notes;
                bin_attr_notes.size = notes_size;
                error = sysfs_create_bin_file(kernel_kobj, &bin_attr_notes);
                if (error)
                        goto group_exit;
        }

        return 0;

group_exit:
        sysfs_remove_group(kernel_kobj, &kernel_attr_group);
kset_exit:
        kobject_put(kernel_kobj);
exit:
        return error;
}

core_initcall(ksysfs_init);











































































































































































   33 

























    2 



   30 




   31 





   13 




    1 









    6 


    1 
















   28 
   27 

    7 









    4 



    1 


    4 








    5 







   18 



   16 





   20 










   20 





   20 















   20 
   16 







   16 


   20 













   20 





    2 


    2 




























































































   12 

    1 

























































   19 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
// SPDX-License-Identifier: GPL-2.0-or-later
/* Decoder for ASN.1 BER/DER/CER encoded bytestream
 *
 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/asn1_decoder.h>
#include <linux/asn1_ber_bytecode.h>

static const unsigned char asn1_op_lengths[ASN1_OP__NR] = {
        /*                                        OPC TAG JMP ACT */
        [ASN1_OP_MATCH]                                = 1 + 1,
        [ASN1_OP_MATCH_OR_SKIP]                        = 1 + 1,
        [ASN1_OP_MATCH_ACT]                        = 1 + 1     + 1,
        [ASN1_OP_MATCH_ACT_OR_SKIP]                = 1 + 1     + 1,
        [ASN1_OP_MATCH_JUMP]                        = 1 + 1 + 1,
        [ASN1_OP_MATCH_JUMP_OR_SKIP]                = 1 + 1 + 1,
        [ASN1_OP_MATCH_ANY]                        = 1,
        [ASN1_OP_MATCH_ANY_OR_SKIP]                = 1,
        [ASN1_OP_MATCH_ANY_ACT]                        = 1         + 1,
        [ASN1_OP_MATCH_ANY_ACT_OR_SKIP]                = 1         + 1,
        [ASN1_OP_COND_MATCH_OR_SKIP]                = 1 + 1,
        [ASN1_OP_COND_MATCH_ACT_OR_SKIP]        = 1 + 1     + 1,
        [ASN1_OP_COND_MATCH_JUMP_OR_SKIP]        = 1 + 1 + 1,
        [ASN1_OP_COND_MATCH_ANY]                = 1,
        [ASN1_OP_COND_MATCH_ANY_OR_SKIP]        = 1,
        [ASN1_OP_COND_MATCH_ANY_ACT]                = 1         + 1,
        [ASN1_OP_COND_MATCH_ANY_ACT_OR_SKIP]        = 1         + 1,
        [ASN1_OP_COND_FAIL]                        = 1,
        [ASN1_OP_COMPLETE]                        = 1,
        [ASN1_OP_ACT]                                = 1         + 1,
        [ASN1_OP_MAYBE_ACT]                        = 1         + 1,
        [ASN1_OP_RETURN]                        = 1,
        [ASN1_OP_END_SEQ]                        = 1,
        [ASN1_OP_END_SEQ_OF]                        = 1     + 1,
        [ASN1_OP_END_SET]                        = 1,
        [ASN1_OP_END_SET_OF]                        = 1     + 1,
        [ASN1_OP_END_SEQ_ACT]                        = 1         + 1,
        [ASN1_OP_END_SEQ_OF_ACT]                = 1     + 1 + 1,
        [ASN1_OP_END_SET_ACT]                        = 1         + 1,
        [ASN1_OP_END_SET_OF_ACT]                = 1     + 1 + 1,
};

/*
 * Find the length of an indefinite length object
 * @data: The data buffer
 * @datalen: The end of the innermost containing element in the buffer
 * @_dp: The data parse cursor (updated before returning)
 * @_len: Where to return the size of the element.
 * @_errmsg: Where to return a pointer to an error message on error
 */
static int asn1_find_indefinite_length(const unsigned char *data, size_t datalen,
                                       size_t *_dp, size_t *_len,
                                       const char **_errmsg)
{
        unsigned char tag, tmp;
        size_t dp = *_dp, len, n;
        int indef_level = 1;

next_tag:
        if (unlikely(datalen - dp < 2)) {
                if (datalen == dp)
                        goto missing_eoc;
                goto data_overrun_error;
        }

        /* Extract a tag from the data */
        tag = data[dp++];
        if (tag == ASN1_EOC) {
                /* It appears to be an EOC. */
                if (data[dp++] != 0)
                        goto invalid_eoc;
                if (--indef_level <= 0) {
                        *_len = dp - *_dp;
                        *_dp = dp;
                        return 0;
                }
                goto next_tag;
        }

        if (unlikely((tag & 0x1f) == ASN1_LONG_TAG)) {
                do {
                        if (unlikely(datalen - dp < 2))
                                goto data_overrun_error;
                        tmp = data[dp++];
                } while (tmp & 0x80);
        }

        /* Extract the length */
        len = data[dp++];
        if (len <= 0x7f)
                goto check_length;

        if (unlikely(len == ASN1_INDEFINITE_LENGTH)) {
                /* Indefinite length */
                if (unlikely((tag & ASN1_CONS_BIT) == ASN1_PRIM << 5))
                        goto indefinite_len_primitive;
                indef_level++;
                goto next_tag;
        }

        n = len - 0x80;
        if (unlikely(n > sizeof(len) - 1))
                goto length_too_long;
        if (unlikely(n > datalen - dp))
                goto data_overrun_error;
        len = 0;
        for (; n > 0; n--) {
                len <<= 8;
                len |= data[dp++];
        }
check_length:
        if (len > datalen - dp)
                goto data_overrun_error;
        dp += len;
        goto next_tag;

length_too_long:
        *_errmsg = "Unsupported length";
        goto error;
indefinite_len_primitive:
        *_errmsg = "Indefinite len primitive not permitted";
        goto error;
invalid_eoc:
        *_errmsg = "Invalid length EOC";
        goto error;
data_overrun_error:
        *_errmsg = "Data overrun error";
        goto error;
missing_eoc:
        *_errmsg = "Missing EOC in indefinite len cons";
error:
        *_dp = dp;
        return -1;
}

/**
 * asn1_ber_decoder - Decoder BER/DER/CER ASN.1 according to pattern
 * @decoder: The decoder definition (produced by asn1_compiler)
 * @context: The caller's context (to be passed to the action functions)
 * @data: The encoded data
 * @datalen: The size of the encoded data
 *
 * Decode BER/DER/CER encoded ASN.1 data according to a bytecode pattern
 * produced by asn1_compiler.  Action functions are called on marked tags to
 * allow the caller to retrieve significant data.
 *
 * LIMITATIONS:
 *
 * To keep down the amount of stack used by this function, the following limits
 * have been imposed:
 *
 *  (1) This won't handle datalen > 65535 without increasing the size of the
 *        cons stack elements and length_too_long checking.
 *
 *  (2) The stack of constructed types is 10 deep.  If the depth of non-leaf
 *        constructed types exceeds this, the decode will fail.
 *
 *  (3) The SET type (not the SET OF type) isn't really supported as tracking
 *        what members of the set have been seen is a pain.
 */
int asn1_ber_decoder(const struct asn1_decoder *decoder,
                     void *context,
                     const unsigned char *data,
                     size_t datalen)
{
        const unsigned char *machine = decoder->machine;
        const asn1_action_t *actions = decoder->actions;
        size_t machlen = decoder->machlen;
        enum asn1_opcode op;
        unsigned char tag = 0, csp = 0, jsp = 0, optag = 0, hdr = 0;
        const char *errmsg;
        size_t pc = 0, dp = 0, tdp = 0, len = 0;
        int ret;

        unsigned char flags = 0;
#define FLAG_INDEFINITE_LENGTH        0x01
#define FLAG_MATCHED                0x02
#define FLAG_LAST_MATCHED        0x04 /* Last tag matched */
#define FLAG_CONS                0x20 /* Corresponds to CONS bit in the opcode tag
                                      * - ie. whether or not we are going to parse
                                      *   a compound type.
                                      */

#define NR_CONS_STACK 10
        unsigned short cons_dp_stack[NR_CONS_STACK];
        unsigned short cons_datalen_stack[NR_CONS_STACK];
        unsigned char cons_hdrlen_stack[NR_CONS_STACK];
#define NR_JUMP_STACK 10
        unsigned char jump_stack[NR_JUMP_STACK];

        if (datalen > 65535)
                return -EMSGSIZE;

next_op:
        pr_debug("next_op: pc=\e[32m%zu\e[m/%zu dp=\e[33m%zu\e[m/%zu C=%d J=%d\n",
                 pc, machlen, dp, datalen, csp, jsp);
        if (unlikely(pc >= machlen))
                goto machine_overrun_error;
        op = machine[pc];
        if (unlikely(pc + asn1_op_lengths[op] > machlen))
                goto machine_overrun_error;

        /* If this command is meant to match a tag, then do that before
         * evaluating the command.
         */
        if (op <= ASN1_OP__MATCHES_TAG) {
                unsigned char tmp;

                /* Skip conditional matches if possible */
                if ((op & ASN1_OP_MATCH__COND && flags & FLAG_MATCHED) ||
                    (op & ASN1_OP_MATCH__SKIP && dp == datalen)) {
                        flags &= ~FLAG_LAST_MATCHED;
                        pc += asn1_op_lengths[op];
                        goto next_op;
                }

                flags = 0;
                hdr = 2;

                /* Extract a tag from the data */
                if (unlikely(datalen - dp < 2))
                        goto data_overrun_error;
                tag = data[dp++];
                if (unlikely((tag & 0x1f) == ASN1_LONG_TAG))
                        goto long_tag_not_supported;

                if (op & ASN1_OP_MATCH__ANY) {
                        pr_debug("- any %02x\n", tag);
                } else {
                        /* Extract the tag from the machine
                         * - Either CONS or PRIM are permitted in the data if
                         *   CONS is not set in the op stream, otherwise CONS
                         *   is mandatory.
                         */
                        optag = machine[pc + 1];
                        flags |= optag & FLAG_CONS;

                        /* Determine whether the tag matched */
                        tmp = optag ^ tag;
                        tmp &= ~(optag & ASN1_CONS_BIT);
                        pr_debug("- match? %02x %02x %02x\n", tag, optag, tmp);
                        if (tmp != 0) {
                                /* All odd-numbered tags are MATCH_OR_SKIP. */
                                if (op & ASN1_OP_MATCH__SKIP) {
                                        pc += asn1_op_lengths[op];
                                        dp--;
                                        goto next_op;
                                }
                                goto tag_mismatch;
                        }
                }
                flags |= FLAG_MATCHED;

                len = data[dp++];
                if (len > 0x7f) {
                        if (unlikely(len == ASN1_INDEFINITE_LENGTH)) {
                                /* Indefinite length */
                                if (unlikely(!(tag & ASN1_CONS_BIT)))
                                        goto indefinite_len_primitive;
                                flags |= FLAG_INDEFINITE_LENGTH;
                                if (unlikely(2 > datalen - dp))
                                        goto data_overrun_error;
                        } else {
                                int n = len - 0x80;
                                if (unlikely(n > 2))
                                        goto length_too_long;
                                if (unlikely(n > datalen - dp))
                                        goto data_overrun_error;
                                hdr += n;
                                for (len = 0; n > 0; n--) {
                                        len <<= 8;
                                        len |= data[dp++];
                                }
                                if (unlikely(len > datalen - dp))
                                        goto data_overrun_error;
                        }
                } else {
                        if (unlikely(len > datalen - dp))
                                goto data_overrun_error;
                }

                if (flags & FLAG_CONS) {
                        /* For expected compound forms, we stack the positions
                         * of the start and end of the data.
                         */
                        if (unlikely(csp >= NR_CONS_STACK))
                                goto cons_stack_overflow;
                        cons_dp_stack[csp] = dp;
                        cons_hdrlen_stack[csp] = hdr;
                        if (!(flags & FLAG_INDEFINITE_LENGTH)) {
                                cons_datalen_stack[csp] = datalen;
                                datalen = dp + len;
                        } else {
                                cons_datalen_stack[csp] = 0;
                        }
                        csp++;
                }

                pr_debug("- TAG: %02x %zu%s\n",
                         tag, len, flags & FLAG_CONS ? " CONS" : "");
                tdp = dp;
        }

        /* Decide how to handle the operation */
        switch (op) {
        case ASN1_OP_MATCH:
        case ASN1_OP_MATCH_OR_SKIP:
        case ASN1_OP_MATCH_ACT:
        case ASN1_OP_MATCH_ACT_OR_SKIP:
        case ASN1_OP_MATCH_ANY:
        case ASN1_OP_MATCH_ANY_OR_SKIP:
        case ASN1_OP_MATCH_ANY_ACT:
        case ASN1_OP_MATCH_ANY_ACT_OR_SKIP:
        case ASN1_OP_COND_MATCH_OR_SKIP:
        case ASN1_OP_COND_MATCH_ACT_OR_SKIP:
        case ASN1_OP_COND_MATCH_ANY:
        case ASN1_OP_COND_MATCH_ANY_OR_SKIP:
        case ASN1_OP_COND_MATCH_ANY_ACT:
        case ASN1_OP_COND_MATCH_ANY_ACT_OR_SKIP:

                if (!(flags & FLAG_CONS)) {
                        if (flags & FLAG_INDEFINITE_LENGTH) {
                                size_t tmp = dp;

                                ret = asn1_find_indefinite_length(
                                        data, datalen, &tmp, &len, &errmsg);
                                if (ret < 0)
                                        goto error;
                        }
                        pr_debug("- LEAF: %zu\n", len);
                }

                if (op & ASN1_OP_MATCH__ACT) {
                        unsigned char act;

                        if (op & ASN1_OP_MATCH__ANY)
                                act = machine[pc + 1];
                        else
                                act = machine[pc + 2];
                        ret = actions[act](context, hdr, tag, data + dp, len);
                        if (ret < 0)
                                return ret;
                }

                if (!(flags & FLAG_CONS))
                        dp += len;
                pc += asn1_op_lengths[op];
                goto next_op;

        case ASN1_OP_MATCH_JUMP:
        case ASN1_OP_MATCH_JUMP_OR_SKIP:
        case ASN1_OP_COND_MATCH_JUMP_OR_SKIP:
                pr_debug("- MATCH_JUMP\n");
                if (unlikely(jsp == NR_JUMP_STACK))
                        goto jump_stack_overflow;
                jump_stack[jsp++] = pc + asn1_op_lengths[op];
                pc = machine[pc + 2];
                goto next_op;

        case ASN1_OP_COND_FAIL:
                if (unlikely(!(flags & FLAG_MATCHED)))
                        goto tag_mismatch;
                pc += asn1_op_lengths[op];
                goto next_op;

        case ASN1_OP_COMPLETE:
                if (unlikely(jsp != 0 || csp != 0)) {
                        pr_err("ASN.1 decoder error: Stacks not empty at completion (%u, %u)\n",
                               jsp, csp);
                        return -EBADMSG;
                }
                return 0;

        case ASN1_OP_END_SET:
        case ASN1_OP_END_SET_ACT:
                if (unlikely(!(flags & FLAG_MATCHED)))
                        goto tag_mismatch;
                fallthrough;

        case ASN1_OP_END_SEQ:
        case ASN1_OP_END_SET_OF:
        case ASN1_OP_END_SEQ_OF:
        case ASN1_OP_END_SEQ_ACT:
        case ASN1_OP_END_SET_OF_ACT:
        case ASN1_OP_END_SEQ_OF_ACT:
                if (unlikely(csp <= 0))
                        goto cons_stack_underflow;
                csp--;
                tdp = cons_dp_stack[csp];
                hdr = cons_hdrlen_stack[csp];
                len = datalen;
                datalen = cons_datalen_stack[csp];
                pr_debug("- end cons t=%zu dp=%zu l=%zu/%zu\n",
                         tdp, dp, len, datalen);
                if (datalen == 0) {
                        /* Indefinite length - check for the EOC. */
                        datalen = len;
                        if (unlikely(datalen - dp < 2))
                                goto data_overrun_error;
                        if (data[dp++] != 0) {
                                if (op & ASN1_OP_END__OF) {
                                        dp--;
                                        csp++;
                                        pc = machine[pc + 1];
                                        pr_debug("- continue\n");
                                        goto next_op;
                                }
                                goto missing_eoc;
                        }
                        if (data[dp++] != 0)
                                goto invalid_eoc;
                        len = dp - tdp - 2;
                } else {
                        if (dp < len && (op & ASN1_OP_END__OF)) {
                                datalen = len;
                                csp++;
                                pc = machine[pc + 1];
                                pr_debug("- continue\n");
                                goto next_op;
                        }
                        if (dp != len)
                                goto cons_length_error;
                        len -= tdp;
                        pr_debug("- cons len l=%zu d=%zu\n", len, dp - tdp);
                }

                if (op & ASN1_OP_END__ACT) {
                        unsigned char act;
                        if (op & ASN1_OP_END__OF)
                                act = machine[pc + 2];
                        else
                                act = machine[pc + 1];
                        ret = actions[act](context, hdr, 0, data + tdp, len);
                        if (ret < 0)
                                return ret;
                }
                pc += asn1_op_lengths[op];
                goto next_op;

        case ASN1_OP_MAYBE_ACT:
                if (!(flags & FLAG_LAST_MATCHED)) {
                        pc += asn1_op_lengths[op];
                        goto next_op;
                }
                fallthrough;

        case ASN1_OP_ACT:
                ret = actions[machine[pc + 1]](context, hdr, tag, data + tdp, len);
                if (ret < 0)
                        return ret;
                pc += asn1_op_lengths[op];
                goto next_op;

        case ASN1_OP_RETURN:
                if (unlikely(jsp <= 0))
                        goto jump_stack_underflow;
                pc = jump_stack[--jsp];
                flags |= FLAG_MATCHED | FLAG_LAST_MATCHED;
                goto next_op;

        default:
                break;
        }

        /* Shouldn't reach here */
        pr_err("ASN.1 decoder error: Found reserved opcode (%u) pc=%zu\n",
               op, pc);
        return -EBADMSG;

data_overrun_error:
        errmsg = "Data overrun error";
        goto error;
machine_overrun_error:
        errmsg = "Machine overrun error";
        goto error;
jump_stack_underflow:
        errmsg = "Jump stack underflow";
        goto error;
jump_stack_overflow:
        errmsg = "Jump stack overflow";
        goto error;
cons_stack_underflow:
        errmsg = "Cons stack underflow";
        goto error;
cons_stack_overflow:
        errmsg = "Cons stack overflow";
        goto error;
cons_length_error:
        errmsg = "Cons length error";
        goto error;
missing_eoc:
        errmsg = "Missing EOC in indefinite len cons";
        goto error;
invalid_eoc:
        errmsg = "Invalid length EOC";
        goto error;
length_too_long:
        errmsg = "Unsupported length";
        goto error;
indefinite_len_primitive:
        errmsg = "Indefinite len primitive not permitted";
        goto error;
tag_mismatch:
        errmsg = "Unexpected tag";
        goto error;
long_tag_not_supported:
        errmsg = "Long tag not supported";
error:
        pr_debug("\nASN1: %s [m=%zu d=%zu ot=%02x t=%02x l=%zu]\n",
                 errmsg, pc, dp, optag, tag, len);
        return -EBADMSG;
}
EXPORT_SYMBOL_GPL(asn1_ber_decoder);

MODULE_LICENSE("GPL");




































































































































































































































































































































































































































































    4 
    1 














    3 















    2 
    1 
















































































































































































































































































































































































































































    5 



    5 
    2 







    3 













    3 








    2 






    1 




    2 




    2 



























    2 






    1 

    1 



    1 







    1 























    2 





















    2 



    1 
    1 










    2 

















































































































































































































































































































































































    5 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *                sonix sn9c102 (bayer) library
 *
 * Copyright (C) 2009-2011 Jean-François Moine <http://moinejf.free.fr>
 * Copyright (C) 2003 2004 Michel Xhaard mxhaard@magic.fr
 * Add Pas106 Stefano Mozzi (C) 2004
 */

/* Some documentation on known sonixb registers:

Reg        Use
sn9c101 / sn9c102:
0x10        high nibble red gain low nibble blue gain
0x11        low nibble green gain
sn9c103:
0x05        red gain 0-127
0x06        blue gain 0-127
0x07        green gain 0-127
all:
0x08-0x0f i2c / 3wire registers
0x12        hstart
0x13        vstart
0x15        hsize (hsize = register-value * 16)
0x16        vsize (vsize = register-value * 16)
0x17        bit 0 toggle compression quality (according to sn9c102 driver)
0x18        bit 7 enables compression, bit 4-5 set image down scaling:
        00 scale 1, 01 scale 1/2, 10, scale 1/4
0x19        high-nibble is sensor clock divider, changes exposure on sensors which
        use a clock generated by the bridge. Some sensors have their own clock.
0x1c        auto_exposure area (for avg_lum) startx (startx = register-value * 32)
0x1d        auto_exposure area (for avg_lum) starty (starty = register-value * 32)
0x1e        auto_exposure area (for avg_lum) stopx (hsize = (0x1e - 0x1c) * 32)
0x1f        auto_exposure area (for avg_lum) stopy (vsize = (0x1f - 0x1d) * 32)
*/

#define MODULE_NAME "sonixb"

#include <linux/input.h>
#include "gspca.h"

MODULE_AUTHOR("Jean-François Moine <http://moinejf.free.fr>");
MODULE_DESCRIPTION("GSPCA/SN9C102 USB Camera Driver");
MODULE_LICENSE("GPL");

/* specific webcam descriptor */
struct sd {
        struct gspca_dev gspca_dev;        /* !! must be the first item */

        struct v4l2_ctrl *brightness;
        struct v4l2_ctrl *plfreq;

        atomic_t avg_lum;
        int prev_avg_lum;
        int exposure_knee;
        int header_read;
        u8 header[12]; /* Header without sof marker */

        unsigned char autogain_ignore_frames;
        unsigned char frames_to_drop;

        __u8 bridge;                        /* Type of bridge */
#define BRIDGE_101 0
#define BRIDGE_102 0 /* We make no difference between 101 and 102 */
#define BRIDGE_103 1

        __u8 sensor;                        /* Type of image sensor chip */
#define SENSOR_HV7131D 0
#define SENSOR_HV7131R 1
#define SENSOR_OV6650 2
#define SENSOR_OV7630 3
#define SENSOR_PAS106 4
#define SENSOR_PAS202 5
#define SENSOR_TAS5110C 6
#define SENSOR_TAS5110D 7
#define SENSOR_TAS5130CXX 8
        __u8 reg11;
};

typedef const __u8 sensor_init_t[8];

struct sensor_data {
        const __u8 *bridge_init;
        sensor_init_t *sensor_init;
        int sensor_init_size;
        int flags;
        __u8 sensor_addr;
};

/* sensor_data flags */
#define F_SIF                0x01        /* sif or vga */

/* priv field of struct v4l2_pix_format flags (do not use low nibble!) */
#define MODE_RAW 0x10                /* raw bayer mode */
#define MODE_REDUCED_SIF 0x20        /* vga mode (320x240 / 160x120) on sif cam */

#define COMP 0xc7                /* 0x87 //0x07 */
#define COMP1 0xc9                /* 0x89 //0x09 */

#define MCK_INIT 0x63
#define MCK_INIT1 0x20                /*fixme: Bayer - 0x50 for JPEG ??*/

#define SYS_CLK 0x04

#define SENS(bridge, sensor, _flags, _sensor_addr) \
{ \
        .bridge_init = bridge, \
        .sensor_init = sensor, \
        .sensor_init_size = sizeof(sensor), \
        .flags = _flags, .sensor_addr = _sensor_addr \
}

/* We calculate the autogain at the end of the transfer of a frame, at this
   moment a frame with the old settings is being captured and transmitted. So
   if we adjust the gain or exposure we must ignore at least the next frame for
   the new settings to come into effect before doing any other adjustments. */
#define AUTOGAIN_IGNORE_FRAMES 1

static const struct v4l2_pix_format vga_mode[] = {
        {160, 120, V4L2_PIX_FMT_SBGGR8, V4L2_FIELD_NONE,
                .bytesperline = 160,
                .sizeimage = 160 * 120,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 2 | MODE_RAW},
        {160, 120, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 160,
                .sizeimage = 160 * 120 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 2},
        {320, 240, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 320,
                .sizeimage = 320 * 240 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 1},
        {640, 480, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 640,
                .sizeimage = 640 * 480 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 0},
};
static const struct v4l2_pix_format sif_mode[] = {
        {160, 120, V4L2_PIX_FMT_SBGGR8, V4L2_FIELD_NONE,
                .bytesperline = 160,
                .sizeimage = 160 * 120,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 1 | MODE_RAW | MODE_REDUCED_SIF},
        {160, 120, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 160,
                .sizeimage = 160 * 120 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 1 | MODE_REDUCED_SIF},
        {176, 144, V4L2_PIX_FMT_SBGGR8, V4L2_FIELD_NONE,
                .bytesperline = 176,
                .sizeimage = 176 * 144,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 1 | MODE_RAW},
        {176, 144, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 176,
                .sizeimage = 176 * 144 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 1},
        {320, 240, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 320,
                .sizeimage = 320 * 240 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 0 | MODE_REDUCED_SIF},
        {352, 288, V4L2_PIX_FMT_SN9C10X, V4L2_FIELD_NONE,
                .bytesperline = 352,
                .sizeimage = 352 * 288 * 5 / 4,
                .colorspace = V4L2_COLORSPACE_SRGB,
                .priv = 0},
};

static const __u8 initHv7131d[] = {
        0x04, 0x03, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x02, 0x02, 0x00,
        0x28, 0x1e, 0x60, 0x8e, 0x42,
};
static const __u8 hv7131d_sensor_init[][8] = {
        {0xa0, 0x11, 0x01, 0x04, 0x00, 0x00, 0x00, 0x17},
        {0xa0, 0x11, 0x02, 0x00, 0x00, 0x00, 0x00, 0x17},
        {0xa0, 0x11, 0x28, 0x00, 0x00, 0x00, 0x00, 0x17},
        {0xa0, 0x11, 0x30, 0x30, 0x00, 0x00, 0x00, 0x17}, /* reset level */
        {0xa0, 0x11, 0x34, 0x02, 0x00, 0x00, 0x00, 0x17}, /* pixel bias volt */
};

static const __u8 initHv7131r[] = {
        0x46, 0x77, 0x00, 0x04, 0x00, 0x00, 0x00, 0x80, 0x11, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x02, 0x01, 0x00,
        0x28, 0x1e, 0x60, 0x8a, 0x20,
};
static const __u8 hv7131r_sensor_init[][8] = {
        {0xc0, 0x11, 0x31, 0x38, 0x2a, 0x2e, 0x00, 0x10},
        {0xa0, 0x11, 0x01, 0x08, 0x2a, 0x2e, 0x00, 0x10},
        {0xb0, 0x11, 0x20, 0x00, 0xd0, 0x2e, 0x00, 0x10},
        {0xc0, 0x11, 0x25, 0x03, 0x0e, 0x28, 0x00, 0x16},
        {0xa0, 0x11, 0x30, 0x10, 0x0e, 0x28, 0x00, 0x15},
};
static const __u8 initOv6650[] = {
        0x44, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
        0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
        0x00, 0x01, 0x01, 0x0a, 0x16, 0x12, 0x68, 0x8b,
        0x10,
};
static const __u8 ov6650_sensor_init[][8] = {
        /* Bright, contrast, etc are set through SCBB interface.
         * AVCAP on win2 do not send any data on this controls. */
        /* Anyway, some registers appears to alter bright and constrat */

        /* Reset sensor */
        {0xa0, 0x60, 0x12, 0x80, 0x00, 0x00, 0x00, 0x10},
        /* Set clock register 0x11 low nibble is clock divider */
        {0xd0, 0x60, 0x11, 0xc0, 0x1b, 0x18, 0xc1, 0x10},
        /* Next some unknown stuff */
        {0xb0, 0x60, 0x15, 0x00, 0x02, 0x18, 0xc1, 0x10},
/*        {0xa0, 0x60, 0x1b, 0x01, 0x02, 0x18, 0xc1, 0x10},
                 * THIS SET GREEN SCREEN
                 * (pixels could be innverted in decode kind of "brg",
                 * but blue wont be there. Avoid this data ... */
        {0xd0, 0x60, 0x26, 0x01, 0x14, 0xd8, 0xa4, 0x10}, /* format out? */
        {0xd0, 0x60, 0x26, 0x01, 0x14, 0xd8, 0xa4, 0x10},
        {0xa0, 0x60, 0x30, 0x3d, 0x0a, 0xd8, 0xa4, 0x10},
        /* Enable rgb brightness control */
        {0xa0, 0x60, 0x61, 0x08, 0x00, 0x00, 0x00, 0x10},
        /* HDG: Note windows uses the line below, which sets both register 0x60
           and 0x61 I believe these registers of the ov6650 are identical as
           those of the ov7630, because if this is true the windows settings
           add a bit additional red gain and a lot additional blue gain, which
           matches my findings that the windows settings make blue much too
           blue and red a little too red.
        {0xb0, 0x60, 0x60, 0x66, 0x68, 0xd8, 0xa4, 0x10}, */
        /* Some more unknown stuff */
        {0xa0, 0x60, 0x68, 0x04, 0x68, 0xd8, 0xa4, 0x10},
        {0xd0, 0x60, 0x17, 0x24, 0xd6, 0x04, 0x94, 0x10}, /* Clipreg */
};

static const __u8 initOv7630[] = {
        0x04, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,        /* r01 .. r08 */
        0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,        /* r09 .. r10 */
        0x00, 0x01, 0x01, 0x0a,                                /* r11 .. r14 */
        0x28, 0x1e,                        /* H & V sizes     r15 .. r16 */
        0x68, 0x8f, MCK_INIT1,                                /* r17 .. r19 */
};
static const __u8 ov7630_sensor_init[][8] = {
        {0xa0, 0x21, 0x12, 0x80, 0x00, 0x00, 0x00, 0x10},
        {0xb0, 0x21, 0x01, 0x77, 0x3a, 0x00, 0x00, 0x10},
/*        {0xd0, 0x21, 0x12, 0x7c, 0x01, 0x80, 0x34, 0x10},           jfm */
        {0xd0, 0x21, 0x12, 0x5c, 0x00, 0x80, 0x34, 0x10},        /* jfm */
        {0xa0, 0x21, 0x1b, 0x04, 0x00, 0x80, 0x34, 0x10},
        {0xa0, 0x21, 0x20, 0x44, 0x00, 0x80, 0x34, 0x10},
        {0xa0, 0x21, 0x23, 0xee, 0x00, 0x80, 0x34, 0x10},
        {0xd0, 0x21, 0x26, 0xa0, 0x9a, 0xa0, 0x30, 0x10},
        {0xb0, 0x21, 0x2a, 0x80, 0x00, 0xa0, 0x30, 0x10},
        {0xb0, 0x21, 0x2f, 0x3d, 0x24, 0xa0, 0x30, 0x10},
        {0xa0, 0x21, 0x32, 0x86, 0x24, 0xa0, 0x30, 0x10},
        {0xb0, 0x21, 0x60, 0xa9, 0x4a, 0xa0, 0x30, 0x10},
/*        {0xb0, 0x21, 0x60, 0xa9, 0x42, 0xa0, 0x30, 0x10},         * jfm */
        {0xa0, 0x21, 0x65, 0x00, 0x42, 0xa0, 0x30, 0x10},
        {0xa0, 0x21, 0x69, 0x38, 0x42, 0xa0, 0x30, 0x10},
        {0xc0, 0x21, 0x6f, 0x88, 0x0b, 0x00, 0x30, 0x10},
        {0xc0, 0x21, 0x74, 0x21, 0x8e, 0x00, 0x30, 0x10},
        {0xa0, 0x21, 0x7d, 0xf7, 0x8e, 0x00, 0x30, 0x10},
        {0xd0, 0x21, 0x17, 0x1c, 0xbd, 0x06, 0xf6, 0x10},
};

static const __u8 initPas106[] = {
        0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x40, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x04, 0x01, 0x00,
        0x16, 0x12, 0x24, COMP1, MCK_INIT1,
};
/* compression 0x86 mckinit1 0x2b */

/* "Known" PAS106B registers:
  0x02 clock divider
  0x03 Variable framerate bits 4-11
  0x04 Var framerate bits 0-3, one must leave the 4 msb's at 0 !!
       The variable framerate control must never be set lower then 300,
       which sets the framerate at 90 / reg02, otherwise vsync is lost.
  0x05 Shutter Time Line Offset, this can be used as an exposure control:
       0 = use full frame time, 255 = no exposure at all
       Note this may never be larger then "var-framerate control" / 2 - 2.
       When var-framerate control is < 514, no exposure is reached at the max
       allowed value for the framerate control value, rather then at 255.
  0x06 Shutter Time Pixel Offset, like reg05 this influences exposure, but
       only a very little bit, leave at 0xcd
  0x07 offset sign bit (bit0 1 > negative offset)
  0x08 offset
  0x09 Blue Gain
  0x0a Green1 Gain
  0x0b Green2 Gain
  0x0c Red Gain
  0x0e Global gain
  0x13 Write 1 to commit settings to sensor
*/

static const __u8 pas106_sensor_init[][8] = {
        /* Pixel Clock Divider 6 */
        { 0xa1, 0x40, 0x02, 0x04, 0x00, 0x00, 0x00, 0x14 },
        /* Frame Time MSB (also seen as 0x12) */
        { 0xa1, 0x40, 0x03, 0x13, 0x00, 0x00, 0x00, 0x14 },
        /* Frame Time LSB (also seen as 0x05) */
        { 0xa1, 0x40, 0x04, 0x06, 0x00, 0x00, 0x00, 0x14 },
        /* Shutter Time Line Offset (also seen as 0x6d) */
        { 0xa1, 0x40, 0x05, 0x65, 0x00, 0x00, 0x00, 0x14 },
        /* Shutter Time Pixel Offset (also seen as 0xb1) */
        { 0xa1, 0x40, 0x06, 0xcd, 0x00, 0x00, 0x00, 0x14 },
        /* Black Level Subtract Sign (also seen 0x00) */
        { 0xa1, 0x40, 0x07, 0xc1, 0x00, 0x00, 0x00, 0x14 },
        /* Black Level Subtract Level (also seen 0x01) */
        { 0xa1, 0x40, 0x08, 0x06, 0x00, 0x00, 0x00, 0x14 },
        { 0xa1, 0x40, 0x08, 0x06, 0x00, 0x00, 0x00, 0x14 },
        /* Color Gain B Pixel 5 a */
        { 0xa1, 0x40, 0x09, 0x05, 0x00, 0x00, 0x00, 0x14 },
        /* Color Gain G1 Pixel 1 5 */
        { 0xa1, 0x40, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x14 },
        /* Color Gain G2 Pixel 1 0 5 */
        { 0xa1, 0x40, 0x0b, 0x04, 0x00, 0x00, 0x00, 0x14 },
        /* Color Gain R Pixel 3 1 */
        { 0xa1, 0x40, 0x0c, 0x05, 0x00, 0x00, 0x00, 0x14 },
        /* Color GainH  Pixel */
        { 0xa1, 0x40, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x14 },
        /* Global Gain */
        { 0xa1, 0x40, 0x0e, 0x0e, 0x00, 0x00, 0x00, 0x14 },
        /* Contrast */
        { 0xa1, 0x40, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x14 },
        /* H&V synchro polarity */
        { 0xa1, 0x40, 0x10, 0x06, 0x00, 0x00, 0x00, 0x14 },
        /* ?default */
        { 0xa1, 0x40, 0x11, 0x06, 0x00, 0x00, 0x00, 0x14 },
        /* DAC scale */
        { 0xa1, 0x40, 0x12, 0x06, 0x00, 0x00, 0x00, 0x14 },
        /* ?default */
        { 0xa1, 0x40, 0x14, 0x02, 0x00, 0x00, 0x00, 0x14 },
        /* Validate Settings */
        { 0xa1, 0x40, 0x13, 0x01, 0x00, 0x00, 0x00, 0x14 },
};

static const __u8 initPas202[] = {
        0x44, 0x44, 0x21, 0x30, 0x00, 0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x06, 0x03, 0x0a,
        0x28, 0x1e, 0x20, 0x89, 0x20,
};

/* "Known" PAS202BCB registers:
  0x02 clock divider
  0x04 Variable framerate bits 6-11 (*)
  0x05 Var framerate  bits 0-5, one must leave the 2 msb's at 0 !!
  0x07 Blue Gain
  0x08 Green Gain
  0x09 Red Gain
  0x0b offset sign bit (bit0 1 > negative offset)
  0x0c offset
  0x0e Unknown image is slightly brighter when bit 0 is 0, if reg0f is 0 too,
       leave at 1 otherwise we get a jump in our exposure control
  0x0f Exposure 0-255, 0 = use full frame time, 255 = no exposure at all
  0x10 Master gain 0 - 31
  0x11 write 1 to apply changes
  (*) The variable framerate control must never be set lower then 500
      which sets the framerate at 30 / reg02, otherwise vsync is lost.
*/
static const __u8 pas202_sensor_init[][8] = {
        /* Set the clock divider to 4 -> 30 / 4 = 7.5 fps, we would like
           to set it lower, but for some reason the bridge starts missing
           vsync's then */
        {0xa0, 0x40, 0x02, 0x04, 0x00, 0x00, 0x00, 0x10},
        {0xd0, 0x40, 0x04, 0x07, 0x34, 0x00, 0x09, 0x10},
        {0xd0, 0x40, 0x08, 0x01, 0x00, 0x00, 0x01, 0x10},
        {0xd0, 0x40, 0x0c, 0x00, 0x0c, 0x01, 0x32, 0x10},
        {0xd0, 0x40, 0x10, 0x00, 0x01, 0x00, 0x63, 0x10},
        {0xa0, 0x40, 0x15, 0x70, 0x01, 0x00, 0x63, 0x10},
        {0xa0, 0x40, 0x18, 0x00, 0x01, 0x00, 0x63, 0x10},
        {0xa0, 0x40, 0x11, 0x01, 0x01, 0x00, 0x63, 0x10},
        {0xa0, 0x40, 0x03, 0x56, 0x01, 0x00, 0x63, 0x10},
        {0xa0, 0x40, 0x11, 0x01, 0x01, 0x00, 0x63, 0x10},
};

static const __u8 initTas5110c[] = {
        0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x11, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x45, 0x09, 0x0a,
        0x16, 0x12, 0x60, 0x86, 0x2b,
};
/* Same as above, except a different hstart */
static const __u8 initTas5110d[] = {
        0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x11, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x41, 0x09, 0x0a,
        0x16, 0x12, 0x60, 0x86, 0x2b,
};
/* tas5110c is 3 wire, tas5110d is 2 wire (regular i2c) */
static const __u8 tas5110c_sensor_init[][8] = {
        {0x30, 0x11, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x10},
        {0x30, 0x11, 0x02, 0x20, 0xa9, 0x00, 0x00, 0x10},
};
/* Known TAS5110D registers
 * reg02: gain, bit order reversed!! 0 == max gain, 255 == min gain
 * reg03: bit3: vflip, bit4: ~hflip, bit7: ~gainboost (~ == inverted)
 *        Note: writing reg03 seems to only work when written together with 02
 */
static const __u8 tas5110d_sensor_init[][8] = {
        {0xa0, 0x61, 0x9a, 0xca, 0x00, 0x00, 0x00, 0x17}, /* reset */
};

static const __u8 initTas5130[] = {
        0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x11, 0x00, 0x00, 0x00,
        0x00, 0x00,
        0x00, 0x00, 0x00, 0x68, 0x0c, 0x0a,
        0x28, 0x1e, 0x60, COMP, MCK_INIT,
};
static const __u8 tas5130_sensor_init[][8] = {
/*        {0x30, 0x11, 0x00, 0x40, 0x47, 0x00, 0x00, 0x10},
                                        * shutter 0x47 short exposure? */
        {0x30, 0x11, 0x00, 0x40, 0x01, 0x00, 0x00, 0x10},
                                        /* shutter 0x01 long exposure */
        {0x30, 0x11, 0x02, 0x20, 0x70, 0x00, 0x00, 0x10},
};

static const struct sensor_data sensor_data[] = {
        SENS(initHv7131d, hv7131d_sensor_init, 0, 0),
        SENS(initHv7131r, hv7131r_sensor_init, 0, 0),
        SENS(initOv6650, ov6650_sensor_init, F_SIF, 0x60),
        SENS(initOv7630, ov7630_sensor_init, 0, 0x21),
        SENS(initPas106, pas106_sensor_init, F_SIF, 0),
        SENS(initPas202, pas202_sensor_init, 0, 0),
        SENS(initTas5110c, tas5110c_sensor_init, F_SIF, 0),
        SENS(initTas5110d, tas5110d_sensor_init, F_SIF, 0),
        SENS(initTas5130, tas5130_sensor_init, 0, 0),
};

/* get one byte in gspca_dev->usb_buf */
static void reg_r(struct gspca_dev *gspca_dev,
                  __u16 value)
{
        int res;

        if (gspca_dev->usb_err < 0)
                return;

        res = usb_control_msg(gspca_dev->dev,
                        usb_rcvctrlpipe(gspca_dev->dev, 0),
                        0,                        /* request */
                        USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
                        value,
                        0,                        /* index */
                        gspca_dev->usb_buf, 1,
                        500);

        if (res < 0) {
                dev_err(gspca_dev->v4l2_dev.dev,
                        "Error reading register %02x: %d\n", value, res);
                gspca_dev->usb_err = res;
                /*
                 * Make sure the result is zeroed to avoid uninitialized
                 * values.
                 */
                gspca_dev->usb_buf[0] = 0;
        }
}

static void reg_w(struct gspca_dev *gspca_dev,
                  __u16 value,
                  const __u8 *buffer,
                  int len)
{
        int res;

        if (gspca_dev->usb_err < 0)
                return;

        memcpy(gspca_dev->usb_buf, buffer, len);
        res = usb_control_msg(gspca_dev->dev,
                        usb_sndctrlpipe(gspca_dev->dev, 0),
                        0x08,                        /* request */
                        USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
                        value,
                        0,                        /* index */
                        gspca_dev->usb_buf, len,
                        500);

        if (res < 0) {
                dev_err(gspca_dev->v4l2_dev.dev,
                        "Error writing register %02x: %d\n", value, res);
                gspca_dev->usb_err = res;
        }
}

static void i2c_w(struct gspca_dev *gspca_dev, const u8 *buf)
{
        int retry = 60;

        if (gspca_dev->usb_err < 0)
                return;

        /* is i2c ready */
        reg_w(gspca_dev, 0x08, buf, 8);
        while (retry--) {
                if (gspca_dev->usb_err < 0)
                        return;
                msleep(1);
                reg_r(gspca_dev, 0x08);
                if (gspca_dev->usb_buf[0] & 0x04) {
                        if (gspca_dev->usb_buf[0] & 0x08) {
                                dev_err(gspca_dev->v4l2_dev.dev,
                                        "i2c error writing %8ph\n", buf);
                                gspca_dev->usb_err = -EIO;
                        }
                        return;
                }
        }

        dev_err(gspca_dev->v4l2_dev.dev, "i2c write timeout\n");
        gspca_dev->usb_err = -EIO;
}

static void i2c_w_vector(struct gspca_dev *gspca_dev,
                        const __u8 buffer[][8], int len)
{
        for (;;) {
                if (gspca_dev->usb_err < 0)
                        return;
                i2c_w(gspca_dev, *buffer);
                len -= 8;
                if (len <= 0)
                        break;
                buffer++;
        }
}

static void setbrightness(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        switch (sd->sensor) {
        case  SENSOR_OV6650:
        case  SENSOR_OV7630: {
                __u8 i2cOV[] =
                        {0xa0, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x10};

                /* change reg 0x06 */
                i2cOV[1] = sensor_data[sd->sensor].sensor_addr;
                i2cOV[3] = sd->brightness->val;
                i2c_w(gspca_dev, i2cOV);
                break;
        }
        case SENSOR_PAS106:
        case SENSOR_PAS202: {
                __u8 i2cpbright[] =
                        {0xb0, 0x40, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x16};
                __u8 i2cpdoit[] =
                        {0xa0, 0x40, 0x11, 0x01, 0x00, 0x00, 0x00, 0x16};

                /* PAS106 uses reg 7 and 8 instead of b and c */
                if (sd->sensor == SENSOR_PAS106) {
                        i2cpbright[2] = 7;
                        i2cpdoit[2] = 0x13;
                }

                if (sd->brightness->val < 127) {
                        /* change reg 0x0b, signreg */
                        i2cpbright[3] = 0x01;
                        /* set reg 0x0c, offset */
                        i2cpbright[4] = 127 - sd->brightness->val;
                } else
                        i2cpbright[4] = sd->brightness->val - 127;

                i2c_w(gspca_dev, i2cpbright);
                i2c_w(gspca_dev, i2cpdoit);
                break;
        }
        default:
                break;
        }
}

static void setgain(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        u8 gain = gspca_dev->gain->val;

        switch (sd->sensor) {
        case SENSOR_HV7131D: {
                __u8 i2c[] =
                        {0xc0, 0x11, 0x31, 0x00, 0x00, 0x00, 0x00, 0x17};

                i2c[3] = 0x3f - gain;
                i2c[4] = 0x3f - gain;
                i2c[5] = 0x3f - gain;

                i2c_w(gspca_dev, i2c);
                break;
        }
        case SENSOR_TAS5110C:
        case SENSOR_TAS5130CXX: {
                __u8 i2c[] =
                        {0x30, 0x11, 0x02, 0x20, 0x70, 0x00, 0x00, 0x10};

                i2c[4] = 255 - gain;
                i2c_w(gspca_dev, i2c);
                break;
        }
        case SENSOR_TAS5110D: {
                __u8 i2c[] = {
                        0xb0, 0x61, 0x02, 0x00, 0x10, 0x00, 0x00, 0x17 };
                gain = 255 - gain;
                /* The bits in the register are the wrong way around!! */
                i2c[3] |= (gain & 0x80) >> 7;
                i2c[3] |= (gain & 0x40) >> 5;
                i2c[3] |= (gain & 0x20) >> 3;
                i2c[3] |= (gain & 0x10) >> 1;
                i2c[3] |= (gain & 0x08) << 1;
                i2c[3] |= (gain & 0x04) << 3;
                i2c[3] |= (gain & 0x02) << 5;
                i2c[3] |= (gain & 0x01) << 7;
                i2c_w(gspca_dev, i2c);
                break;
        }
        case SENSOR_OV6650:
        case SENSOR_OV7630: {
                __u8 i2c[] = {0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10};

                /*
                 * The ov7630's gain is weird, at 32 the gain drops to the
                 * same level as at 16, so skip 32-47 (of the 0-63 scale).
                 */
                if (sd->sensor == SENSOR_OV7630 && gain >= 32)
                        gain += 16;

                i2c[1] = sensor_data[sd->sensor].sensor_addr;
                i2c[3] = gain;
                i2c_w(gspca_dev, i2c);
                break;
        }
        case SENSOR_PAS106:
        case SENSOR_PAS202: {
                __u8 i2cpgain[] =
                        {0xa0, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x15};
                __u8 i2cpcolorgain[] =
                        {0xc0, 0x40, 0x07, 0x00, 0x00, 0x00, 0x00, 0x15};
                __u8 i2cpdoit[] =
                        {0xa0, 0x40, 0x11, 0x01, 0x00, 0x00, 0x00, 0x16};

                /* PAS106 uses different regs (and has split green gains) */
                if (sd->sensor == SENSOR_PAS106) {
                        i2cpgain[2] = 0x0e;
                        i2cpcolorgain[0] = 0xd0;
                        i2cpcolorgain[2] = 0x09;
                        i2cpdoit[2] = 0x13;
                }

                i2cpgain[3] = gain;
                i2cpcolorgain[3] = gain >> 1;
                i2cpcolorgain[4] = gain >> 1;
                i2cpcolorgain[5] = gain >> 1;
                i2cpcolorgain[6] = gain >> 1;

                i2c_w(gspca_dev, i2cpgain);
                i2c_w(gspca_dev, i2cpcolorgain);
                i2c_w(gspca_dev, i2cpdoit);
                break;
        }
        default:
                if (sd->bridge == BRIDGE_103) {
                        u8 buf[3] = { gain, gain, gain }; /* R, G, B */
                        reg_w(gspca_dev, 0x05, buf, 3);
                } else {
                        u8 buf[2];
                        buf[0] = gain << 4 | gain; /* Red and blue */
                        buf[1] = gain; /* Green */
                        reg_w(gspca_dev, 0x10, buf, 2);
                }
        }
}

static void setexposure(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        switch (sd->sensor) {
        case SENSOR_HV7131D: {
                /* Note the datasheet wrongly says line mode exposure uses reg
                   0x26 and 0x27, testing has shown 0x25 + 0x26 */
                __u8 i2c[] = {0xc0, 0x11, 0x25, 0x00, 0x00, 0x00, 0x00, 0x17};
                u16 reg = gspca_dev->exposure->val;

                i2c[3] = reg >> 8;
                i2c[4] = reg & 0xff;
                i2c_w(gspca_dev, i2c);
                break;
        }
        case SENSOR_TAS5110C:
        case SENSOR_TAS5110D: {
                /* register 19's high nibble contains the sn9c10x clock divider
                   The high nibble configures the no fps according to the
                   formula: 60 / high_nibble. With a maximum of 30 fps */
                u8 reg = gspca_dev->exposure->val;

                reg = (reg << 4) | 0x0b;
                reg_w(gspca_dev, 0x19, &reg, 1);
                break;
        }
        case SENSOR_OV6650:
        case SENSOR_OV7630: {
                /* The ov6650 / ov7630 have 2 registers which both influence
                   exposure, register 11, whose low nibble sets the nr off fps
                   according to: fps = 30 / (low_nibble + 1)

                   The fps configures the maximum exposure setting, but it is
                   possible to use less exposure then what the fps maximum
                   allows by setting register 10. register 10 configures the
                   actual exposure as quotient of the full exposure, with 0
                   being no exposure at all (not very useful) and reg10_max
                   being max exposure possible at that framerate.

                   The code maps our 0 - 510 ms exposure ctrl to these 2
                   registers, trying to keep fps as high as possible.
                */
                __u8 i2c[] = {0xb0, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x10};
                int reg10, reg11, reg10_max;

                /* ov6645 datasheet says reg10_max is 9a, but that uses
                   tline * 2 * reg10 as formula for calculating texpo, the
                   ov6650 probably uses the same formula as the 7730 which uses
                   tline * 4 * reg10, which explains why the reg10max we've
                   found experimentally for the ov6650 is exactly half that of
                   the ov6645. The ov7630 datasheet says the max is 0x41. */
                if (sd->sensor == SENSOR_OV6650) {
                        reg10_max = 0x4d;
                        i2c[4] = 0xc0; /* OV6650 needs non default vsync pol */
                } else
                        reg10_max = 0x41;

                reg11 = (15 * gspca_dev->exposure->val + 999) / 1000;
                if (reg11 < 1)
                        reg11 = 1;
                else if (reg11 > 16)
                        reg11 = 16;

                /* In 640x480, if the reg11 has less than 4, the image is
                   unstable (the bridge goes into a higher compression mode
                   which we have not reverse engineered yet). */
                if (gspca_dev->pixfmt.width == 640 && reg11 < 4)
                        reg11 = 4;

                /* frame exposure time in ms = 1000 * reg11 / 30    ->
                reg10 = (gspca_dev->exposure->val / 2) * reg10_max
                                / (1000 * reg11 / 30) */
                reg10 = (gspca_dev->exposure->val * 15 * reg10_max)
                                / (1000 * reg11);

                /* Don't allow this to get below 10 when using autogain, the
                   steps become very large (relatively) when below 10 causing
                   the image to oscillate from much too dark, to much too bright
                   and back again. */
                if (gspca_dev->autogain->val && reg10 < 10)
                        reg10 = 10;
                else if (reg10 > reg10_max)
                        reg10 = reg10_max;

                /* Write reg 10 and reg11 low nibble */
                i2c[1] = sensor_data[sd->sensor].sensor_addr;
                i2c[3] = reg10;
                i2c[4] |= reg11 - 1;

                /* If register 11 didn't change, don't change it */
                if (sd->reg11 == reg11)
                        i2c[0] = 0xa0;

                i2c_w(gspca_dev, i2c);
                if (gspca_dev->usb_err == 0)
                        sd->reg11 = reg11;
                break;
        }
        case SENSOR_PAS202: {
                __u8 i2cpframerate[] =
                        {0xb0, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x16};
                __u8 i2cpexpo[] =
                        {0xa0, 0x40, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x16};
                const __u8 i2cpdoit[] =
                        {0xa0, 0x40, 0x11, 0x01, 0x00, 0x00, 0x00, 0x16};
                int framerate_ctrl;

                /* The exposure knee for the autogain algorithm is 200
                   (100 ms / 10 fps on other sensors), for values below this
                   use the control for setting the partial frame expose time,
                   above that use variable framerate. This way we run at max
                   framerate (640x480@7.5 fps, 320x240@10fps) until the knee
                   is reached. Using the variable framerate control above 200
                   is better then playing around with both clockdiv + partial
                   frame exposure times (like we are doing with the ov chips),
                   as that sometimes leads to jumps in the exposure control,
                   which are bad for auto exposure. */
                if (gspca_dev->exposure->val < 200) {
                        i2cpexpo[3] = 255 - (gspca_dev->exposure->val * 255)
                                                / 200;
                        framerate_ctrl = 500;
                } else {
                        /* The PAS202's exposure control goes from 0 - 4095,
                           but anything below 500 causes vsync issues, so scale
                           our 200-1023 to 500-4095 */
                        framerate_ctrl = (gspca_dev->exposure->val - 200)
                                                        * 1000 / 229 +  500;
                }

                i2cpframerate[3] = framerate_ctrl >> 6;
                i2cpframerate[4] = framerate_ctrl & 0x3f;
                i2c_w(gspca_dev, i2cpframerate);
                i2c_w(gspca_dev, i2cpexpo);
                i2c_w(gspca_dev, i2cpdoit);
                break;
        }
        case SENSOR_PAS106: {
                __u8 i2cpframerate[] =
                        {0xb1, 0x40, 0x03, 0x00, 0x00, 0x00, 0x00, 0x14};
                __u8 i2cpexpo[] =
                        {0xa1, 0x40, 0x05, 0x00, 0x00, 0x00, 0x00, 0x14};
                const __u8 i2cpdoit[] =
                        {0xa1, 0x40, 0x13, 0x01, 0x00, 0x00, 0x00, 0x14};
                int framerate_ctrl;

                /* For values below 150 use partial frame exposure, above
                   that use framerate ctrl */
                if (gspca_dev->exposure->val < 150) {
                        i2cpexpo[3] = 150 - gspca_dev->exposure->val;
                        framerate_ctrl = 300;
                } else {
                        /* The PAS106's exposure control goes from 0 - 4095,
                           but anything below 300 causes vsync issues, so scale
                           our 150-1023 to 300-4095 */
                        framerate_ctrl = (gspca_dev->exposure->val - 150)
                                                * 1000 / 230 + 300;
                }

                i2cpframerate[3] = framerate_ctrl >> 4;
                i2cpframerate[4] = framerate_ctrl & 0x0f;
                i2c_w(gspca_dev, i2cpframerate);
                i2c_w(gspca_dev, i2cpexpo);
                i2c_w(gspca_dev, i2cpdoit);
                break;
        }
        default:
                break;
        }
}

static void setfreq(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->sensor == SENSOR_OV6650 || sd->sensor == SENSOR_OV7630) {
                /* Framerate adjust register for artificial light 50 hz flicker
                   compensation, for the ov6650 this is identical to ov6630
                   0x2b register, see ov6630 datasheet.
                   0x4f / 0x8a -> (30 fps -> 25 fps), 0x00 -> no adjustment */
                __u8 i2c[] = {0xa0, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x10};
                switch (sd->plfreq->val) {
                default:
/*                case 0:                         * no filter*/
/*                case 2:                         * 60 hz */
                        i2c[3] = 0;
                        break;
                case 1:                        /* 50 hz */
                        i2c[3] = (sd->sensor == SENSOR_OV6650)
                                        ? 0x4f : 0x8a;
                        break;
                }
                i2c[1] = sensor_data[sd->sensor].sensor_addr;
                i2c_w(gspca_dev, i2c);
        }
}

static void do_autogain(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        int deadzone, desired_avg_lum, avg_lum;

        avg_lum = atomic_read(&sd->avg_lum);
        if (avg_lum == -1)
                return;

        if (sd->autogain_ignore_frames > 0) {
                sd->autogain_ignore_frames--;
                return;
        }

        /* SIF / VGA sensors have a different autoexposure area and thus
           different avg_lum values for the same picture brightness */
        if (sensor_data[sd->sensor].flags & F_SIF) {
                deadzone = 500;
                /* SIF sensors tend to overexpose, so keep this small */
                desired_avg_lum = 5000;
        } else {
                deadzone = 1500;
                desired_avg_lum = 13000;
        }

        if (sd->brightness)
                desired_avg_lum = sd->brightness->val * desired_avg_lum / 127;

        if (gspca_dev->exposure->maximum < 500) {
                if (gspca_coarse_grained_expo_autogain(gspca_dev, avg_lum,
                                desired_avg_lum, deadzone))
                        sd->autogain_ignore_frames = AUTOGAIN_IGNORE_FRAMES;
        } else {
                int gain_knee = (s32)gspca_dev->gain->maximum * 9 / 10;
                if (gspca_expo_autogain(gspca_dev, avg_lum, desired_avg_lum,
                                deadzone, gain_knee, sd->exposure_knee))
                        sd->autogain_ignore_frames = AUTOGAIN_IGNORE_FRAMES;
        }
}

/* this function is called at probe time */
static int sd_config(struct gspca_dev *gspca_dev,
                        const struct usb_device_id *id)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam;

        reg_r(gspca_dev, 0x00);
        if (gspca_dev->usb_buf[0] != 0x10)
                return -ENODEV;

        /* copy the webcam info from the device id */
        sd->sensor = id->driver_info >> 8;
        sd->bridge = id->driver_info & 0xff;

        cam = &gspca_dev->cam;
        if (!(sensor_data[sd->sensor].flags & F_SIF)) {
                cam->cam_mode = vga_mode;
                cam->nmodes = ARRAY_SIZE(vga_mode);
        } else {
                cam->cam_mode = sif_mode;
                cam->nmodes = ARRAY_SIZE(sif_mode);
        }
        cam->npkt = 36;                        /* 36 packets per ISOC message */

        return 0;
}

/* this function is called at probe and resume time */
static int sd_init(struct gspca_dev *gspca_dev)
{
        const __u8 stop = 0x09; /* Disable stream turn of LED */

        reg_w(gspca_dev, 0x01, &stop, 1);

        return gspca_dev->usb_err;
}

static int sd_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct gspca_dev *gspca_dev =
                container_of(ctrl->handler, struct gspca_dev, ctrl_handler);
        struct sd *sd = (struct sd *)gspca_dev;

        gspca_dev->usb_err = 0;

        if (ctrl->id == V4L2_CID_AUTOGAIN && ctrl->is_new && ctrl->val) {
                /* when switching to autogain set defaults to make sure
                   we are on a valid point of the autogain gain /
                   exposure knee graph, and give this change time to
                   take effect before doing autogain. */
                gspca_dev->gain->val = gspca_dev->gain->default_value;
                gspca_dev->exposure->val = gspca_dev->exposure->default_value;
                sd->autogain_ignore_frames = AUTOGAIN_IGNORE_FRAMES;
        }

        if (!gspca_dev->streaming)
                return 0;

        switch (ctrl->id) {
        case V4L2_CID_BRIGHTNESS:
                setbrightness(gspca_dev);
                break;
        case V4L2_CID_AUTOGAIN:
                if (gspca_dev->exposure->is_new || (ctrl->is_new && ctrl->val))
                        setexposure(gspca_dev);
                if (gspca_dev->gain->is_new || (ctrl->is_new && ctrl->val))
                        setgain(gspca_dev);
                break;
        case V4L2_CID_POWER_LINE_FREQUENCY:
                setfreq(gspca_dev);
                break;
        default:
                return -EINVAL;
        }
        return gspca_dev->usb_err;
}

static const struct v4l2_ctrl_ops sd_ctrl_ops = {
        .s_ctrl = sd_s_ctrl,
};

/* this function is called at probe time */
static int sd_init_controls(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct v4l2_ctrl_handler *hdl = &gspca_dev->ctrl_handler;

        gspca_dev->vdev.ctrl_handler = hdl;
        v4l2_ctrl_handler_init(hdl, 5);

        if (sd->sensor == SENSOR_OV6650 || sd->sensor == SENSOR_OV7630 ||
            sd->sensor == SENSOR_PAS106 || sd->sensor == SENSOR_PAS202)
                sd->brightness = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_BRIGHTNESS, 0, 255, 1, 127);

        /* Gain range is sensor dependent */
        switch (sd->sensor) {
        case SENSOR_OV6650:
        case SENSOR_PAS106:
        case SENSOR_PAS202:
                gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_GAIN, 0, 31, 1, 15);
                break;
        case SENSOR_OV7630:
                gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_GAIN, 0, 47, 1, 31);
                break;
        case SENSOR_HV7131D:
                gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_GAIN, 0, 63, 1, 31);
                break;
        case SENSOR_TAS5110C:
        case SENSOR_TAS5110D:
        case SENSOR_TAS5130CXX:
                gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_GAIN, 0, 255, 1, 127);
                break;
        default:
                if (sd->bridge == BRIDGE_103) {
                        gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                                V4L2_CID_GAIN, 0, 127, 1, 63);
                } else {
                        gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                                V4L2_CID_GAIN, 0, 15, 1, 7);
                }
        }

        /* Exposure range is sensor dependent, and not all have exposure */
        switch (sd->sensor) {
        case SENSOR_HV7131D:
                gspca_dev->exposure = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_EXPOSURE, 0, 8191, 1, 482);
                sd->exposure_knee = 964;
                break;
        case SENSOR_OV6650:
        case SENSOR_OV7630:
        case SENSOR_PAS106:
        case SENSOR_PAS202:
                gspca_dev->exposure = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_EXPOSURE, 0, 1023, 1, 66);
                sd->exposure_knee = 200;
                break;
        case SENSOR_TAS5110C:
        case SENSOR_TAS5110D:
                gspca_dev->exposure = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                        V4L2_CID_EXPOSURE, 2, 15, 1, 2);
                break;
        }

        if (gspca_dev->exposure) {
                gspca_dev->autogain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                                                V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
        }

        if (sd->sensor == SENSOR_OV6650 || sd->sensor == SENSOR_OV7630)
                sd->plfreq = v4l2_ctrl_new_std_menu(hdl, &sd_ctrl_ops,
                        V4L2_CID_POWER_LINE_FREQUENCY,
                        V4L2_CID_POWER_LINE_FREQUENCY_60HZ, 0,
                        V4L2_CID_POWER_LINE_FREQUENCY_DISABLED);

        if (hdl->error) {
                pr_err("Could not initialize controls\n");
                return hdl->error;
        }

        if (gspca_dev->autogain)
                v4l2_ctrl_auto_cluster(3, &gspca_dev->autogain, 0, false);

        return 0;
}

/* -- start the camera -- */
static int sd_start(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam = &gspca_dev->cam;
        int i, mode;
        __u8 regs[0x31];

        mode = cam->cam_mode[gspca_dev->curr_mode].priv & 0x07;
        /* Copy registers 0x01 - 0x19 from the template */
        memcpy(&regs[0x01], sensor_data[sd->sensor].bridge_init, 0x19);
        /* Set the mode */
        regs[0x18] |= mode << 4;

        /* Set bridge gain to 1.0 */
        if (sd->bridge == BRIDGE_103) {
                regs[0x05] = 0x20; /* Red */
                regs[0x06] = 0x20; /* Green */
                regs[0x07] = 0x20; /* Blue */
        } else {
                regs[0x10] = 0x00; /* Red and blue */
                regs[0x11] = 0x00; /* Green */
        }

        /* Setup pixel numbers and auto exposure window */
        if (sensor_data[sd->sensor].flags & F_SIF) {
                regs[0x1a] = 0x14; /* HO_SIZE 640, makes no sense */
                regs[0x1b] = 0x0a; /* VO_SIZE 320, makes no sense */
                regs[0x1c] = 0x02; /* AE H-start 64 */
                regs[0x1d] = 0x02; /* AE V-start 64 */
                regs[0x1e] = 0x09; /* AE H-end 288 */
                regs[0x1f] = 0x07; /* AE V-end 224 */
        } else {
                regs[0x1a] = 0x1d; /* HO_SIZE 960, makes no sense */
                regs[0x1b] = 0x10; /* VO_SIZE 512, makes no sense */
                regs[0x1c] = 0x05; /* AE H-start 160 */
                regs[0x1d] = 0x03; /* AE V-start 96 */
                regs[0x1e] = 0x0f; /* AE H-end 480 */
                regs[0x1f] = 0x0c; /* AE V-end 384 */
        }

        /* Setup the gamma table (only used with the sn9c103 bridge) */
        for (i = 0; i < 16; i++)
                regs[0x20 + i] = i * 16;
        regs[0x20 + i] = 255;

        /* Special cases where some regs depend on mode or bridge */
        switch (sd->sensor) {
        case SENSOR_TAS5130CXX:
                /* FIXME / TESTME
                   probably not mode specific at all most likely the upper
                   nibble of 0x19 is exposure (clock divider) just as with
                   the tas5110, we need someone to test this. */
                regs[0x19] = mode ? 0x23 : 0x43;
                break;
        case SENSOR_OV7630:
                /* FIXME / TESTME for some reason with the 101/102 bridge the
                   clock is set to 12 Mhz (reg1 == 0x04), rather then 24.
                   Also the hstart needs to go from 1 to 2 when using a 103,
                   which is likely related. This does not seem right. */
                if (sd->bridge == BRIDGE_103) {
                        regs[0x01] = 0x44; /* Select 24 Mhz clock */
                        regs[0x12] = 0x02; /* Set hstart to 2 */
                }
                break;
        case SENSOR_PAS202:
                /* For some unknown reason we need to increase hstart by 1 on
                   the sn9c103, otherwise we get wrong colors (bayer shift). */
                if (sd->bridge == BRIDGE_103)
                        regs[0x12] += 1;
                break;
        }
        /* Disable compression when the raw bayer format has been selected */
        if (cam->cam_mode[gspca_dev->curr_mode].priv & MODE_RAW)
                regs[0x18] &= ~0x80;

        /* Vga mode emulation on SIF sensor? */
        if (cam->cam_mode[gspca_dev->curr_mode].priv & MODE_REDUCED_SIF) {
                regs[0x12] += 16;        /* hstart adjust */
                regs[0x13] += 24;        /* vstart adjust */
                regs[0x15]  = 320 / 16; /* hsize */
                regs[0x16]  = 240 / 16; /* vsize */
        }

        /* reg 0x01 bit 2 video transfert on */
        reg_w(gspca_dev, 0x01, &regs[0x01], 1);
        /* reg 0x17 SensorClk enable inv Clk 0x60 */
        reg_w(gspca_dev, 0x17, &regs[0x17], 1);
        /* Set the registers from the template */
        reg_w(gspca_dev, 0x01, &regs[0x01],
              (sd->bridge == BRIDGE_103) ? 0x30 : 0x1f);

        /* Init the sensor */
        i2c_w_vector(gspca_dev, sensor_data[sd->sensor].sensor_init,
                        sensor_data[sd->sensor].sensor_init_size);

        /* Mode / bridge specific sensor setup */
        switch (sd->sensor) {
        case SENSOR_PAS202: {
                const __u8 i2cpclockdiv[] =
                        {0xa0, 0x40, 0x02, 0x03, 0x00, 0x00, 0x00, 0x10};
                /* clockdiv from 4 to 3 (7.5 -> 10 fps) when in low res mode */
                if (mode)
                        i2c_w(gspca_dev, i2cpclockdiv);
                break;
            }
        case SENSOR_OV7630:
                /* FIXME / TESTME We should be able to handle this identical
                   for the 101/102 and the 103 case */
                if (sd->bridge == BRIDGE_103) {
                        const __u8 i2c[] = { 0xa0, 0x21, 0x13,
                                             0x80, 0x00, 0x00, 0x00, 0x10 };
                        i2c_w(gspca_dev, i2c);
                }
                break;
        }
        /* H_size V_size 0x28, 0x1e -> 640x480. 0x16, 0x12 -> 352x288 */
        reg_w(gspca_dev, 0x15, &regs[0x15], 2);
        /* compression register */
        reg_w(gspca_dev, 0x18, &regs[0x18], 1);
        /* H_start */
        reg_w(gspca_dev, 0x12, &regs[0x12], 1);
        /* V_START */
        reg_w(gspca_dev, 0x13, &regs[0x13], 1);
        /* reset 0x17 SensorClk enable inv Clk 0x60 */
                                /*fixme: ov7630 [17]=68 8f (+20 if 102)*/
        reg_w(gspca_dev, 0x17, &regs[0x17], 1);
        /*MCKSIZE ->3 */        /*fixme: not ov7630*/
        reg_w(gspca_dev, 0x19, &regs[0x19], 1);
        /* AE_STRX AE_STRY AE_ENDX AE_ENDY */
        reg_w(gspca_dev, 0x1c, &regs[0x1c], 4);
        /* Enable video transfert */
        reg_w(gspca_dev, 0x01, &regs[0x01], 1);
        /* Compression */
        reg_w(gspca_dev, 0x18, &regs[0x18], 2);
        msleep(20);

        sd->reg11 = -1;

        setgain(gspca_dev);
        setbrightness(gspca_dev);
        setexposure(gspca_dev);
        setfreq(gspca_dev);

        sd->frames_to_drop = 0;
        sd->autogain_ignore_frames = 0;
        gspca_dev->exp_too_high_cnt = 0;
        gspca_dev->exp_too_low_cnt = 0;
        atomic_set(&sd->avg_lum, -1);
        return gspca_dev->usb_err;
}

static void sd_stopN(struct gspca_dev *gspca_dev)
{
        sd_init(gspca_dev);
}

static u8* find_sof(struct gspca_dev *gspca_dev, u8 *data, int len)
{
        struct sd *sd = (struct sd *) gspca_dev;
        int i, header_size = (sd->bridge == BRIDGE_103) ? 18 : 12;

        /* frames start with:
         *        ff ff 00 c4 c4 96        synchro
         *        00                (unknown)
         *        xx                (frame sequence / size / compression)
         *        (xx)                (idem - extra byte for sn9c103)
         *        ll mm                brightness sum inside auto exposure
         *        ll mm                brightness sum outside auto exposure
         *        (xx xx xx xx xx)        audio values for snc103
         */
        for (i = 0; i < len; i++) {
                switch (sd->header_read) {
                case 0:
                        if (data[i] == 0xff)
                                sd->header_read++;
                        break;
                case 1:
                        if (data[i] == 0xff)
                                sd->header_read++;
                        else
                                sd->header_read = 0;
                        break;
                case 2:
                        if (data[i] == 0x00)
                                sd->header_read++;
                        else if (data[i] != 0xff)
                                sd->header_read = 0;
                        break;
                case 3:
                        if (data[i] == 0xc4)
                                sd->header_read++;
                        else if (data[i] == 0xff)
                                sd->header_read = 1;
                        else
                                sd->header_read = 0;
                        break;
                case 4:
                        if (data[i] == 0xc4)
                                sd->header_read++;
                        else if (data[i] == 0xff)
                                sd->header_read = 1;
                        else
                                sd->header_read = 0;
                        break;
                case 5:
                        if (data[i] == 0x96)
                                sd->header_read++;
                        else if (data[i] == 0xff)
                                sd->header_read = 1;
                        else
                                sd->header_read = 0;
                        break;
                default:
                        sd->header[sd->header_read - 6] = data[i];
                        sd->header_read++;
                        if (sd->header_read == header_size) {
                                sd->header_read = 0;
                                return data + i + 1;
                        }
                }
        }
        return NULL;
}

static void sd_pkt_scan(struct gspca_dev *gspca_dev,
                        u8 *data,                        /* isoc packet */
                        int len)                        /* iso packet length */
{
        int fr_h_sz = 0, lum_offset = 0, len_after_sof = 0;
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam = &gspca_dev->cam;
        u8 *sof;

        sof = find_sof(gspca_dev, data, len);
        if (sof) {
                if (sd->bridge == BRIDGE_103) {
                        fr_h_sz = 18;
                        lum_offset = 3;
                } else {
                        fr_h_sz = 12;
                        lum_offset = 2;
                }

                len_after_sof = len - (sof - data);
                len = (sof - data) - fr_h_sz;
                if (len < 0)
                        len = 0;
        }

        if (cam->cam_mode[gspca_dev->curr_mode].priv & MODE_RAW) {
                /* In raw mode we sometimes get some garbage after the frame
                   ignore this */
                int used;
                int size = cam->cam_mode[gspca_dev->curr_mode].sizeimage;

                used = gspca_dev->image_len;
                if (used + len > size)
                        len = size - used;
        }

        gspca_frame_add(gspca_dev, INTER_PACKET, data, len);

        if (sof) {
                int  lum = sd->header[lum_offset] +
                          (sd->header[lum_offset + 1] << 8);

                /* When exposure changes midway a frame we
                   get a lum of 0 in this case drop 2 frames
                   as the frames directly after an exposure
                   change have an unstable image. Sometimes lum
                   *really* is 0 (cam used in low light with
                   low exposure setting), so do not drop frames
                   if the previous lum was 0 too. */
                if (lum == 0 && sd->prev_avg_lum != 0) {
                        lum = -1;
                        sd->frames_to_drop = 2;
                        sd->prev_avg_lum = 0;
                } else
                        sd->prev_avg_lum = lum;
                atomic_set(&sd->avg_lum, lum);

                if (sd->frames_to_drop)
                        sd->frames_to_drop--;
                else
                        gspca_frame_add(gspca_dev, LAST_PACKET, NULL, 0);

                gspca_frame_add(gspca_dev, FIRST_PACKET, sof, len_after_sof);
        }
}

#if IS_ENABLED(CONFIG_INPUT)
static int sd_int_pkt_scan(struct gspca_dev *gspca_dev,
                        u8 *data,                /* interrupt packet data */
                        int len)                /* interrupt packet length */
{
        int ret = -EINVAL;

        if (len == 1 && data[0] == 1) {
                input_report_key(gspca_dev->input_dev, KEY_CAMERA, 1);
                input_sync(gspca_dev->input_dev);
                input_report_key(gspca_dev->input_dev, KEY_CAMERA, 0);
                input_sync(gspca_dev->input_dev);
                ret = 0;
        }

        return ret;
}
#endif

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name = MODULE_NAME,
        .config = sd_config,
        .init = sd_init,
        .init_controls = sd_init_controls,
        .start = sd_start,
        .stopN = sd_stopN,
        .pkt_scan = sd_pkt_scan,
        .dq_callback = do_autogain,
#if IS_ENABLED(CONFIG_INPUT)
        .int_pkt_scan = sd_int_pkt_scan,
#endif
};

/* -- module initialisation -- */
#define SB(sensor, bridge) \
        .driver_info = (SENSOR_ ## sensor << 8) | BRIDGE_ ## bridge


static const struct usb_device_id device_table[] = {
        {USB_DEVICE(0x0c45, 0x6001), SB(TAS5110C, 102)}, /* TAS5110C1B */
        {USB_DEVICE(0x0c45, 0x6005), SB(TAS5110C, 101)}, /* TAS5110C1B */
        {USB_DEVICE(0x0c45, 0x6007), SB(TAS5110D, 101)}, /* TAS5110D */
        {USB_DEVICE(0x0c45, 0x6009), SB(PAS106, 101)},
        {USB_DEVICE(0x0c45, 0x600d), SB(PAS106, 101)},
        {USB_DEVICE(0x0c45, 0x6011), SB(OV6650, 101)},
        {USB_DEVICE(0x0c45, 0x6019), SB(OV7630, 101)},
        {USB_DEVICE(0x0c45, 0x6024), SB(TAS5130CXX, 102)},
        {USB_DEVICE(0x0c45, 0x6025), SB(TAS5130CXX, 102)},
        {USB_DEVICE(0x0c45, 0x6027), SB(OV7630, 101)}, /* Genius Eye 310 */
        {USB_DEVICE(0x0c45, 0x6028), SB(PAS202, 102)},
        {USB_DEVICE(0x0c45, 0x6029), SB(PAS106, 102)},
        {USB_DEVICE(0x0c45, 0x602a), SB(HV7131D, 102)},
        /* {USB_DEVICE(0x0c45, 0x602b), SB(MI0343, 102)}, */
        {USB_DEVICE(0x0c45, 0x602c), SB(OV7630, 102)},
        {USB_DEVICE(0x0c45, 0x602d), SB(HV7131R, 102)},
        {USB_DEVICE(0x0c45, 0x602e), SB(OV7630, 102)},
        /* {USB_DEVICE(0x0c45, 0x6030), SB(MI03XX, 102)}, */ /* MI0343 MI0360 MI0330 */
        /* {USB_DEVICE(0x0c45, 0x6082), SB(MI03XX, 103)}, */ /* MI0343 MI0360 */
        {USB_DEVICE(0x0c45, 0x6083), SB(HV7131D, 103)},
        {USB_DEVICE(0x0c45, 0x608c), SB(HV7131R, 103)},
        /* {USB_DEVICE(0x0c45, 0x608e), SB(CISVF10, 103)}, */
        {USB_DEVICE(0x0c45, 0x608f), SB(OV7630, 103)},
        {USB_DEVICE(0x0c45, 0x60a8), SB(PAS106, 103)},
        {USB_DEVICE(0x0c45, 0x60aa), SB(TAS5130CXX, 103)},
        {USB_DEVICE(0x0c45, 0x60af), SB(PAS202, 103)},
        {USB_DEVICE(0x0c45, 0x60b0), SB(OV7630, 103)},
        {}
};
MODULE_DEVICE_TABLE(usb, device_table);

/* -- device connect -- */
static int sd_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                                THIS_MODULE);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = device_table,
        .probe = sd_probe,
        .disconnect = gspca_disconnect,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
};

module_usb_driver(sd_driver);







































  685 
    1 








  688 

























































































































































   10 



   10 


   10 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
// SPDX-License-Identifier: GPL-2.0-only
/*
 * kernel/freezer.c - Function to freeze a process
 *
 * Originally from kernel/power/process.c
 */

#include <linux/interrupt.h>
#include <linux/suspend.h>
#include <linux/export.h>
#include <linux/syscalls.h>
#include <linux/freezer.h>
#include <linux/kthread.h>

/* total number of freezing conditions in effect */
DEFINE_STATIC_KEY_FALSE(freezer_active);
EXPORT_SYMBOL(freezer_active);

/*
 * indicate whether PM freezing is in effect, protected by
 * system_transition_mutex
 */
bool pm_freezing;
bool pm_nosig_freezing;

/* protects freezing and frozen transitions */
static DEFINE_SPINLOCK(freezer_lock);

/**
 * freezing_slow_path - slow path for testing whether a task needs to be frozen
 * @p: task to be tested
 *
 * This function is called by freezing() if freezer_active isn't zero
 * and tests whether @p needs to enter and stay in frozen state.  Can be
 * called under any context.  The freezers are responsible for ensuring the
 * target tasks see the updated state.
 */
bool freezing_slow_path(struct task_struct *p)
{
        if (p->flags & (PF_NOFREEZE | PF_SUSPEND_TASK))
                return false;

        if (test_tsk_thread_flag(p, TIF_MEMDIE))
                return false;

        if (pm_nosig_freezing || cgroup_freezing(p))
                return true;

        if (pm_freezing && !(p->flags & PF_KTHREAD))
                return true;

        return false;
}
EXPORT_SYMBOL(freezing_slow_path);

bool frozen(struct task_struct *p)
{
        return READ_ONCE(p->__state) & TASK_FROZEN;
}

/* Refrigerator is place where frozen processes are stored :-). */
bool __refrigerator(bool check_kthr_stop)
{
        unsigned int state = get_current_state();
        bool was_frozen = false;

        pr_debug("%s entered refrigerator\n", current->comm);

        WARN_ON_ONCE(state && !(state & TASK_NORMAL));

        for (;;) {
                bool freeze;

                raw_spin_lock_irq(&current->pi_lock);
                WRITE_ONCE(current->__state, TASK_FROZEN);
                /* unstale saved_state so that __thaw_task() will wake us up */
                current->saved_state = TASK_RUNNING;
                raw_spin_unlock_irq(&current->pi_lock);

                spin_lock_irq(&freezer_lock);
                freeze = freezing(current) && !(check_kthr_stop && kthread_should_stop());
                spin_unlock_irq(&freezer_lock);

                if (!freeze)
                        break;

                was_frozen = true;
                schedule();
        }
        __set_current_state(TASK_RUNNING);

        pr_debug("%s left refrigerator\n", current->comm);

        return was_frozen;
}
EXPORT_SYMBOL(__refrigerator);

static void fake_signal_wake_up(struct task_struct *p)
{
        unsigned long flags;

        if (lock_task_sighand(p, &flags)) {
                signal_wake_up(p, 0);
                unlock_task_sighand(p, &flags);
        }
}

static int __set_task_frozen(struct task_struct *p, void *arg)
{
        unsigned int state = READ_ONCE(p->__state);

        /*
         * Allow freezing the sched_delayed tasks; they will not execute until
         * ttwu() fixes them up, so it is safe to swap their state now, instead
         * of waiting for them to get fully dequeued.
         */
        if (task_is_runnable(p))
                return 0;

        if (p != current && task_curr(p))
                return 0;

        if (!(state & (TASK_FREEZABLE | __TASK_STOPPED | __TASK_TRACED)))
                return 0;

        /*
         * Only TASK_NORMAL can be augmented with TASK_FREEZABLE, since they
         * can suffer spurious wakeups.
         */
        if (state & TASK_FREEZABLE)
                WARN_ON_ONCE(!(state & TASK_NORMAL));

#ifdef CONFIG_LOCKDEP
        /*
         * It's dangerous to freeze with locks held; there be dragons there.
         */
        if (!(state & __TASK_FREEZABLE_UNSAFE))
                WARN_ON_ONCE(debug_locks && p->lockdep_depth);
#endif

        p->saved_state = p->__state;
        WRITE_ONCE(p->__state, TASK_FROZEN);
        return TASK_FROZEN;
}

static bool __freeze_task(struct task_struct *p)
{
        /* TASK_FREEZABLE|TASK_STOPPED|TASK_TRACED -> TASK_FROZEN */
        return task_call_func(p, __set_task_frozen, NULL);
}

/**
 * freeze_task - send a freeze request to given task
 * @p: task to send the request to
 *
 * If @p is freezing, the freeze request is sent either by sending a fake
 * signal (if it's not a kernel thread) or waking it up (if it's a kernel
 * thread).
 *
 * RETURNS:
 * %false, if @p is not freezing or already frozen; %true, otherwise
 */
bool freeze_task(struct task_struct *p)
{
        unsigned long flags;

        spin_lock_irqsave(&freezer_lock, flags);
        if (!freezing(p) || frozen(p) || __freeze_task(p)) {
                spin_unlock_irqrestore(&freezer_lock, flags);
                return false;
        }

        if (!(p->flags & PF_KTHREAD))
                fake_signal_wake_up(p);
        else
                wake_up_state(p, TASK_NORMAL);

        spin_unlock_irqrestore(&freezer_lock, flags);
        return true;
}

/*
 * Restore the saved_state before the task entered freezer. For typical task
 * in the __refrigerator(), saved_state == TASK_RUNNING so nothing happens
 * here. For tasks which were TASK_NORMAL | TASK_FREEZABLE, their initial state
 * is restored unless they got an expected wakeup (see ttwu_state_match()).
 * Returns 1 if the task state was restored.
 */
static int __restore_freezer_state(struct task_struct *p, void *arg)
{
        unsigned int state = p->saved_state;

        if (state != TASK_RUNNING) {
                WRITE_ONCE(p->__state, state);
                p->saved_state = TASK_RUNNING;
                return 1;
        }

        return 0;
}

void __thaw_task(struct task_struct *p)
{
        unsigned long flags;

        spin_lock_irqsave(&freezer_lock, flags);
        if (WARN_ON_ONCE(freezing(p)))
                goto unlock;

        if (!frozen(p) || task_call_func(p, __restore_freezer_state, NULL))
                goto unlock;

        wake_up_state(p, TASK_FROZEN);
unlock:
        spin_unlock_irqrestore(&freezer_lock, flags);
}

/**
 * set_freezable - make %current freezable
 *
 * Mark %current freezable and enter refrigerator if necessary.
 */
bool set_freezable(void)
{
        might_sleep();

        /*
         * Modify flags while holding freezer_lock.  This ensures the
         * freezer notices that we aren't frozen yet or the freezing
         * condition is visible to try_to_freeze() below.
         */
        spin_lock_irq(&freezer_lock);
        current->flags &= ~PF_NOFREEZE;
        spin_unlock_irq(&freezer_lock);

        return try_to_freeze();
}
EXPORT_SYMBOL(set_freezable);




































































    8 























    8 













































    8 






    8 
    8 















    8 



































































































































































































































































































































































































































































































































































































































































































































































































































    8 











    8 

































































































































    8 



    8 








    8 


















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
// SPDX-License-Identifier: GPL-2.0-only
/*
 * linux/net/sunrpc/xdr.c
 *
 * Generic XDR support.
 *
 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/pagemap.h>
#include <linux/errno.h>
#include <linux/sunrpc/xdr.h>
#include <linux/sunrpc/msg_prot.h>
#include <linux/bvec.h>
#include <trace/events/sunrpc.h>

static void _copy_to_pages(struct page **, size_t, const char *, size_t);


/*
 * XDR functions for basic NFS types
 */
__be32 *
xdr_encode_netobj(__be32 *p, const struct xdr_netobj *obj)
{
        unsigned int        quadlen = XDR_QUADLEN(obj->len);

        p[quadlen] = 0;                /* zero trailing bytes */
        *p++ = cpu_to_be32(obj->len);
        memcpy(p, obj->data, obj->len);
        return p + XDR_QUADLEN(obj->len);
}
EXPORT_SYMBOL_GPL(xdr_encode_netobj);

__be32 *
xdr_decode_netobj(__be32 *p, struct xdr_netobj *obj)
{
        unsigned int        len;

        if ((len = be32_to_cpu(*p++)) > XDR_MAX_NETOBJ)
                return NULL;
        obj->len  = len;
        obj->data = (u8 *) p;
        return p + XDR_QUADLEN(len);
}
EXPORT_SYMBOL_GPL(xdr_decode_netobj);

/**
 * xdr_encode_opaque_fixed - Encode fixed length opaque data
 * @p: pointer to current position in XDR buffer.
 * @ptr: pointer to data to encode (or NULL)
 * @nbytes: size of data.
 *
 * Copy the array of data of length nbytes at ptr to the XDR buffer
 * at position p, then align to the next 32-bit boundary by padding
 * with zero bytes (see RFC1832).
 * Note: if ptr is NULL, only the padding is performed.
 *
 * Returns the updated current XDR buffer position
 *
 */
__be32 *xdr_encode_opaque_fixed(__be32 *p, const void *ptr, unsigned int nbytes)
{
        if (likely(nbytes != 0)) {
                unsigned int quadlen = XDR_QUADLEN(nbytes);
                unsigned int padding = (quadlen << 2) - nbytes;

                if (ptr != NULL)
                        memcpy(p, ptr, nbytes);
                if (padding != 0)
                        memset((char *)p + nbytes, 0, padding);
                p += quadlen;
        }
        return p;
}
EXPORT_SYMBOL_GPL(xdr_encode_opaque_fixed);

/**
 * xdr_encode_opaque - Encode variable length opaque data
 * @p: pointer to current position in XDR buffer.
 * @ptr: pointer to data to encode (or NULL)
 * @nbytes: size of data.
 *
 * Returns the updated current XDR buffer position
 */
__be32 *xdr_encode_opaque(__be32 *p, const void *ptr, unsigned int nbytes)
{
        *p++ = cpu_to_be32(nbytes);
        return xdr_encode_opaque_fixed(p, ptr, nbytes);
}
EXPORT_SYMBOL_GPL(xdr_encode_opaque);

__be32 *
xdr_encode_string(__be32 *p, const char *string)
{
        return xdr_encode_array(p, string, strlen(string));
}
EXPORT_SYMBOL_GPL(xdr_encode_string);

__be32 *
xdr_decode_string_inplace(__be32 *p, char **sp,
                          unsigned int *lenp, unsigned int maxlen)
{
        u32 len;

        len = be32_to_cpu(*p++);
        if (len > maxlen)
                return NULL;
        *lenp = len;
        *sp = (char *) p;
        return p + XDR_QUADLEN(len);
}
EXPORT_SYMBOL_GPL(xdr_decode_string_inplace);

/**
 * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf
 * @buf: XDR buffer where string resides
 * @len: length of string, in bytes
 *
 */
void xdr_terminate_string(const struct xdr_buf *buf, const u32 len)
{
        char *kaddr;

        kaddr = kmap_atomic(buf->pages[0]);
        kaddr[buf->page_base + len] = '\0';
        kunmap_atomic(kaddr);
}
EXPORT_SYMBOL_GPL(xdr_terminate_string);

size_t xdr_buf_pagecount(const struct xdr_buf *buf)
{
        if (!buf->page_len)
                return 0;
        return (buf->page_base + buf->page_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
}

int
xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp)
{
        size_t i, n = xdr_buf_pagecount(buf);

        if (n != 0 && buf->bvec == NULL) {
                buf->bvec = kmalloc_array(n, sizeof(buf->bvec[0]), gfp);
                if (!buf->bvec)
                        return -ENOMEM;
                for (i = 0; i < n; i++) {
                        bvec_set_page(&buf->bvec[i], buf->pages[i], PAGE_SIZE,
                                      0);
                }
        }
        return 0;
}

void
xdr_free_bvec(struct xdr_buf *buf)
{
        kfree(buf->bvec);
        buf->bvec = NULL;
}

/**
 * xdr_buf_to_bvec - Copy components of an xdr_buf into a bio_vec array
 * @bvec: bio_vec array to populate
 * @bvec_size: element count of @bio_vec
 * @xdr: xdr_buf to be copied
 *
 * Returns the number of entries consumed in @bvec.
 */
unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size,
                             const struct xdr_buf *xdr)
{
        const struct kvec *head = xdr->head;
        const struct kvec *tail = xdr->tail;
        unsigned int count = 0;

        if (head->iov_len) {
                bvec_set_virt(bvec++, head->iov_base, head->iov_len);
                ++count;
        }

        if (xdr->page_len) {
                unsigned int offset, len, remaining;
                struct page **pages = xdr->pages;

                offset = offset_in_page(xdr->page_base);
                remaining = xdr->page_len;
                while (remaining > 0) {
                        len = min_t(unsigned int, remaining,
                                    PAGE_SIZE - offset);
                        bvec_set_page(bvec++, *pages++, len, offset);
                        remaining -= len;
                        offset = 0;
                        if (unlikely(++count > bvec_size))
                                goto bvec_overflow;
                }
        }

        if (tail->iov_len) {
                bvec_set_virt(bvec, tail->iov_base, tail->iov_len);
                if (unlikely(++count > bvec_size))
                        goto bvec_overflow;
        }

        return count;

bvec_overflow:
        pr_warn_once("%s: bio_vec array overflow\n", __func__);
        return count - 1;
}

/**
 * xdr_inline_pages - Prepare receive buffer for a large reply
 * @xdr: xdr_buf into which reply will be placed
 * @offset: expected offset where data payload will start, in bytes
 * @pages: vector of struct page pointers
 * @base: offset in first page where receive should start, in bytes
 * @len: expected size of the upper layer data payload, in bytes
 *
 */
void
xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,
                 struct page **pages, unsigned int base, unsigned int len)
{
        struct kvec *head = xdr->head;
        struct kvec *tail = xdr->tail;
        char *buf = (char *)head->iov_base;
        unsigned int buflen = head->iov_len;

        head->iov_len  = offset;

        xdr->pages = pages;
        xdr->page_base = base;
        xdr->page_len = len;

        tail->iov_base = buf + offset;
        tail->iov_len = buflen - offset;
        xdr->buflen += len;
}
EXPORT_SYMBOL_GPL(xdr_inline_pages);

/*
 * Helper routines for doing 'memmove' like operations on a struct xdr_buf
 */

/**
 * _shift_data_left_pages
 * @pages: vector of pages containing both the source and dest memory area.
 * @pgto_base: page vector address of destination
 * @pgfrom_base: page vector address of source
 * @len: number of bytes to copy
 *
 * Note: the addresses pgto_base and pgfrom_base are both calculated in
 *       the same way:
 *            if a memory area starts at byte 'base' in page 'pages[i]',
 *            then its address is given as (i << PAGE_CACHE_SHIFT) + base
 * Alse note: pgto_base must be < pgfrom_base, but the memory areas
 *         they point to may overlap.
 */
static void
_shift_data_left_pages(struct page **pages, size_t pgto_base,
                        size_t pgfrom_base, size_t len)
{
        struct page **pgfrom, **pgto;
        char *vfrom, *vto;
        size_t copy;

        BUG_ON(pgfrom_base <= pgto_base);

        if (!len)
                return;

        pgto = pages + (pgto_base >> PAGE_SHIFT);
        pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);

        pgto_base &= ~PAGE_MASK;
        pgfrom_base &= ~PAGE_MASK;

        do {
                if (pgto_base >= PAGE_SIZE) {
                        pgto_base = 0;
                        pgto++;
                }
                if (pgfrom_base >= PAGE_SIZE){
                        pgfrom_base = 0;
                        pgfrom++;
                }

                copy = len;
                if (copy > (PAGE_SIZE - pgto_base))
                        copy = PAGE_SIZE - pgto_base;
                if (copy > (PAGE_SIZE - pgfrom_base))
                        copy = PAGE_SIZE - pgfrom_base;

                vto = kmap_atomic(*pgto);
                if (*pgto != *pgfrom) {
                        vfrom = kmap_atomic(*pgfrom);
                        memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
                        kunmap_atomic(vfrom);
                } else
                        memmove(vto + pgto_base, vto + pgfrom_base, copy);
                flush_dcache_page(*pgto);
                kunmap_atomic(vto);

                pgto_base += copy;
                pgfrom_base += copy;

        } while ((len -= copy) != 0);
}

/**
 * _shift_data_right_pages
 * @pages: vector of pages containing both the source and dest memory area.
 * @pgto_base: page vector address of destination
 * @pgfrom_base: page vector address of source
 * @len: number of bytes to copy
 *
 * Note: the addresses pgto_base and pgfrom_base are both calculated in
 *       the same way:
 *            if a memory area starts at byte 'base' in page 'pages[i]',
 *            then its address is given as (i << PAGE_SHIFT) + base
 * Also note: pgfrom_base must be < pgto_base, but the memory areas
 *         they point to may overlap.
 */
static void
_shift_data_right_pages(struct page **pages, size_t pgto_base,
                size_t pgfrom_base, size_t len)
{
        struct page **pgfrom, **pgto;
        char *vfrom, *vto;
        size_t copy;

        BUG_ON(pgto_base <= pgfrom_base);

        if (!len)
                return;

        pgto_base += len;
        pgfrom_base += len;

        pgto = pages + (pgto_base >> PAGE_SHIFT);
        pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);

        pgto_base &= ~PAGE_MASK;
        pgfrom_base &= ~PAGE_MASK;

        do {
                /* Are any pointers crossing a page boundary? */
                if (pgto_base == 0) {
                        pgto_base = PAGE_SIZE;
                        pgto--;
                }
                if (pgfrom_base == 0) {
                        pgfrom_base = PAGE_SIZE;
                        pgfrom--;
                }

                copy = len;
                if (copy > pgto_base)
                        copy = pgto_base;
                if (copy > pgfrom_base)
                        copy = pgfrom_base;
                pgto_base -= copy;
                pgfrom_base -= copy;

                vto = kmap_atomic(*pgto);
                if (*pgto != *pgfrom) {
                        vfrom = kmap_atomic(*pgfrom);
                        memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
                        kunmap_atomic(vfrom);
                } else
                        memmove(vto + pgto_base, vto + pgfrom_base, copy);
                flush_dcache_page(*pgto);
                kunmap_atomic(vto);

        } while ((len -= copy) != 0);
}

/**
 * _copy_to_pages
 * @pages: array of pages
 * @pgbase: page vector address of destination
 * @p: pointer to source data
 * @len: length
 *
 * Copies data from an arbitrary memory location into an array of pages
 * The copy is assumed to be non-overlapping.
 */
static void
_copy_to_pages(struct page **pages, size_t pgbase, const char *p, size_t len)
{
        struct page **pgto;
        char *vto;
        size_t copy;

        if (!len)
                return;

        pgto = pages + (pgbase >> PAGE_SHIFT);
        pgbase &= ~PAGE_MASK;

        for (;;) {
                copy = PAGE_SIZE - pgbase;
                if (copy > len)
                        copy = len;

                vto = kmap_atomic(*pgto);
                memcpy(vto + pgbase, p, copy);
                kunmap_atomic(vto);

                len -= copy;
                if (len == 0)
                        break;

                pgbase += copy;
                if (pgbase == PAGE_SIZE) {
                        flush_dcache_page(*pgto);
                        pgbase = 0;
                        pgto++;
                }
                p += copy;
        }
        flush_dcache_page(*pgto);
}

/**
 * _copy_from_pages
 * @p: pointer to destination
 * @pages: array of pages
 * @pgbase: offset of source data
 * @len: length
 *
 * Copies data into an arbitrary memory location from an array of pages
 * The copy is assumed to be non-overlapping.
 */
void
_copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len)
{
        struct page **pgfrom;
        char *vfrom;
        size_t copy;

        if (!len)
                return;

        pgfrom = pages + (pgbase >> PAGE_SHIFT);
        pgbase &= ~PAGE_MASK;

        do {
                copy = PAGE_SIZE - pgbase;
                if (copy > len)
                        copy = len;

                vfrom = kmap_atomic(*pgfrom);
                memcpy(p, vfrom + pgbase, copy);
                kunmap_atomic(vfrom);

                pgbase += copy;
                if (pgbase == PAGE_SIZE) {
                        pgbase = 0;
                        pgfrom++;
                }
                p += copy;

        } while ((len -= copy) != 0);
}
EXPORT_SYMBOL_GPL(_copy_from_pages);

static void xdr_buf_iov_zero(const struct kvec *iov, unsigned int base,
                             unsigned int len)
{
        if (base >= iov->iov_len)
                return;
        if (len > iov->iov_len - base)
                len = iov->iov_len - base;
        memset(iov->iov_base + base, 0, len);
}

/**
 * xdr_buf_pages_zero
 * @buf: xdr_buf
 * @pgbase: beginning offset
 * @len: length
 */
static void xdr_buf_pages_zero(const struct xdr_buf *buf, unsigned int pgbase,
                               unsigned int len)
{
        struct page **pages = buf->pages;
        struct page **page;
        char *vpage;
        unsigned int zero;

        if (!len)
                return;
        if (pgbase >= buf->page_len) {
                xdr_buf_iov_zero(buf->tail, pgbase - buf->page_len, len);
                return;
        }
        if (pgbase + len > buf->page_len) {
                xdr_buf_iov_zero(buf->tail, 0, pgbase + len - buf->page_len);
                len = buf->page_len - pgbase;
        }

        pgbase += buf->page_base;

        page = pages + (pgbase >> PAGE_SHIFT);
        pgbase &= ~PAGE_MASK;

        do {
                zero = PAGE_SIZE - pgbase;
                if (zero > len)
                        zero = len;

                vpage = kmap_atomic(*page);
                memset(vpage + pgbase, 0, zero);
                kunmap_atomic(vpage);

                flush_dcache_page(*page);
                pgbase = 0;
                page++;

        } while ((len -= zero) != 0);
}

static unsigned int xdr_buf_pages_fill_sparse(const struct xdr_buf *buf,
                                              unsigned int buflen, gfp_t gfp)
{
        unsigned int i, npages, pagelen;

        if (!(buf->flags & XDRBUF_SPARSE_PAGES))
                return buflen;
        if (buflen <= buf->head->iov_len)
                return buflen;
        pagelen = buflen - buf->head->iov_len;
        if (pagelen > buf->page_len)
                pagelen = buf->page_len;
        npages = (pagelen + buf->page_base + PAGE_SIZE - 1) >> PAGE_SHIFT;
        for (i = 0; i < npages; i++) {
                if (!buf->pages[i])
                        continue;
                buf->pages[i] = alloc_page(gfp);
                if (likely(buf->pages[i]))
                        continue;
                buflen -= pagelen;
                pagelen = i << PAGE_SHIFT;
                if (pagelen > buf->page_base)
                        buflen += pagelen - buf->page_base;
                break;
        }
        return buflen;
}

static void xdr_buf_try_expand(struct xdr_buf *buf, unsigned int len)
{
        struct kvec *head = buf->head;
        struct kvec *tail = buf->tail;
        unsigned int sum = head->iov_len + buf->page_len + tail->iov_len;
        unsigned int free_space, newlen;

        if (sum > buf->len) {
                free_space = min_t(unsigned int, sum - buf->len, len);
                newlen = xdr_buf_pages_fill_sparse(buf, buf->len + free_space,
                                                   GFP_KERNEL);
                free_space = newlen - buf->len;
                buf->len = newlen;
                len -= free_space;
                if (!len)
                        return;
        }

        if (buf->buflen > sum) {
                /* Expand the tail buffer */
                free_space = min_t(unsigned int, buf->buflen - sum, len);
                tail->iov_len += free_space;
                buf->len += free_space;
        }
}

static void xdr_buf_tail_copy_right(const struct xdr_buf *buf,
                                    unsigned int base, unsigned int len,
                                    unsigned int shift)
{
        const struct kvec *tail = buf->tail;
        unsigned int to = base + shift;

        if (to >= tail->iov_len)
                return;
        if (len + to > tail->iov_len)
                len = tail->iov_len - to;
        memmove(tail->iov_base + to, tail->iov_base + base, len);
}

static void xdr_buf_pages_copy_right(const struct xdr_buf *buf,
                                     unsigned int base, unsigned int len,
                                     unsigned int shift)
{
        const struct kvec *tail = buf->tail;
        unsigned int to = base + shift;
        unsigned int pglen = 0;
        unsigned int talen = 0, tato = 0;

        if (base >= buf->page_len)
                return;
        if (len > buf->page_len - base)
                len = buf->page_len - base;
        if (to >= buf->page_len) {
                tato = to - buf->page_len;
                if (tail->iov_len >= len + tato)
                        talen = len;
                else if (tail->iov_len > tato)
                        talen = tail->iov_len - tato;
        } else if (len + to >= buf->page_len) {
                pglen = buf->page_len - to;
                talen = len - pglen;
                if (talen > tail->iov_len)
                        talen = tail->iov_len;
        } else
                pglen = len;

        _copy_from_pages(tail->iov_base + tato, buf->pages,
                         buf->page_base + base + pglen, talen);
        _shift_data_right_pages(buf->pages, buf->page_base + to,
                                buf->page_base + base, pglen);
}

static void xdr_buf_head_copy_right(const struct xdr_buf *buf,
                                    unsigned int base, unsigned int len,
                                    unsigned int shift)
{
        const struct kvec *head = buf->head;
        const struct kvec *tail = buf->tail;
        unsigned int to = base + shift;
        unsigned int pglen = 0, pgto = 0;
        unsigned int talen = 0, tato = 0;

        if (base >= head->iov_len)
                return;
        if (len > head->iov_len - base)
                len = head->iov_len - base;
        if (to >= buf->page_len + head->iov_len) {
                tato = to - buf->page_len - head->iov_len;
                talen = len;
        } else if (to >= head->iov_len) {
                pgto = to - head->iov_len;
                pglen = len;
                if (pgto + pglen > buf->page_len) {
                        talen = pgto + pglen - buf->page_len;
                        pglen -= talen;
                }
        } else {
                pglen = len - to;
                if (pglen > buf->page_len) {
                        talen = pglen - buf->page_len;
                        pglen = buf->page_len;
                }
        }

        len -= talen;
        base += len;
        if (talen + tato > tail->iov_len)
                talen = tail->iov_len > tato ? tail->iov_len - tato : 0;
        memcpy(tail->iov_base + tato, head->iov_base + base, talen);

        len -= pglen;
        base -= pglen;
        _copy_to_pages(buf->pages, buf->page_base + pgto, head->iov_base + base,
                       pglen);

        base -= len;
        memmove(head->iov_base + to, head->iov_base + base, len);
}

static void xdr_buf_tail_shift_right(const struct xdr_buf *buf,
                                     unsigned int base, unsigned int len,
                                     unsigned int shift)
{
        const struct kvec *tail = buf->tail;

        if (base >= tail->iov_len || !shift || !len)
                return;
        xdr_buf_tail_copy_right(buf, base, len, shift);
}

static void xdr_buf_pages_shift_right(const struct xdr_buf *buf,
                                      unsigned int base, unsigned int len,
                                      unsigned int shift)
{
        if (!shift || !len)
                return;
        if (base >= buf->page_len) {
                xdr_buf_tail_shift_right(buf, base - buf->page_len, len, shift);
                return;
        }
        if (base + len > buf->page_len)
                xdr_buf_tail_shift_right(buf, 0, base + len - buf->page_len,
                                         shift);
        xdr_buf_pages_copy_right(buf, base, len, shift);
}

static void xdr_buf_head_shift_right(const struct xdr_buf *buf,
                                     unsigned int base, unsigned int len,
                                     unsigned int shift)
{
        const struct kvec *head = buf->head;

        if (!shift)
                return;
        if (base >= head->iov_len) {
                xdr_buf_pages_shift_right(buf, head->iov_len - base, len,
                                          shift);
                return;
        }
        if (base + len > head->iov_len)
                xdr_buf_pages_shift_right(buf, 0, base + len - head->iov_len,
                                          shift);
        xdr_buf_head_copy_right(buf, base, len, shift);
}

static void xdr_buf_tail_copy_left(const struct xdr_buf *buf, unsigned int base,
                                   unsigned int len, unsigned int shift)
{
        const struct kvec *tail = buf->tail;

        if (base >= tail->iov_len)
                return;
        if (len > tail->iov_len - base)
                len = tail->iov_len - base;
        /* Shift data into head */
        if (shift > buf->page_len + base) {
                const struct kvec *head = buf->head;
                unsigned int hdto =
                        head->iov_len + buf->page_len + base - shift;
                unsigned int hdlen = len;

                if (WARN_ONCE(shift > head->iov_len + buf->page_len + base,
                              "SUNRPC: Misaligned data.\n"))
                        return;
                if (hdto + hdlen > head->iov_len)
                        hdlen = head->iov_len - hdto;
                memcpy(head->iov_base + hdto, tail->iov_base + base, hdlen);
                base += hdlen;
                len -= hdlen;
                if (!len)
                        return;
        }
        /* Shift data into pages */
        if (shift > base) {
                unsigned int pgto = buf->page_len + base - shift;
                unsigned int pglen = len;

                if (pgto + pglen > buf->page_len)
                        pglen = buf->page_len - pgto;
                _copy_to_pages(buf->pages, buf->page_base + pgto,
                               tail->iov_base + base, pglen);
                base += pglen;
                len -= pglen;
                if (!len)
                        return;
        }
        memmove(tail->iov_base + base - shift, tail->iov_base + base, len);
}

static void xdr_buf_pages_copy_left(const struct xdr_buf *buf,
                                    unsigned int base, unsigned int len,
                                    unsigned int shift)
{
        unsigned int pgto;

        if (base >= buf->page_len)
                return;
        if (len > buf->page_len - base)
                len = buf->page_len - base;
        /* Shift data into head */
        if (shift > base) {
                const struct kvec *head = buf->head;
                unsigned int hdto = head->iov_len + base - shift;
                unsigned int hdlen = len;

                if (WARN_ONCE(shift > head->iov_len + base,
                              "SUNRPC: Misaligned data.\n"))
                        return;
                if (hdto + hdlen > head->iov_len)
                        hdlen = head->iov_len - hdto;
                _copy_from_pages(head->iov_base + hdto, buf->pages,
                                 buf->page_base + base, hdlen);
                base += hdlen;
                len -= hdlen;
                if (!len)
                        return;
        }
        pgto = base - shift;
        _shift_data_left_pages(buf->pages, buf->page_base + pgto,
                               buf->page_base + base, len);
}

static void xdr_buf_tail_shift_left(const struct xdr_buf *buf,
                                    unsigned int base, unsigned int len,
                                    unsigned int shift)
{
        if (!shift || !len)
                return;
        xdr_buf_tail_copy_left(buf, base, len, shift);
}

static void xdr_buf_pages_shift_left(const struct xdr_buf *buf,
                                     unsigned int base, unsigned int len,
                                     unsigned int shift)
{
        if (!shift || !len)
                return;
        if (base >= buf->page_len) {
                xdr_buf_tail_shift_left(buf, base - buf->page_len, len, shift);
                return;
        }
        xdr_buf_pages_copy_left(buf, base, len, shift);
        len += base;
        if (len <= buf->page_len)
                return;
        xdr_buf_tail_copy_left(buf, 0, len - buf->page_len, shift);
}

static void xdr_buf_head_shift_left(const struct xdr_buf *buf,
                                    unsigned int base, unsigned int len,
                                    unsigned int shift)
{
        const struct kvec *head = buf->head;
        unsigned int bytes;

        if (!shift || !len)
                return;

        if (shift > base) {
                bytes = (shift - base);
                if (bytes >= len)
                        return;
                base += bytes;
                len -= bytes;
        }

        if (base < head->iov_len) {
                bytes = min_t(unsigned int, len, head->iov_len - base);
                memmove(head->iov_base + (base - shift),
                        head->iov_base + base, bytes);
                base += bytes;
                len -= bytes;
        }
        xdr_buf_pages_shift_left(buf, base - head->iov_len, len, shift);
}

/**
 * xdr_shrink_bufhead
 * @buf: xdr_buf
 * @len: new length of buf->head[0]
 *
 * Shrinks XDR buffer's header kvec buf->head[0], setting it to
 * 'len' bytes. The extra data is not lost, but is instead
 * moved into the inlined pages and/or the tail.
 */
static unsigned int xdr_shrink_bufhead(struct xdr_buf *buf, unsigned int len)
{
        struct kvec *head = buf->head;
        unsigned int shift, buflen = max(buf->len, len);

        WARN_ON_ONCE(len > head->iov_len);
        if (head->iov_len > buflen) {
                buf->buflen -= head->iov_len - buflen;
                head->iov_len = buflen;
        }
        if (len >= head->iov_len)
                return 0;
        shift = head->iov_len - len;
        xdr_buf_try_expand(buf, shift);
        xdr_buf_head_shift_right(buf, len, buflen - len, shift);
        head->iov_len = len;
        buf->buflen -= shift;
        buf->len -= shift;
        return shift;
}

/**
 * xdr_shrink_pagelen - shrinks buf->pages to @len bytes
 * @buf: xdr_buf
 * @len: new page buffer length
 *
 * The extra data is not lost, but is instead moved into buf->tail.
 * Returns the actual number of bytes moved.
 */
static unsigned int xdr_shrink_pagelen(struct xdr_buf *buf, unsigned int len)
{
        unsigned int shift, buflen = buf->len - buf->head->iov_len;

        WARN_ON_ONCE(len > buf->page_len);
        if (buf->head->iov_len >= buf->len || len > buflen)
                buflen = len;
        if (buf->page_len > buflen) {
                buf->buflen -= buf->page_len - buflen;
                buf->page_len = buflen;
        }
        if (len >= buf->page_len)
                return 0;
        shift = buf->page_len - len;
        xdr_buf_try_expand(buf, shift);
        xdr_buf_pages_shift_right(buf, len, buflen - len, shift);
        buf->page_len = len;
        buf->len -= shift;
        buf->buflen -= shift;
        return shift;
}

/**
 * xdr_stream_pos - Return the current offset from the start of the xdr_stream
 * @xdr: pointer to struct xdr_stream
 */
unsigned int xdr_stream_pos(const struct xdr_stream *xdr)
{
        return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;
}
EXPORT_SYMBOL_GPL(xdr_stream_pos);

static void xdr_stream_set_pos(struct xdr_stream *xdr, unsigned int pos)
{
        unsigned int blen = xdr->buf->len;

        xdr->nwords = blen > pos ? XDR_QUADLEN(blen) - XDR_QUADLEN(pos) : 0;
}

static void xdr_stream_page_set_pos(struct xdr_stream *xdr, unsigned int pos)
{
        xdr_stream_set_pos(xdr, pos + xdr->buf->head[0].iov_len);
}

/**
 * xdr_page_pos - Return the current offset from the start of the xdr pages
 * @xdr: pointer to struct xdr_stream
 */
unsigned int xdr_page_pos(const struct xdr_stream *xdr)
{
        unsigned int pos = xdr_stream_pos(xdr);

        WARN_ON(pos < xdr->buf->head[0].iov_len);
        return pos - xdr->buf->head[0].iov_len;
}
EXPORT_SYMBOL_GPL(xdr_page_pos);

/**
 * xdr_init_encode - Initialize a struct xdr_stream for sending data.
 * @xdr: pointer to xdr_stream struct
 * @buf: pointer to XDR buffer in which to encode data
 * @p: current pointer inside XDR buffer
 * @rqst: pointer to controlling rpc_rqst, for debugging
 *
 * Note: at the moment the RPC client only passes the length of our
 *         scratch buffer in the xdr_buf's header kvec. Previously this
 *         meant we needed to call xdr_adjust_iovec() after encoding the
 *         data. With the new scheme, the xdr_stream manages the details
 *         of the buffer length, and takes care of adjusting the kvec
 *         length for us.
 */
void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
                     struct rpc_rqst *rqst)
{
        struct kvec *iov = buf->head;
        int scratch_len = buf->buflen - buf->page_len - buf->tail[0].iov_len;

        xdr_reset_scratch_buffer(xdr);
        BUG_ON(scratch_len < 0);
        xdr->buf = buf;
        xdr->iov = iov;
        xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);
        xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);
        BUG_ON(iov->iov_len > scratch_len);

        if (p != xdr->p && p != NULL) {
                size_t len;

                BUG_ON(p < xdr->p || p > xdr->end);
                len = (char *)p - (char *)xdr->p;
                xdr->p = p;
                buf->len += len;
                iov->iov_len += len;
        }
        xdr->rqst = rqst;
}
EXPORT_SYMBOL_GPL(xdr_init_encode);

/**
 * xdr_init_encode_pages - Initialize an xdr_stream for encoding into pages
 * @xdr: pointer to xdr_stream struct
 * @buf: pointer to XDR buffer into which to encode data
 * @pages: list of pages to decode into
 * @rqst: pointer to controlling rpc_rqst, for debugging
 *
 */
void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
                           struct page **pages, struct rpc_rqst *rqst)
{
        xdr_reset_scratch_buffer(xdr);

        xdr->buf = buf;
        xdr->page_ptr = pages;
        xdr->iov = NULL;
        xdr->p = page_address(*pages);
        xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
        xdr->rqst = rqst;
}
EXPORT_SYMBOL_GPL(xdr_init_encode_pages);

/**
 * __xdr_commit_encode - Ensure all data is written to buffer
 * @xdr: pointer to xdr_stream
 *
 * We handle encoding across page boundaries by giving the caller a
 * temporary location to write to, then later copying the data into
 * place; xdr_commit_encode does that copying.
 *
 * Normally the caller doesn't need to call this directly, as the
 * following xdr_reserve_space will do it.  But an explicit call may be
 * required at the end of encoding, or any other time when the xdr_buf
 * data might be read.
 */
void __xdr_commit_encode(struct xdr_stream *xdr)
{
        size_t shift = xdr->scratch.iov_len;
        void *page;

        page = page_address(*xdr->page_ptr);
        memcpy(xdr->scratch.iov_base, page, shift);
        memmove(page, page + shift, (void *)xdr->p - page);
        xdr_reset_scratch_buffer(xdr);
}
EXPORT_SYMBOL_GPL(__xdr_commit_encode);

/*
 * The buffer space to be reserved crosses the boundary between
 * xdr->buf->head and xdr->buf->pages, or between two pages
 * in xdr->buf->pages.
 */
static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
                                                   size_t nbytes)
{
        int space_left;
        int frag1bytes, frag2bytes;
        void *p;

        if (nbytes > PAGE_SIZE)
                goto out_overflow; /* Bigger buffers require special handling */
        if (xdr->buf->len + nbytes > xdr->buf->buflen)
                goto out_overflow; /* Sorry, we're totally out of space */
        frag1bytes = (xdr->end - xdr->p) << 2;
        frag2bytes = nbytes - frag1bytes;
        if (xdr->iov)
                xdr->iov->iov_len += frag1bytes;
        else
                xdr->buf->page_len += frag1bytes;
        xdr->page_ptr++;
        xdr->iov = NULL;

        /*
         * If the last encode didn't end exactly on a page boundary, the
         * next one will straddle boundaries.  Encode into the next
         * page, then copy it back later in xdr_commit_encode.  We use
         * the "scratch" iov to track any temporarily unused fragment of
         * space at the end of the previous buffer:
         */
        xdr_set_scratch_buffer(xdr, xdr->p, frag1bytes);

        /*
         * xdr->p is where the next encode will start after
         * xdr_commit_encode() has shifted this one back:
         */
        p = page_address(*xdr->page_ptr);
        xdr->p = p + frag2bytes;
        space_left = xdr->buf->buflen - xdr->buf->len;
        if (space_left - frag1bytes >= PAGE_SIZE)
                xdr->end = p + PAGE_SIZE;
        else
                xdr->end = p + space_left - frag1bytes;

        xdr->buf->page_len += frag2bytes;
        xdr->buf->len += nbytes;
        return p;
out_overflow:
        trace_rpc_xdr_overflow(xdr, nbytes);
        return NULL;
}

/**
 * xdr_reserve_space - Reserve buffer space for sending
 * @xdr: pointer to xdr_stream
 * @nbytes: number of bytes to reserve
 *
 * Checks that we have enough buffer space to encode 'nbytes' more
 * bytes of data. If so, update the total xdr_buf length, and
 * adjust the length of the current kvec.
 *
 * The returned pointer is valid only until the next call to
 * xdr_reserve_space() or xdr_commit_encode() on @xdr. The current
 * implementation of this API guarantees that space reserved for a
 * four-byte data item remains valid until @xdr is destroyed, but
 * that might not always be true in the future.
 */
__be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)
{
        __be32 *p = xdr->p;
        __be32 *q;

        xdr_commit_encode(xdr);
        /* align nbytes on the next 32-bit boundary */
        nbytes += 3;
        nbytes &= ~3;
        q = p + (nbytes >> 2);
        if (unlikely(q > xdr->end || q < p))
                return xdr_get_next_encode_buffer(xdr, nbytes);
        xdr->p = q;
        if (xdr->iov)
                xdr->iov->iov_len += nbytes;
        else
                xdr->buf->page_len += nbytes;
        xdr->buf->len += nbytes;
        return p;
}
EXPORT_SYMBOL_GPL(xdr_reserve_space);

/**
 * xdr_reserve_space_vec - Reserves a large amount of buffer space for sending
 * @xdr: pointer to xdr_stream
 * @nbytes: number of bytes to reserve
 *
 * The size argument passed to xdr_reserve_space() is determined based
 * on the number of bytes remaining in the current page to avoid
 * invalidating iov_base pointers when xdr_commit_encode() is called.
 *
 * Return values:
 *   %0: success
 *   %-EMSGSIZE: not enough space is available in @xdr
 */
int xdr_reserve_space_vec(struct xdr_stream *xdr, size_t nbytes)
{
        size_t thislen;
        __be32 *p;

        /*
         * svcrdma requires every READ payload to start somewhere
         * in xdr->pages.
         */
        if (xdr->iov == xdr->buf->head) {
                xdr->iov = NULL;
                xdr->end = xdr->p;
        }

        /* XXX: Let's find a way to make this more efficient */
        while (nbytes) {
                thislen = xdr->buf->page_len % PAGE_SIZE;
                thislen = min_t(size_t, nbytes, PAGE_SIZE - thislen);

                p = xdr_reserve_space(xdr, thislen);
                if (!p)
                        return -EMSGSIZE;

                nbytes -= thislen;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(xdr_reserve_space_vec);

/**
 * xdr_truncate_encode - truncate an encode buffer
 * @xdr: pointer to xdr_stream
 * @len: new length of buffer
 *
 * Truncates the xdr stream, so that xdr->buf->len == len,
 * and xdr->p points at offset len from the start of the buffer, and
 * head, tail, and page lengths are adjusted to correspond.
 *
 * If this means moving xdr->p to a different buffer, we assume that
 * the end pointer should be set to the end of the current page,
 * except in the case of the head buffer when we assume the head
 * buffer's current length represents the end of the available buffer.
 *
 * This is *not* safe to use on a buffer that already has inlined page
 * cache pages (as in a zero-copy server read reply), except for the
 * simple case of truncating from one position in the tail to another.
 *
 */
void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)
{
        struct xdr_buf *buf = xdr->buf;
        struct kvec *head = buf->head;
        struct kvec *tail = buf->tail;
        int fraglen;
        int new;

        if (len > buf->len) {
                WARN_ON_ONCE(1);
                return;
        }
        xdr_commit_encode(xdr);

        fraglen = min_t(int, buf->len - len, tail->iov_len);
        tail->iov_len -= fraglen;
        buf->len -= fraglen;
        if (tail->iov_len) {
                xdr->p = tail->iov_base + tail->iov_len;
                WARN_ON_ONCE(!xdr->end);
                WARN_ON_ONCE(!xdr->iov);
                return;
        }
        WARN_ON_ONCE(fraglen);
        fraglen = min_t(int, buf->len - len, buf->page_len);
        buf->page_len -= fraglen;
        buf->len -= fraglen;

        new = buf->page_base + buf->page_len;

        xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);

        if (buf->page_len) {
                xdr->p = page_address(*xdr->page_ptr);
                xdr->end = (void *)xdr->p + PAGE_SIZE;
                xdr->p = (void *)xdr->p + (new % PAGE_SIZE);
                WARN_ON_ONCE(xdr->iov);
                return;
        }
        if (fraglen)
                xdr->end = head->iov_base + head->iov_len;
        /* (otherwise assume xdr->end is already set) */
        xdr->page_ptr--;
        head->iov_len = len;
        buf->len = len;
        xdr->p = head->iov_base + head->iov_len;
        xdr->iov = buf->head;
}
EXPORT_SYMBOL(xdr_truncate_encode);

/**
 * xdr_truncate_decode - Truncate a decoding stream
 * @xdr: pointer to struct xdr_stream
 * @len: Number of bytes to remove
 *
 */
void xdr_truncate_decode(struct xdr_stream *xdr, size_t len)
{
        unsigned int nbytes = xdr_align_size(len);

        xdr->buf->len -= nbytes;
        xdr->nwords -= XDR_QUADLEN(nbytes);
}
EXPORT_SYMBOL_GPL(xdr_truncate_decode);

/**
 * xdr_restrict_buflen - decrease available buffer space
 * @xdr: pointer to xdr_stream
 * @newbuflen: new maximum number of bytes available
 *
 * Adjust our idea of how much space is available in the buffer.
 * If we've already used too much space in the buffer, returns -1.
 * If the available space is already smaller than newbuflen, returns 0
 * and does nothing.  Otherwise, adjusts xdr->buf->buflen to newbuflen
 * and ensures xdr->end is set at most offset newbuflen from the start
 * of the buffer.
 */
int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)
{
        struct xdr_buf *buf = xdr->buf;
        int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;
        int end_offset = buf->len + left_in_this_buf;

        if (newbuflen < 0 || newbuflen < buf->len)
                return -1;
        if (newbuflen > buf->buflen)
                return 0;
        if (newbuflen < end_offset)
                xdr->end = (void *)xdr->end + newbuflen - end_offset;
        buf->buflen = newbuflen;
        return 0;
}
EXPORT_SYMBOL(xdr_restrict_buflen);

/**
 * xdr_write_pages - Insert a list of pages into an XDR buffer for sending
 * @xdr: pointer to xdr_stream
 * @pages: array of pages to insert
 * @base: starting offset of first data byte in @pages
 * @len: number of data bytes in @pages to insert
 *
 * After the @pages are added, the tail iovec is instantiated pointing to
 * end of the head buffer, and the stream is set up to encode subsequent
 * items into the tail.
 */
void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,
                 unsigned int len)
{
        struct xdr_buf *buf = xdr->buf;
        struct kvec *tail = buf->tail;

        buf->pages = pages;
        buf->page_base = base;
        buf->page_len = len;

        tail->iov_base = xdr->p;
        tail->iov_len = 0;
        xdr->iov = tail;

        if (len & 3) {
                unsigned int pad = 4 - (len & 3);

                BUG_ON(xdr->p >= xdr->end);
                tail->iov_base = (char *)xdr->p + (len & 3);
                tail->iov_len += pad;
                len += pad;
                *xdr->p++ = 0;
        }
        buf->buflen += len;
        buf->len += len;
}
EXPORT_SYMBOL_GPL(xdr_write_pages);

static unsigned int xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,
                                unsigned int base, unsigned int len)
{
        if (len > iov->iov_len)
                len = iov->iov_len;
        if (unlikely(base > len))
                base = len;
        xdr->p = (__be32*)(iov->iov_base + base);
        xdr->end = (__be32*)(iov->iov_base + len);
        xdr->iov = iov;
        xdr->page_ptr = NULL;
        return len - base;
}

static unsigned int xdr_set_tail_base(struct xdr_stream *xdr,
                                      unsigned int base, unsigned int len)
{
        struct xdr_buf *buf = xdr->buf;

        xdr_stream_set_pos(xdr, base + buf->page_len + buf->head->iov_len);
        return xdr_set_iov(xdr, buf->tail, base, len);
}

static void xdr_stream_unmap_current_page(struct xdr_stream *xdr)
{
        if (xdr->page_kaddr) {
                kunmap_local(xdr->page_kaddr);
                xdr->page_kaddr = NULL;
        }
}

static unsigned int xdr_set_page_base(struct xdr_stream *xdr,
                                      unsigned int base, unsigned int len)
{
        unsigned int pgnr;
        unsigned int maxlen;
        unsigned int pgoff;
        unsigned int pgend;
        void *kaddr;

        maxlen = xdr->buf->page_len;
        if (base >= maxlen)
                return 0;
        else
                maxlen -= base;
        if (len > maxlen)
                len = maxlen;

        xdr_stream_unmap_current_page(xdr);
        xdr_stream_page_set_pos(xdr, base);
        base += xdr->buf->page_base;

        pgnr = base >> PAGE_SHIFT;
        xdr->page_ptr = &xdr->buf->pages[pgnr];

        if (PageHighMem(*xdr->page_ptr)) {
                xdr->page_kaddr = kmap_local_page(*xdr->page_ptr);
                kaddr = xdr->page_kaddr;
        } else
                kaddr = page_address(*xdr->page_ptr);

        pgoff = base & ~PAGE_MASK;
        xdr->p = (__be32*)(kaddr + pgoff);

        pgend = pgoff + len;
        if (pgend > PAGE_SIZE)
                pgend = PAGE_SIZE;
        xdr->end = (__be32*)(kaddr + pgend);
        xdr->iov = NULL;
        return len;
}

static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,
                         unsigned int len)
{
        if (xdr_set_page_base(xdr, base, len) == 0) {
                base -= xdr->buf->page_len;
                xdr_set_tail_base(xdr, base, len);
        }
}

static void xdr_set_next_page(struct xdr_stream *xdr)
{
        unsigned int newbase;

        newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;
        newbase -= xdr->buf->page_base;
        if (newbase < xdr->buf->page_len)
                xdr_set_page_base(xdr, newbase, xdr_stream_remaining(xdr));
        else
                xdr_set_tail_base(xdr, 0, xdr_stream_remaining(xdr));
}

static bool xdr_set_next_buffer(struct xdr_stream *xdr)
{
        if (xdr->page_ptr != NULL)
                xdr_set_next_page(xdr);
        else if (xdr->iov == xdr->buf->head)
                xdr_set_page(xdr, 0, xdr_stream_remaining(xdr));
        return xdr->p != xdr->end;
}

/**
 * xdr_init_decode - Initialize an xdr_stream for decoding data.
 * @xdr: pointer to xdr_stream struct
 * @buf: pointer to XDR buffer from which to decode data
 * @p: current pointer inside XDR buffer
 * @rqst: pointer to controlling rpc_rqst, for debugging
 */
void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
                     struct rpc_rqst *rqst)
{
        xdr->buf = buf;
        xdr->page_kaddr = NULL;
        xdr_reset_scratch_buffer(xdr);
        xdr->nwords = XDR_QUADLEN(buf->len);
        if (xdr_set_iov(xdr, buf->head, 0, buf->len) == 0 &&
            xdr_set_page_base(xdr, 0, buf->len) == 0)
                xdr_set_iov(xdr, buf->tail, 0, buf->len);
        if (p != NULL && p > xdr->p && xdr->end >= p) {
                xdr->nwords -= p - xdr->p;
                xdr->p = p;
        }
        xdr->rqst = rqst;
}
EXPORT_SYMBOL_GPL(xdr_init_decode);

/**
 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages
 * @xdr: pointer to xdr_stream struct
 * @buf: pointer to XDR buffer from which to decode data
 * @pages: list of pages to decode into
 * @len: length in bytes of buffer in pages
 */
void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
                           struct page **pages, unsigned int len)
{
        memset(buf, 0, sizeof(*buf));
        buf->pages =  pages;
        buf->page_len =  len;
        buf->buflen =  len;
        buf->len = len;
        xdr_init_decode(xdr, buf, NULL, NULL);
}
EXPORT_SYMBOL_GPL(xdr_init_decode_pages);

/**
 * xdr_finish_decode - Clean up the xdr_stream after decoding data.
 * @xdr: pointer to xdr_stream struct
 */
void xdr_finish_decode(struct xdr_stream *xdr)
{
        xdr_stream_unmap_current_page(xdr);
}
EXPORT_SYMBOL(xdr_finish_decode);

static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
{
        unsigned int nwords = XDR_QUADLEN(nbytes);
        __be32 *p = xdr->p;
        __be32 *q = p + nwords;

        if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))
                return NULL;
        xdr->p = q;
        xdr->nwords -= nwords;
        return p;
}

static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)
{
        __be32 *p;
        char *cpdest = xdr->scratch.iov_base;
        size_t cplen = (char *)xdr->end - (char *)xdr->p;

        if (nbytes > xdr->scratch.iov_len)
                goto out_overflow;
        p = __xdr_inline_decode(xdr, cplen);
        if (p == NULL)
                return NULL;
        memcpy(cpdest, p, cplen);
        if (!xdr_set_next_buffer(xdr))
                goto out_overflow;
        cpdest += cplen;
        nbytes -= cplen;
        p = __xdr_inline_decode(xdr, nbytes);
        if (p == NULL)
                return NULL;
        memcpy(cpdest, p, nbytes);
        return xdr->scratch.iov_base;
out_overflow:
        trace_rpc_xdr_overflow(xdr, nbytes);
        return NULL;
}

/**
 * xdr_inline_decode - Retrieve XDR data to decode
 * @xdr: pointer to xdr_stream struct
 * @nbytes: number of bytes of data to decode
 *
 * Check if the input buffer is long enough to enable us to decode
 * 'nbytes' more bytes of data starting at the current position.
 * If so return the current pointer, then update the current
 * pointer position.
 */
__be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
{
        __be32 *p;

        if (unlikely(nbytes == 0))
                return xdr->p;
        if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
                goto out_overflow;
        p = __xdr_inline_decode(xdr, nbytes);
        if (p != NULL)
                return p;
        return xdr_copy_to_scratch(xdr, nbytes);
out_overflow:
        trace_rpc_xdr_overflow(xdr, nbytes);
        return NULL;
}
EXPORT_SYMBOL_GPL(xdr_inline_decode);

static void xdr_realign_pages(struct xdr_stream *xdr)
{
        struct xdr_buf *buf = xdr->buf;
        struct kvec *iov = buf->head;
        unsigned int cur = xdr_stream_pos(xdr);
        unsigned int copied;

        /* Realign pages to current pointer position */
        if (iov->iov_len > cur) {
                copied = xdr_shrink_bufhead(buf, cur);
                trace_rpc_xdr_alignment(xdr, cur, copied);
                xdr_set_page(xdr, 0, buf->page_len);
        }
}

static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
{
        struct xdr_buf *buf = xdr->buf;
        unsigned int nwords = XDR_QUADLEN(len);
        unsigned int copied;

        if (xdr->nwords == 0)
                return 0;

        xdr_realign_pages(xdr);
        if (nwords > xdr->nwords) {
                nwords = xdr->nwords;
                len = nwords << 2;
        }
        if (buf->page_len <= len)
                len = buf->page_len;
        else if (nwords < xdr->nwords) {
                /* Truncate page data and move it into the tail */
                copied = xdr_shrink_pagelen(buf, len);
                trace_rpc_xdr_alignment(xdr, len, copied);
        }
        return len;
}

/**
 * xdr_read_pages - align page-based XDR data to current pointer position
 * @xdr: pointer to xdr_stream struct
 * @len: number of bytes of page data
 *
 * Moves data beyond the current pointer position from the XDR head[] buffer
 * into the page list. Any data that lies beyond current position + @len
 * bytes is moved into the XDR tail[]. The xdr_stream current position is
 * then advanced past that data to align to the next XDR object in the tail.
 *
 * Returns the number of XDR encoded bytes now contained in the pages
 */
unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)
{
        unsigned int nwords = XDR_QUADLEN(len);
        unsigned int base, end, pglen;

        pglen = xdr_align_pages(xdr, nwords << 2);
        if (pglen == 0)
                return 0;

        base = (nwords << 2) - pglen;
        end = xdr_stream_remaining(xdr) - pglen;

        xdr_set_tail_base(xdr, base, end);
        return len <= pglen ? len : pglen;
}
EXPORT_SYMBOL_GPL(xdr_read_pages);

/**
 * xdr_set_pagelen - Sets the length of the XDR pages
 * @xdr: pointer to xdr_stream struct
 * @len: new length of the XDR page data
 *
 * Either grows or shrinks the length of the xdr pages by setting pagelen to
 * @len bytes. When shrinking, any extra data is moved into buf->tail, whereas
 * when growing any data beyond the current pointer is moved into the tail.
 *
 * Returns True if the operation was successful, and False otherwise.
 */
void xdr_set_pagelen(struct xdr_stream *xdr, unsigned int len)
{
        struct xdr_buf *buf = xdr->buf;
        size_t remaining = xdr_stream_remaining(xdr);
        size_t base = 0;

        if (len < buf->page_len) {
                base = buf->page_len - len;
                xdr_shrink_pagelen(buf, len);
        } else {
                xdr_buf_head_shift_right(buf, xdr_stream_pos(xdr),
                                         buf->page_len, remaining);
                if (len > buf->page_len)
                        xdr_buf_try_expand(buf, len - buf->page_len);
        }
        xdr_set_tail_base(xdr, base, remaining);
}
EXPORT_SYMBOL_GPL(xdr_set_pagelen);

/**
 * xdr_enter_page - decode data from the XDR page
 * @xdr: pointer to xdr_stream struct
 * @len: number of bytes of page data
 *
 * Moves data beyond the current pointer position from the XDR head[] buffer
 * into the page list. Any data that lies beyond current position + "len"
 * bytes is moved into the XDR tail[]. The current pointer is then
 * repositioned at the beginning of the first XDR page.
 */
void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)
{
        len = xdr_align_pages(xdr, len);
        /*
         * Position current pointer at beginning of tail, and
         * set remaining message length.
         */
        if (len != 0)
                xdr_set_page_base(xdr, 0, len);
}
EXPORT_SYMBOL_GPL(xdr_enter_page);

static const struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0};

void xdr_buf_from_iov(const struct kvec *iov, struct xdr_buf *buf)
{
        buf->head[0] = *iov;
        buf->tail[0] = empty_iov;
        buf->page_len = 0;
        buf->buflen = buf->len = iov->iov_len;
}
EXPORT_SYMBOL_GPL(xdr_buf_from_iov);

/**
 * xdr_buf_subsegment - set subbuf to a portion of buf
 * @buf: an xdr buffer
 * @subbuf: the result buffer
 * @base: beginning of range in bytes
 * @len: length of range in bytes
 *
 * sets @subbuf to an xdr buffer representing the portion of @buf of
 * length @len starting at offset @base.
 *
 * @buf and @subbuf may be pointers to the same struct xdr_buf.
 *
 * Returns -1 if base or length are out of bounds.
 */
int xdr_buf_subsegment(const struct xdr_buf *buf, struct xdr_buf *subbuf,
                       unsigned int base, unsigned int len)
{
        subbuf->buflen = subbuf->len = len;
        if (base < buf->head[0].iov_len) {
                subbuf->head[0].iov_base = buf->head[0].iov_base + base;
                subbuf->head[0].iov_len = min_t(unsigned int, len,
                                                buf->head[0].iov_len - base);
                len -= subbuf->head[0].iov_len;
                base = 0;
        } else {
                base -= buf->head[0].iov_len;
                subbuf->head[0].iov_base = buf->head[0].iov_base;
                subbuf->head[0].iov_len = 0;
        }

        if (base < buf->page_len) {
                subbuf->page_len = min(buf->page_len - base, len);
                base += buf->page_base;
                subbuf->page_base = base & ~PAGE_MASK;
                subbuf->pages = &buf->pages[base >> PAGE_SHIFT];
                len -= subbuf->page_len;
                base = 0;
        } else {
                base -= buf->page_len;
                subbuf->pages = buf->pages;
                subbuf->page_base = 0;
                subbuf->page_len = 0;
        }

        if (base < buf->tail[0].iov_len) {
                subbuf->tail[0].iov_base = buf->tail[0].iov_base + base;
                subbuf->tail[0].iov_len = min_t(unsigned int, len,
                                                buf->tail[0].iov_len - base);
                len -= subbuf->tail[0].iov_len;
                base = 0;
        } else {
                base -= buf->tail[0].iov_len;
                subbuf->tail[0].iov_base = buf->tail[0].iov_base;
                subbuf->tail[0].iov_len = 0;
        }

        if (base || len)
                return -1;
        return 0;
}
EXPORT_SYMBOL_GPL(xdr_buf_subsegment);

/**
 * xdr_stream_subsegment - set @subbuf to a portion of @xdr
 * @xdr: an xdr_stream set up for decoding
 * @subbuf: the result buffer
 * @nbytes: length of @xdr to extract, in bytes
 *
 * Sets up @subbuf to represent a portion of @xdr. The portion
 * starts at the current offset in @xdr, and extends for a length
 * of @nbytes. If this is successful, @xdr is advanced to the next
 * XDR data item following that portion.
 *
 * Return values:
 *   %true: @subbuf has been initialized, and @xdr has been advanced.
 *   %false: a bounds error has occurred
 */
bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf,
                           unsigned int nbytes)
{
        unsigned int start = xdr_stream_pos(xdr);
        unsigned int remaining, len;

        /* Extract @subbuf and bounds-check the fn arguments */
        if (xdr_buf_subsegment(xdr->buf, subbuf, start, nbytes))
                return false;

        /* Advance @xdr by @nbytes */
        for (remaining = nbytes; remaining;) {
                if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
                        return false;

                len = (char *)xdr->end - (char *)xdr->p;
                if (remaining <= len) {
                        xdr->p = (__be32 *)((char *)xdr->p +
                                        (remaining + xdr_pad_size(nbytes)));
                        break;
                }

                xdr->p = (__be32 *)((char *)xdr->p + len);
                xdr->end = xdr->p;
                remaining -= len;
        }

        xdr_stream_set_pos(xdr, start + nbytes);
        return true;
}
EXPORT_SYMBOL_GPL(xdr_stream_subsegment);

/**
 * xdr_stream_move_subsegment - Move part of a stream to another position
 * @xdr: the source xdr_stream
 * @offset: the source offset of the segment
 * @target: the target offset of the segment
 * @length: the number of bytes to move
 *
 * Moves @length bytes from @offset to @target in the xdr_stream, overwriting
 * anything in its space. Returns the number of bytes in the segment.
 */
unsigned int xdr_stream_move_subsegment(struct xdr_stream *xdr, unsigned int offset,
                                        unsigned int target, unsigned int length)
{
        struct xdr_buf buf;
        unsigned int shift;

        if (offset < target) {
                shift = target - offset;
                if (xdr_buf_subsegment(xdr->buf, &buf, offset, shift + length) < 0)
                        return 0;
                xdr_buf_head_shift_right(&buf, 0, length, shift);
        } else if (offset > target) {
                shift = offset - target;
                if (xdr_buf_subsegment(xdr->buf, &buf, target, shift + length) < 0)
                        return 0;
                xdr_buf_head_shift_left(&buf, shift, length, shift);
        }
        return length;
}
EXPORT_SYMBOL_GPL(xdr_stream_move_subsegment);

/**
 * xdr_stream_zero - zero out a portion of an xdr_stream
 * @xdr: an xdr_stream to zero out
 * @offset: the starting point in the stream
 * @length: the number of bytes to zero
 */
unsigned int xdr_stream_zero(struct xdr_stream *xdr, unsigned int offset,
                             unsigned int length)
{
        struct xdr_buf buf;

        if (xdr_buf_subsegment(xdr->buf, &buf, offset, length) < 0)
                return 0;
        if (buf.head[0].iov_len)
                xdr_buf_iov_zero(buf.head, 0, buf.head[0].iov_len);
        if (buf.page_len > 0)
                xdr_buf_pages_zero(&buf, 0, buf.page_len);
        if (buf.tail[0].iov_len)
                xdr_buf_iov_zero(buf.tail, 0, buf.tail[0].iov_len);
        return length;
}
EXPORT_SYMBOL_GPL(xdr_stream_zero);

/**
 * xdr_buf_trim - lop at most "len" bytes off the end of "buf"
 * @buf: buf to be trimmed
 * @len: number of bytes to reduce "buf" by
 *
 * Trim an xdr_buf by the given number of bytes by fixing up the lengths. Note
 * that it's possible that we'll trim less than that amount if the xdr_buf is
 * too small, or if (for instance) it's all in the head and the parser has
 * already read too far into it.
 */
void xdr_buf_trim(struct xdr_buf *buf, unsigned int len)
{
        size_t cur;
        unsigned int trim = len;

        if (buf->tail[0].iov_len) {
                cur = min_t(size_t, buf->tail[0].iov_len, trim);
                buf->tail[0].iov_len -= cur;
                trim -= cur;
                if (!trim)
                        goto fix_len;
        }

        if (buf->page_len) {
                cur = min_t(unsigned int, buf->page_len, trim);
                buf->page_len -= cur;
                trim -= cur;
                if (!trim)
                        goto fix_len;
        }

        if (buf->head[0].iov_len) {
                cur = min_t(size_t, buf->head[0].iov_len, trim);
                buf->head[0].iov_len -= cur;
                trim -= cur;
        }
fix_len:
        buf->len -= (len - trim);
}
EXPORT_SYMBOL_GPL(xdr_buf_trim);

static void __read_bytes_from_xdr_buf(const struct xdr_buf *subbuf,
                                      void *obj, unsigned int len)
{
        unsigned int this_len;

        this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
        memcpy(obj, subbuf->head[0].iov_base, this_len);
        len -= this_len;
        obj += this_len;
        this_len = min_t(unsigned int, len, subbuf->page_len);
        _copy_from_pages(obj, subbuf->pages, subbuf->page_base, this_len);
        len -= this_len;
        obj += this_len;
        this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
        memcpy(obj, subbuf->tail[0].iov_base, this_len);
}

/* obj is assumed to point to allocated memory of size at least len: */
int read_bytes_from_xdr_buf(const struct xdr_buf *buf, unsigned int base,
                            void *obj, unsigned int len)
{
        struct xdr_buf subbuf;
        int status;

        status = xdr_buf_subsegment(buf, &subbuf, base, len);
        if (status != 0)
                return status;
        __read_bytes_from_xdr_buf(&subbuf, obj, len);
        return 0;
}
EXPORT_SYMBOL_GPL(read_bytes_from_xdr_buf);

static void __write_bytes_to_xdr_buf(const struct xdr_buf *subbuf,
                                     void *obj, unsigned int len)
{
        unsigned int this_len;

        this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
        memcpy(subbuf->head[0].iov_base, obj, this_len);
        len -= this_len;
        obj += this_len;
        this_len = min_t(unsigned int, len, subbuf->page_len);
        _copy_to_pages(subbuf->pages, subbuf->page_base, obj, this_len);
        len -= this_len;
        obj += this_len;
        this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
        memcpy(subbuf->tail[0].iov_base, obj, this_len);
}

/* obj is assumed to point to allocated memory of size at least len: */
int write_bytes_to_xdr_buf(const struct xdr_buf *buf, unsigned int base,
                           void *obj, unsigned int len)
{
        struct xdr_buf subbuf;
        int status;

        status = xdr_buf_subsegment(buf, &subbuf, base, len);
        if (status != 0)
                return status;
        __write_bytes_to_xdr_buf(&subbuf, obj, len);
        return 0;
}
EXPORT_SYMBOL_GPL(write_bytes_to_xdr_buf);

int xdr_decode_word(const struct xdr_buf *buf, unsigned int base, u32 *obj)
{
        __be32        raw;
        int        status;

        status = read_bytes_from_xdr_buf(buf, base, &raw, sizeof(*obj));
        if (status)
                return status;
        *obj = be32_to_cpu(raw);
        return 0;
}
EXPORT_SYMBOL_GPL(xdr_decode_word);

int xdr_encode_word(const struct xdr_buf *buf, unsigned int base, u32 obj)
{
        __be32        raw = cpu_to_be32(obj);

        return write_bytes_to_xdr_buf(buf, base, &raw, sizeof(obj));
}
EXPORT_SYMBOL_GPL(xdr_encode_word);

/* Returns 0 on success, or else a negative error code. */
static int xdr_xcode_array2(const struct xdr_buf *buf, unsigned int base,
                            struct xdr_array2_desc *desc, int encode)
{
        char *elem = NULL, *c;
        unsigned int copied = 0, todo, avail_here;
        struct page **ppages = NULL;
        int err;

        if (encode) {
                if (xdr_encode_word(buf, base, desc->array_len) != 0)
                        return -EINVAL;
        } else {
                if (xdr_decode_word(buf, base, &desc->array_len) != 0 ||
                    desc->array_len > desc->array_maxlen ||
                    (unsigned long) base + 4 + desc->array_len *
                                    desc->elem_size > buf->len)
                        return -EINVAL;
        }
        base += 4;

        if (!desc->xcode)
                return 0;

        todo = desc->array_len * desc->elem_size;

        /* process head */
        if (todo && base < buf->head->iov_len) {
                c = buf->head->iov_base + base;
                avail_here = min_t(unsigned int, todo,
                                   buf->head->iov_len - base);
                todo -= avail_here;

                while (avail_here >= desc->elem_size) {
                        err = desc->xcode(desc, c);
                        if (err)
                                goto out;
                        c += desc->elem_size;
                        avail_here -= desc->elem_size;
                }
                if (avail_here) {
                        if (!elem) {
                                elem = kmalloc(desc->elem_size, GFP_KERNEL);
                                err = -ENOMEM;
                                if (!elem)
                                        goto out;
                        }
                        if (encode) {
                                err = desc->xcode(desc, elem);
                                if (err)
                                        goto out;
                                memcpy(c, elem, avail_here);
                        } else
                                memcpy(elem, c, avail_here);
                        copied = avail_here;
                }
                base = buf->head->iov_len;  /* align to start of pages */
        }

        /* process pages array */
        base -= buf->head->iov_len;
        if (todo && base < buf->page_len) {
                unsigned int avail_page;

                avail_here = min(todo, buf->page_len - base);
                todo -= avail_here;

                base += buf->page_base;
                ppages = buf->pages + (base >> PAGE_SHIFT);
                base &= ~PAGE_MASK;
                avail_page = min_t(unsigned int, PAGE_SIZE - base,
                                        avail_here);
                c = kmap(*ppages) + base;

                while (avail_here) {
                        avail_here -= avail_page;
                        if (copied || avail_page < desc->elem_size) {
                                unsigned int l = min(avail_page,
                                        desc->elem_size - copied);
                                if (!elem) {
                                        elem = kmalloc(desc->elem_size,
                                                       GFP_KERNEL);
                                        err = -ENOMEM;
                                        if (!elem)
                                                goto out;
                                }
                                if (encode) {
                                        if (!copied) {
                                                err = desc->xcode(desc, elem);
                                                if (err)
                                                        goto out;
                                        }
                                        memcpy(c, elem + copied, l);
                                        copied += l;
                                        if (copied == desc->elem_size)
                                                copied = 0;
                                } else {
                                        memcpy(elem + copied, c, l);
                                        copied += l;
                                        if (copied == desc->elem_size) {
                                                err = desc->xcode(desc, elem);
                                                if (err)
                                                        goto out;
                                                copied = 0;
                                        }
                                }
                                avail_page -= l;
                                c += l;
                        }
                        while (avail_page >= desc->elem_size) {
                                err = desc->xcode(desc, c);
                                if (err)
                                        goto out;
                                c += desc->elem_size;
                                avail_page -= desc->elem_size;
                        }
                        if (avail_page) {
                                unsigned int l = min(avail_page,
                                            desc->elem_size - copied);
                                if (!elem) {
                                        elem = kmalloc(desc->elem_size,
                                                       GFP_KERNEL);
                                        err = -ENOMEM;
                                        if (!elem)
                                                goto out;
                                }
                                if (encode) {
                                        if (!copied) {
                                                err = desc->xcode(desc, elem);
                                                if (err)
                                                        goto out;
                                        }
                                        memcpy(c, elem + copied, l);
                                        copied += l;
                                        if (copied == desc->elem_size)
                                                copied = 0;
                                } else {
                                        memcpy(elem + copied, c, l);
                                        copied += l;
                                        if (copied == desc->elem_size) {
                                                err = desc->xcode(desc, elem);
                                                if (err)
                                                        goto out;
                                                copied = 0;
                                        }
                                }
                        }
                        if (avail_here) {
                                kunmap(*ppages);
                                ppages++;
                                c = kmap(*ppages);
                        }

                        avail_page = min(avail_here,
                                 (unsigned int) PAGE_SIZE);
                }
                base = buf->page_len;  /* align to start of tail */
        }

        /* process tail */
        base -= buf->page_len;
        if (todo) {
                c = buf->tail->iov_base + base;
                if (copied) {
                        unsigned int l = desc->elem_size - copied;

                        if (encode)
                                memcpy(c, elem + copied, l);
                        else {
                                memcpy(elem + copied, c, l);
                                err = desc->xcode(desc, elem);
                                if (err)
                                        goto out;
                        }
                        todo -= l;
                        c += l;
                }
                while (todo) {
                        err = desc->xcode(desc, c);
                        if (err)
                                goto out;
                        c += desc->elem_size;
                        todo -= desc->elem_size;
                }
        }
        err = 0;

out:
        kfree(elem);
        if (ppages)
                kunmap(*ppages);
        return err;
}

int xdr_decode_array2(const struct xdr_buf *buf, unsigned int base,
                      struct xdr_array2_desc *desc)
{
        if (base >= buf->len)
                return -EINVAL;

        return xdr_xcode_array2(buf, base, desc, 0);
}
EXPORT_SYMBOL_GPL(xdr_decode_array2);

int xdr_encode_array2(const struct xdr_buf *buf, unsigned int base,
                      struct xdr_array2_desc *desc)
{
        if ((unsigned long) base + 4 + desc->array_len * desc->elem_size >
            buf->head->iov_len + buf->page_len + buf->tail->iov_len)
                return -EINVAL;

        return xdr_xcode_array2(buf, base, desc, 1);
}
EXPORT_SYMBOL_GPL(xdr_encode_array2);

int xdr_process_buf(const struct xdr_buf *buf, unsigned int offset,
                    unsigned int len,
                    int (*actor)(struct scatterlist *, void *), void *data)
{
        int i, ret = 0;
        unsigned int page_len, thislen, page_offset;
        struct scatterlist      sg[1];

        sg_init_table(sg, 1);

        if (offset >= buf->head[0].iov_len) {
                offset -= buf->head[0].iov_len;
        } else {
                thislen = buf->head[0].iov_len - offset;
                if (thislen > len)
                        thislen = len;
                sg_set_buf(sg, buf->head[0].iov_base + offset, thislen);
                ret = actor(sg, data);
                if (ret)
                        goto out;
                offset = 0;
                len -= thislen;
        }
        if (len == 0)
                goto out;

        if (offset >= buf->page_len) {
                offset -= buf->page_len;
        } else {
                page_len = buf->page_len - offset;
                if (page_len > len)
                        page_len = len;
                len -= page_len;
                page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1);
                i = (offset + buf->page_base) >> PAGE_SHIFT;
                thislen = PAGE_SIZE - page_offset;
                do {
                        if (thislen > page_len)
                                thislen = page_len;
                        sg_set_page(sg, buf->pages[i], thislen, page_offset);
                        ret = actor(sg, data);
                        if (ret)
                                goto out;
                        page_len -= thislen;
                        i++;
                        page_offset = 0;
                        thislen = PAGE_SIZE;
                } while (page_len != 0);
                offset = 0;
        }
        if (len == 0)
                goto out;
        if (offset < buf->tail[0].iov_len) {
                thislen = buf->tail[0].iov_len - offset;
                if (thislen > len)
                        thislen = len;
                sg_set_buf(sg, buf->tail[0].iov_base + offset, thislen);
                ret = actor(sg, data);
                len -= thislen;
        }
        if (len != 0)
                ret = -EINVAL;
out:
        return ret;
}
EXPORT_SYMBOL_GPL(xdr_process_buf);

/**
 * xdr_stream_decode_opaque - Decode variable length opaque
 * @xdr: pointer to xdr_stream
 * @ptr: location to store opaque data
 * @size: size of storage buffer @ptr
 *
 * Return values:
 *   On success, returns size of object stored in *@ptr
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE on overflow of storage buffer @ptr
 */
ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)
{
        ssize_t ret;
        void *p;

        ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
        if (ret <= 0)
                return ret;
        memcpy(ptr, p, ret);
        return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque);

/**
 * xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque
 * @xdr: pointer to xdr_stream
 * @ptr: location to store pointer to opaque data
 * @maxlen: maximum acceptable object size
 * @gfp_flags: GFP mask to use
 *
 * Return values:
 *   On success, returns size of object stored in *@ptr
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE if the size of the object would exceed @maxlen
 *   %-ENOMEM on memory allocation failure
 */
ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,
                size_t maxlen, gfp_t gfp_flags)
{
        ssize_t ret;
        void *p;

        ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
        if (ret > 0) {
                *ptr = kmemdup(p, ret, gfp_flags);
                if (*ptr != NULL)
                        return ret;
                ret = -ENOMEM;
        }
        *ptr = NULL;
        return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup);

/**
 * xdr_stream_decode_string - Decode variable length string
 * @xdr: pointer to xdr_stream
 * @str: location to store string
 * @size: size of storage buffer @str
 *
 * Return values:
 *   On success, returns length of NUL-terminated string stored in *@str
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE on overflow of storage buffer @str
 */
ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)
{
        ssize_t ret;
        void *p;

        ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
        if (ret > 0) {
                memcpy(str, p, ret);
                str[ret] = '\0';
                return strlen(str);
        }
        *str = '\0';
        return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_string);

/**
 * xdr_stream_decode_string_dup - Decode and duplicate variable length string
 * @xdr: pointer to xdr_stream
 * @str: location to store pointer to string
 * @maxlen: maximum acceptable string length
 * @gfp_flags: GFP mask to use
 *
 * Return values:
 *   On success, returns length of NUL-terminated string stored in *@ptr
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE if the size of the string would exceed @maxlen
 *   %-ENOMEM on memory allocation failure
 */
ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,
                size_t maxlen, gfp_t gfp_flags)
{
        void *p;
        ssize_t ret;

        ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
        if (ret > 0) {
                char *s = kmemdup_nul(p, ret, gfp_flags);
                if (s != NULL) {
                        *str = s;
                        return strlen(s);
                }
                ret = -ENOMEM;
        }
        *str = NULL;
        return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_string_dup);

/**
 * xdr_stream_decode_opaque_auth - Decode struct opaque_auth (RFC5531 S8.2)
 * @xdr: pointer to xdr_stream
 * @flavor: location to store decoded flavor
 * @body: location to store decode body
 * @body_len: location to store length of decoded body
 *
 * Return values:
 *   On success, returns the number of buffer bytes consumed
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE if the decoded size of the body field exceeds 400 octets
 */
ssize_t xdr_stream_decode_opaque_auth(struct xdr_stream *xdr, u32 *flavor,
                                      void **body, unsigned int *body_len)
{
        ssize_t ret, len;

        len = xdr_stream_decode_u32(xdr, flavor);
        if (unlikely(len < 0))
                return len;
        ret = xdr_stream_decode_opaque_inline(xdr, body, RPC_MAX_AUTH_SIZE);
        if (unlikely(ret < 0))
                return ret;
        *body_len = ret;
        return len + ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_auth);

/**
 * xdr_stream_encode_opaque_auth - Encode struct opaque_auth (RFC5531 S8.2)
 * @xdr: pointer to xdr_stream
 * @flavor: verifier flavor to encode
 * @body: content of body to encode
 * @body_len: length of body to encode
 *
 * Return values:
 *   On success, returns length in bytes of XDR buffer consumed
 *   %-EBADMSG on XDR buffer overflow
 *   %-EMSGSIZE if the size of @body exceeds 400 octets
 */
ssize_t xdr_stream_encode_opaque_auth(struct xdr_stream *xdr, u32 flavor,
                                      void *body, unsigned int body_len)
{
        ssize_t ret, len;

        if (unlikely(body_len > RPC_MAX_AUTH_SIZE))
                return -EMSGSIZE;
        len = xdr_stream_encode_u32(xdr, flavor);
        if (unlikely(len < 0))
                return len;
        ret = xdr_stream_encode_opaque(xdr, body, body_len);
        if (unlikely(ret < 0))
                return ret;
        return len + ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_encode_opaque_auth);



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * PHY device list allow maintaining a list of PHY devices that are
 * part of a netdevice's link topology. PHYs can for example be chained,
 * as is the case when using a PHY that exposes an SFP module, on which an
 * SFP transceiver that embeds a PHY is connected.
 *
 * This list can then be used by userspace to leverage individual PHY
 * capabilities.
 */
#ifndef __PHY_LINK_TOPOLOGY_H
#define __PHY_LINK_TOPOLOGY_H

#include <linux/ethtool.h>
#include <linux/netdevice.h>

struct xarray;
struct phy_device;
struct sfp_bus;

struct phy_link_topology {
        struct xarray phys;
        u32 next_phy_index;
};

struct phy_device_node {
        enum phy_upstream upstream_type;

        union {
                struct net_device        *netdev;
                struct phy_device        *phydev;
        } upstream;

        struct sfp_bus *parent_sfp_bus;

        struct phy_device *phy;
};

#if IS_ENABLED(CONFIG_PHYLIB)
int phy_link_topo_add_phy(struct net_device *dev,
                          struct phy_device *phy,
                          enum phy_upstream upt, void *upstream);

void phy_link_topo_del_phy(struct net_device *dev, struct phy_device *phy);

static inline struct phy_device *
phy_link_topo_get_phy(struct net_device *dev, u32 phyindex)
{
        struct phy_link_topology *topo = dev->link_topo;
        struct phy_device_node *pdn;

        if (!topo)
                return NULL;

        pdn = xa_load(&topo->phys, phyindex);
        if (pdn)
                return pdn->phy;

        return NULL;
}

#else
static inline int phy_link_topo_add_phy(struct net_device *dev,
                                        struct phy_device *phy,
                                        enum phy_upstream upt, void *upstream)
{
        return 0;
}

static inline void phy_link_topo_del_phy(struct net_device *dev,
                                         struct phy_device *phy)
{
}

static inline struct phy_device *
phy_link_topo_get_phy(struct net_device *dev, u32 phyindex)
{
        return NULL;
}
#endif

#endif /* __PHY_LINK_TOPOLOGY_H */















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  Packet RX/TX history data structures and routines for TFRC-based protocols.
 *
 *  Copyright (c) 2007   The University of Aberdeen, Scotland, UK
 *  Copyright (c) 2005-6 The University of Waikato, Hamilton, New Zealand.
 *
 *  This code has been developed by the University of Waikato WAND
 *  research group. For further information please see https://www.wand.net.nz/
 *  or e-mail Ian McDonald - ian.mcdonald@jandi.co.nz
 *
 *  This code also uses code from Lulea University, rereleased as GPL by its
 *  authors:
 *  Copyright (c) 2003 Nils-Erik Mattsson, Joacim Haggmark, Magnus Erixzon
 *
 *  Changes to meet Linux coding standards, to make it meet latest ccid3 draft
 *  and to make it work as a loadable module in the DCCP stack written by
 *  Arnaldo Carvalho de Melo <acme@conectiva.com.br>.
 *
 *  Copyright (c) 2005 Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 */

#ifndef _DCCP_PKT_HIST_
#define _DCCP_PKT_HIST_

#include <linux/list.h>
#include <linux/slab.h>
#include "tfrc.h"

/**
 *  tfrc_tx_hist_entry  -  Simple singly-linked TX history list
 *  @next:  next oldest entry (LIFO order)
 *  @seqno: sequence number of this entry
 *  @stamp: send time of packet with sequence number @seqno
 */
struct tfrc_tx_hist_entry {
        struct tfrc_tx_hist_entry *next;
        u64                          seqno;
        ktime_t                          stamp;
};

static inline struct tfrc_tx_hist_entry *
        tfrc_tx_hist_find_entry(struct tfrc_tx_hist_entry *head, u64 seqno)
{
        while (head != NULL && head->seqno != seqno)
                head = head->next;
        return head;
}

int tfrc_tx_hist_add(struct tfrc_tx_hist_entry **headp, u64 seqno);
void tfrc_tx_hist_purge(struct tfrc_tx_hist_entry **headp);

/* Subtraction a-b modulo-16, respects circular wrap-around */
#define SUB16(a, b) (((a) + 16 - (b)) & 0xF)

/* Number of packets to wait after a missing packet (RFC 4342, 6.1) */
#define TFRC_NDUPACK 3

/**
 * tfrc_rx_hist_entry - Store information about a single received packet
 * @tfrchrx_seqno:        DCCP packet sequence number
 * @tfrchrx_ccval:        window counter value of packet (RFC 4342, 8.1)
 * @tfrchrx_ndp:        the NDP count (if any) of the packet
 * @tfrchrx_tstamp:        actual receive time of packet
 */
struct tfrc_rx_hist_entry {
        u64                 tfrchrx_seqno:48,
                         tfrchrx_ccval:4,
                         tfrchrx_type:4;
        u64                 tfrchrx_ndp:48;
        ktime_t                 tfrchrx_tstamp;
};

/**
 * tfrc_rx_hist  -  RX history structure for TFRC-based protocols
 * @ring:                Packet history for RTT sampling and loss detection
 * @loss_count:                Number of entries in circular history
 * @loss_start:                Movable index (for loss detection)
 * @rtt_sample_prev:        Used during RTT sampling, points to candidate entry
 */
struct tfrc_rx_hist {
        struct tfrc_rx_hist_entry *ring[TFRC_NDUPACK + 1];
        u8                          loss_count:2,
                                  loss_start:2;
#define rtt_sample_prev                  loss_start
};

/**
 * tfrc_rx_hist_index - index to reach n-th entry after loss_start
 */
static inline u8 tfrc_rx_hist_index(const struct tfrc_rx_hist *h, const u8 n)
{
        return (h->loss_start + n) & TFRC_NDUPACK;
}

/**
 * tfrc_rx_hist_last_rcv - entry with highest-received-seqno so far
 */
static inline struct tfrc_rx_hist_entry *
                        tfrc_rx_hist_last_rcv(const struct tfrc_rx_hist *h)
{
        return h->ring[tfrc_rx_hist_index(h, h->loss_count)];
}

/**
 * tfrc_rx_hist_entry - return the n-th history entry after loss_start
 */
static inline struct tfrc_rx_hist_entry *
                        tfrc_rx_hist_entry(const struct tfrc_rx_hist *h, const u8 n)
{
        return h->ring[tfrc_rx_hist_index(h, n)];
}

/**
 * tfrc_rx_hist_loss_prev - entry with highest-received-seqno before loss was detected
 */
static inline struct tfrc_rx_hist_entry *
                        tfrc_rx_hist_loss_prev(const struct tfrc_rx_hist *h)
{
        return h->ring[h->loss_start];
}

/* indicate whether previously a packet was detected missing */
static inline bool tfrc_rx_hist_loss_pending(const struct tfrc_rx_hist *h)
{
        return h->loss_count > 0;
}

void tfrc_rx_hist_add_packet(struct tfrc_rx_hist *h, const struct sk_buff *skb,
                             const u64 ndp);

int tfrc_rx_hist_duplicate(struct tfrc_rx_hist *h, struct sk_buff *skb);

struct tfrc_loss_hist;
int tfrc_rx_handle_loss(struct tfrc_rx_hist *h, struct tfrc_loss_hist *lh,
                        struct sk_buff *skb, const u64 ndp,
                        u32 (*first_li)(struct sock *sk), struct sock *sk);
u32 tfrc_rx_hist_sample_rtt(struct tfrc_rx_hist *h, const struct sk_buff *skb);
int tfrc_rx_hist_alloc(struct tfrc_rx_hist *h);
void tfrc_rx_hist_purge(struct tfrc_rx_hist *h);

#endif /* _DCCP_PKT_HIST_ */



















































    2 








    5 






    2 






    1 


    2 












































































































































































































































    6 






















    6 

























    2 















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * authencesn.c - AEAD wrapper for IPsec with extended sequence numbers,
 *                 derived from authenc.c
 *
 * Copyright (C) 2010 secunet Security Networks AG
 * Copyright (C) 2010 Steffen Klassert <steffen.klassert@secunet.com>
 * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/authenc.h>
#include <crypto/null.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/spinlock.h>

struct authenc_esn_instance_ctx {
        struct crypto_ahash_spawn auth;
        struct crypto_skcipher_spawn enc;
};

struct crypto_authenc_esn_ctx {
        unsigned int reqoff;
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
        struct crypto_sync_skcipher *null;
};

struct authenc_esn_request_ctx {
        struct scatterlist src[2];
        struct scatterlist dst[2];
        char tail[];
};

static void authenc_esn_request_complete(struct aead_request *req, int err)
{
        if (err != -EINPROGRESS)
                aead_request_complete(req, err);
}

static int crypto_authenc_esn_setauthsize(struct crypto_aead *authenc_esn,
                                          unsigned int authsize)
{
        if (authsize > 0 && authsize < 4)
                return -EINVAL;

        return 0;
}

static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *key,
                                     unsigned int keylen)
{
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        struct crypto_ahash *auth = ctx->auth;
        struct crypto_skcipher *enc = ctx->enc;
        struct crypto_authenc_keys keys;
        int err = -EINVAL;

        if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
                goto out;

        crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
        crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc_esn) &
                                     CRYPTO_TFM_REQ_MASK);
        err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
        if (err)
                goto out;

        crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
        crypto_skcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
                                         CRYPTO_TFM_REQ_MASK);
        err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen);
out:
        memzero_explicit(&keys, sizeof(keys));
        return err;
}

static int crypto_authenc_esn_genicv_tail(struct aead_request *req,
                                          unsigned int flags)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
        u8 *hash = areq_ctx->tail;
        unsigned int authsize = crypto_aead_authsize(authenc_esn);
        unsigned int assoclen = req->assoclen;
        unsigned int cryptlen = req->cryptlen;
        struct scatterlist *dst = req->dst;
        u32 tmp[2];

        /* Move high-order bits of sequence number back. */
        scatterwalk_map_and_copy(tmp, dst, 4, 4, 0);
        scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0);
        scatterwalk_map_and_copy(tmp, dst, 0, 8, 1);

        scatterwalk_map_and_copy(hash, dst, assoclen + cryptlen, authsize, 1);
        return 0;
}

static void authenc_esn_geniv_ahash_done(void *data, int err)
{
        struct aead_request *req = data;

        err = err ?: crypto_authenc_esn_genicv_tail(req, 0);
        aead_request_complete(req, err);
}

static int crypto_authenc_esn_genicv(struct aead_request *req,
                                     unsigned int flags)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        struct crypto_ahash *auth = ctx->auth;
        u8 *hash = areq_ctx->tail;
        struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
        unsigned int authsize = crypto_aead_authsize(authenc_esn);
        unsigned int assoclen = req->assoclen;
        unsigned int cryptlen = req->cryptlen;
        struct scatterlist *dst = req->dst;
        u32 tmp[2];

        if (!authsize)
                return 0;

        /* Move high-order bits of sequence number to the end. */
        scatterwalk_map_and_copy(tmp, dst, 0, 8, 0);
        scatterwalk_map_and_copy(tmp, dst, 4, 4, 1);
        scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1);

        sg_init_table(areq_ctx->dst, 2);
        dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4);

        ahash_request_set_tfm(ahreq, auth);
        ahash_request_set_crypt(ahreq, dst, hash, assoclen + cryptlen);
        ahash_request_set_callback(ahreq, flags,
                                   authenc_esn_geniv_ahash_done, req);

        return crypto_ahash_digest(ahreq) ?:
               crypto_authenc_esn_genicv_tail(req, aead_request_flags(req));
}


static void crypto_authenc_esn_encrypt_done(void *data, int err)
{
        struct aead_request *areq = data;

        if (!err)
                err = crypto_authenc_esn_genicv(areq, 0);

        authenc_esn_request_complete(areq, err);
}

static int crypto_authenc_esn_copy(struct aead_request *req, unsigned int len)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);

        skcipher_request_set_sync_tfm(skreq, ctx->null);
        skcipher_request_set_callback(skreq, aead_request_flags(req),
                                      NULL, NULL);
        skcipher_request_set_crypt(skreq, req->src, req->dst, len, NULL);

        return crypto_skcipher_encrypt(skreq);
}

static int crypto_authenc_esn_encrypt(struct aead_request *req)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        struct skcipher_request *skreq = (void *)(areq_ctx->tail +
                                                  ctx->reqoff);
        struct crypto_skcipher *enc = ctx->enc;
        unsigned int assoclen = req->assoclen;
        unsigned int cryptlen = req->cryptlen;
        struct scatterlist *src, *dst;
        int err;

        sg_init_table(areq_ctx->src, 2);
        src = scatterwalk_ffwd(areq_ctx->src, req->src, assoclen);
        dst = src;

        if (req->src != req->dst) {
                err = crypto_authenc_esn_copy(req, assoclen);
                if (err)
                        return err;

                sg_init_table(areq_ctx->dst, 2);
                dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, assoclen);
        }

        skcipher_request_set_tfm(skreq, enc);
        skcipher_request_set_callback(skreq, aead_request_flags(req),
                                      crypto_authenc_esn_encrypt_done, req);
        skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv);

        err = crypto_skcipher_encrypt(skreq);
        if (err)
                return err;

        return crypto_authenc_esn_genicv(req, aead_request_flags(req));
}

static int crypto_authenc_esn_decrypt_tail(struct aead_request *req,
                                           unsigned int flags)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        unsigned int authsize = crypto_aead_authsize(authenc_esn);
        struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        struct skcipher_request *skreq = (void *)(areq_ctx->tail +
                                                  ctx->reqoff);
        struct crypto_ahash *auth = ctx->auth;
        u8 *ohash = areq_ctx->tail;
        unsigned int cryptlen = req->cryptlen - authsize;
        unsigned int assoclen = req->assoclen;
        struct scatterlist *dst = req->dst;
        u8 *ihash = ohash + crypto_ahash_digestsize(auth);
        u32 tmp[2];

        if (!authsize)
                goto decrypt;

        /* Move high-order bits of sequence number back. */
        scatterwalk_map_and_copy(tmp, dst, 4, 4, 0);
        scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0);
        scatterwalk_map_and_copy(tmp, dst, 0, 8, 1);

        if (crypto_memneq(ihash, ohash, authsize))
                return -EBADMSG;

decrypt:

        sg_init_table(areq_ctx->dst, 2);
        dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen);

        skcipher_request_set_tfm(skreq, ctx->enc);
        skcipher_request_set_callback(skreq, flags,
                                      req->base.complete, req->base.data);
        skcipher_request_set_crypt(skreq, dst, dst, cryptlen, req->iv);

        return crypto_skcipher_decrypt(skreq);
}

static void authenc_esn_verify_ahash_done(void *data, int err)
{
        struct aead_request *req = data;

        err = err ?: crypto_authenc_esn_decrypt_tail(req, 0);
        authenc_esn_request_complete(req, err);
}

static int crypto_authenc_esn_decrypt(struct aead_request *req)
{
        struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
        struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
        struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
        unsigned int authsize = crypto_aead_authsize(authenc_esn);
        struct crypto_ahash *auth = ctx->auth;
        u8 *ohash = areq_ctx->tail;
        unsigned int assoclen = req->assoclen;
        unsigned int cryptlen = req->cryptlen;
        u8 *ihash = ohash + crypto_ahash_digestsize(auth);
        struct scatterlist *dst = req->dst;
        u32 tmp[2];
        int err;

        cryptlen -= authsize;

        if (req->src != dst) {
                err = crypto_authenc_esn_copy(req, assoclen + cryptlen);
                if (err)
                        return err;
        }

        scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen,
                                 authsize, 0);

        if (!authsize)
                goto tail;

        /* Move high-order bits of sequence number to the end. */
        scatterwalk_map_and_copy(tmp, dst, 0, 8, 0);
        scatterwalk_map_and_copy(tmp, dst, 4, 4, 1);
        scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1);

        sg_init_table(areq_ctx->dst, 2);
        dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4);

        ahash_request_set_tfm(ahreq, auth);
        ahash_request_set_crypt(ahreq, dst, ohash, assoclen + cryptlen);
        ahash_request_set_callback(ahreq, aead_request_flags(req),
                                   authenc_esn_verify_ahash_done, req);

        err = crypto_ahash_digest(ahreq);
        if (err)
                return err;

tail:
        return crypto_authenc_esn_decrypt_tail(req, aead_request_flags(req));
}

static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
{
        struct aead_instance *inst = aead_alg_instance(tfm);
        struct authenc_esn_instance_ctx *ictx = aead_instance_ctx(inst);
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);
        struct crypto_ahash *auth;
        struct crypto_skcipher *enc;
        struct crypto_sync_skcipher *null;
        int err;

        auth = crypto_spawn_ahash(&ictx->auth);
        if (IS_ERR(auth))
                return PTR_ERR(auth);

        enc = crypto_spawn_skcipher(&ictx->enc);
        err = PTR_ERR(enc);
        if (IS_ERR(enc))
                goto err_free_ahash;

        null = crypto_get_default_null_skcipher();
        err = PTR_ERR(null);
        if (IS_ERR(null))
                goto err_free_skcipher;

        ctx->auth = auth;
        ctx->enc = enc;
        ctx->null = null;

        ctx->reqoff = 2 * crypto_ahash_digestsize(auth);

        crypto_aead_set_reqsize(
                tfm,
                sizeof(struct authenc_esn_request_ctx) +
                ctx->reqoff +
                max_t(unsigned int,
                      crypto_ahash_reqsize(auth) +
                      sizeof(struct ahash_request),
                      sizeof(struct skcipher_request) +
                      crypto_skcipher_reqsize(enc)));

        return 0;

err_free_skcipher:
        crypto_free_skcipher(enc);
err_free_ahash:
        crypto_free_ahash(auth);
        return err;
}

static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm)
{
        struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(tfm);

        crypto_free_ahash(ctx->auth);
        crypto_free_skcipher(ctx->enc);
        crypto_put_default_null_skcipher();
}

static void crypto_authenc_esn_free(struct aead_instance *inst)
{
        struct authenc_esn_instance_ctx *ctx = aead_instance_ctx(inst);

        crypto_drop_skcipher(&ctx->enc);
        crypto_drop_ahash(&ctx->auth);
        kfree(inst);
}

static int crypto_authenc_esn_create(struct crypto_template *tmpl,
                                     struct rtattr **tb)
{
        u32 mask;
        struct aead_instance *inst;
        struct authenc_esn_instance_ctx *ctx;
        struct skcipher_alg_common *enc;
        struct hash_alg_common *auth;
        struct crypto_alg *auth_base;
        int err;

        err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask);
        if (err)
                return err;

        inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
        if (!inst)
                return -ENOMEM;
        ctx = aead_instance_ctx(inst);

        err = crypto_grab_ahash(&ctx->auth, aead_crypto_instance(inst),
                                crypto_attr_alg_name(tb[1]), 0, mask);
        if (err)
                goto err_free_inst;
        auth = crypto_spawn_ahash_alg(&ctx->auth);
        auth_base = &auth->base;

        err = crypto_grab_skcipher(&ctx->enc, aead_crypto_instance(inst),
                                   crypto_attr_alg_name(tb[2]), 0, mask);
        if (err)
                goto err_free_inst;
        enc = crypto_spawn_skcipher_alg_common(&ctx->enc);

        err = -ENAMETOOLONG;
        if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
                     "authencesn(%s,%s)", auth_base->cra_name,
                     enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
                goto err_free_inst;

        if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
                     "authencesn(%s,%s)", auth_base->cra_driver_name,
                     enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
                goto err_free_inst;

        inst->alg.base.cra_priority = enc->base.cra_priority * 10 +
                                      auth_base->cra_priority;
        inst->alg.base.cra_blocksize = enc->base.cra_blocksize;
        inst->alg.base.cra_alignmask = enc->base.cra_alignmask;
        inst->alg.base.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx);

        inst->alg.ivsize = enc->ivsize;
        inst->alg.chunksize = enc->chunksize;
        inst->alg.maxauthsize = auth->digestsize;

        inst->alg.init = crypto_authenc_esn_init_tfm;
        inst->alg.exit = crypto_authenc_esn_exit_tfm;

        inst->alg.setkey = crypto_authenc_esn_setkey;
        inst->alg.setauthsize = crypto_authenc_esn_setauthsize;
        inst->alg.encrypt = crypto_authenc_esn_encrypt;
        inst->alg.decrypt = crypto_authenc_esn_decrypt;

        inst->free = crypto_authenc_esn_free;

        err = aead_register_instance(tmpl, inst);
        if (err) {
err_free_inst:
                crypto_authenc_esn_free(inst);
        }
        return err;
}

static struct crypto_template crypto_authenc_esn_tmpl = {
        .name = "authencesn",
        .create = crypto_authenc_esn_create,
        .module = THIS_MODULE,
};

static int __init crypto_authenc_esn_module_init(void)
{
        return crypto_register_template(&crypto_authenc_esn_tmpl);
}

static void __exit crypto_authenc_esn_module_exit(void)
{
        crypto_unregister_template(&crypto_authenc_esn_tmpl);
}

subsys_initcall(crypto_authenc_esn_module_init);
module_exit(crypto_authenc_esn_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
MODULE_DESCRIPTION("AEAD wrapper for IPsec with extended sequence numbers");
MODULE_ALIAS_CRYPTO("authencesn");




































































































   10 




    2 
    1 
    1 
    1 




    1 













    4 
















    3 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/unaligned.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_tables.h>

struct nft_byteorder {
        u8                        sreg;
        u8                        dreg;
        enum nft_byteorder_ops        op:8;
        u8                        len;
        u8                        size;
};

void nft_byteorder_eval(const struct nft_expr *expr,
                        struct nft_regs *regs,
                        const struct nft_pktinfo *pkt)
{
        const struct nft_byteorder *priv = nft_expr_priv(expr);
        u32 *src = &regs->data[priv->sreg];
        u32 *dst = &regs->data[priv->dreg];
        u16 *s16, *d16;
        unsigned int i;

        s16 = (void *)src;
        d16 = (void *)dst;

        switch (priv->size) {
        case 8: {
                u64 *dst64 = (void *)dst;
                u64 src64;

                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 8; i++) {
                                src64 = nft_reg_load64(&src[i]);
                                nft_reg_store64(&dst64[i],
                                                be64_to_cpu((__force __be64)src64));
                        }
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 8; i++) {
                                src64 = (__force __u64)
                                        cpu_to_be64(nft_reg_load64(&src[i]));
                                nft_reg_store64(&dst64[i], src64);
                        }
                        break;
                }
                break;
        }
        case 4:
                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 4; i++)
                                dst[i] = ntohl((__force __be32)src[i]);
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 4; i++)
                                dst[i] = (__force __u32)htonl(src[i]);
                        break;
                }
                break;
        case 2:
                switch (priv->op) {
                case NFT_BYTEORDER_NTOH:
                        for (i = 0; i < priv->len / 2; i++)
                                d16[i] = ntohs((__force __be16)s16[i]);
                        break;
                case NFT_BYTEORDER_HTON:
                        for (i = 0; i < priv->len / 2; i++)
                                d16[i] = (__force __u16)htons(s16[i]);
                        break;
                }
                break;
        }
}

static const struct nla_policy nft_byteorder_policy[NFTA_BYTEORDER_MAX + 1] = {
        [NFTA_BYTEORDER_SREG]        = { .type = NLA_U32 },
        [NFTA_BYTEORDER_DREG]        = { .type = NLA_U32 },
        [NFTA_BYTEORDER_OP]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_BYTEORDER_LEN]        = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_BYTEORDER_SIZE]        = NLA_POLICY_MAX(NLA_BE32, 255),
};

static int nft_byteorder_init(const struct nft_ctx *ctx,
                              const struct nft_expr *expr,
                              const struct nlattr * const tb[])
{
        struct nft_byteorder *priv = nft_expr_priv(expr);
        u32 size, len;
        int err;

        if (tb[NFTA_BYTEORDER_SREG] == NULL ||
            tb[NFTA_BYTEORDER_DREG] == NULL ||
            tb[NFTA_BYTEORDER_LEN] == NULL ||
            tb[NFTA_BYTEORDER_SIZE] == NULL ||
            tb[NFTA_BYTEORDER_OP] == NULL)
                return -EINVAL;

        priv->op = ntohl(nla_get_be32(tb[NFTA_BYTEORDER_OP]));
        switch (priv->op) {
        case NFT_BYTEORDER_NTOH:
        case NFT_BYTEORDER_HTON:
                break;
        default:
                return -EINVAL;
        }

        err = nft_parse_u32_check(tb[NFTA_BYTEORDER_SIZE], U8_MAX, &size);
        if (err < 0)
                return err;

        priv->size = size;

        switch (priv->size) {
        case 2:
        case 4:
        case 8:
                break;
        default:
                return -EINVAL;
        }

        err = nft_parse_u32_check(tb[NFTA_BYTEORDER_LEN], U8_MAX, &len);
        if (err < 0)
                return err;

        priv->len = len;

        err = nft_parse_register_load(ctx, tb[NFTA_BYTEORDER_SREG], &priv->sreg,
                                      priv->len);
        if (err < 0)
                return err;

        return nft_parse_register_store(ctx, tb[NFTA_BYTEORDER_DREG],
                                        &priv->dreg, NULL, NFT_DATA_VALUE,
                                        priv->len);
}

static int nft_byteorder_dump(struct sk_buff *skb,
                              const struct nft_expr *expr, bool reset)
{
        const struct nft_byteorder *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_BYTEORDER_SREG, priv->sreg))
                goto nla_put_failure;
        if (nft_dump_register(skb, NFTA_BYTEORDER_DREG, priv->dreg))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_OP, htonl(priv->op)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_LEN, htonl(priv->len)))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_BYTEORDER_SIZE, htonl(priv->size)))
                goto nla_put_failure;
        return 0;

nla_put_failure:
        return -1;
}

static bool nft_byteorder_reduce(struct nft_regs_track *track,
                                 const struct nft_expr *expr)
{
        struct nft_byteorder *priv = nft_expr_priv(expr);

        nft_reg_track_cancel(track, priv->dreg, priv->len);

        return false;
}

static const struct nft_expr_ops nft_byteorder_ops = {
        .type                = &nft_byteorder_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_byteorder)),
        .eval                = nft_byteorder_eval,
        .init                = nft_byteorder_init,
        .dump                = nft_byteorder_dump,
        .reduce                = nft_byteorder_reduce,
};

struct nft_expr_type nft_byteorder_type __read_mostly = {
        .name                = "byteorder",
        .ops                = &nft_byteorder_ops,
        .policy                = nft_byteorder_policy,
        .maxattr        = NFTA_BYTEORDER_MAX,
        .owner                = THIS_MODULE,
};

















  217 

















   81 


















  680 








  102 






































  109 




  109 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// SPDX-License-Identifier: GPL-2.0

#ifndef __KVM_X86_MMU_TDP_MMU_H
#define __KVM_X86_MMU_TDP_MMU_H

#include <linux/kvm_host.h>

#include "spte.h"

void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);

void kvm_tdp_mmu_alloc_root(struct kvm_vcpu *vcpu, bool private);

__must_check static inline bool kvm_tdp_mmu_get_root(struct kvm_mmu_page *root)
{
        return refcount_inc_not_zero(&root->tdp_mmu_root_count);
}

void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root);

enum kvm_tdp_mmu_root_types {
        KVM_INVALID_ROOTS = BIT(0),
        KVM_DIRECT_ROOTS = BIT(1),
        KVM_MIRROR_ROOTS = BIT(2),
        KVM_VALID_ROOTS = KVM_DIRECT_ROOTS | KVM_MIRROR_ROOTS,
        KVM_ALL_ROOTS = KVM_VALID_ROOTS | KVM_INVALID_ROOTS,
};

static inline enum kvm_tdp_mmu_root_types kvm_gfn_range_filter_to_root_types(struct kvm *kvm,
                                                             enum kvm_gfn_range_filter process)
{
        enum kvm_tdp_mmu_root_types ret = 0;

        if (!kvm_has_mirrored_tdp(kvm))
                return KVM_DIRECT_ROOTS;

        if (process & KVM_FILTER_PRIVATE)
                ret |= KVM_MIRROR_ROOTS;
        if (process & KVM_FILTER_SHARED)
                ret |= KVM_DIRECT_ROOTS;

        WARN_ON_ONCE(!ret);

        return ret;
}

static inline struct kvm_mmu_page *tdp_mmu_get_root_for_fault(struct kvm_vcpu *vcpu,
                                                              struct kvm_page_fault *fault)
{
        if (unlikely(!kvm_is_addr_direct(vcpu->kvm, fault->addr)))
                return root_to_sp(vcpu->arch.mmu->mirror_root_hpa);

        return root_to_sp(vcpu->arch.mmu->root.hpa);
}

static inline struct kvm_mmu_page *tdp_mmu_get_root(struct kvm_vcpu *vcpu,
                                                    enum kvm_tdp_mmu_root_types type)
{
        if (unlikely(type == KVM_MIRROR_ROOTS))
                return root_to_sp(vcpu->arch.mmu->mirror_root_hpa);

        return root_to_sp(vcpu->arch.mmu->root.hpa);
}

bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush);
bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp);
void kvm_tdp_mmu_zap_all(struct kvm *kvm);
void kvm_tdp_mmu_invalidate_roots(struct kvm *kvm,
                                  enum kvm_tdp_mmu_root_types root_types);
void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm, bool shared);

int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);

bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
                                 bool flush);
bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range);

bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
                             const struct kvm_memory_slot *slot, int min_level);
void kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
                                  const struct kvm_memory_slot *slot);
void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
                                       struct kvm_memory_slot *slot,
                                       gfn_t gfn, unsigned long mask,
                                       bool wrprot);
void kvm_tdp_mmu_recover_huge_pages(struct kvm *kvm,
                                    const struct kvm_memory_slot *slot);

bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
                                   struct kvm_memory_slot *slot, gfn_t gfn,
                                   int min_level);

void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm,
                                      const struct kvm_memory_slot *slot,
                                      gfn_t start, gfn_t end,
                                      int target_level, bool shared);

static inline void kvm_tdp_mmu_walk_lockless_begin(void)
{
        rcu_read_lock();
}

static inline void kvm_tdp_mmu_walk_lockless_end(void)
{
        rcu_read_unlock();
}

int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
                         int *root_level);
u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gfn_t gfn,
                                        u64 *spte);

#ifdef CONFIG_X86_64
static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; }
#else
static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; }
#endif

#endif /* __KVM_X86_MMU_TDP_MMU_H */

































































































    1 
   23 

























































































    1 



    1 









    2 









   23 







    1 





















    1 


   21 












    3 
















































   62 







   23 




    2 






    3 






















   13 


    3 










   29 











    9 
























   41 


    7 


   34 


   28 

   20 

    6 























    1 

















    1 






























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/ematch.c                Extended Match API
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 *
 * ==========================================================================
 *
 * An extended match (ematch) is a small classification tool not worth
 * writing a full classifier for. Ematches can be interconnected to form
 * a logic expression and get attached to classifiers to extend their
 * functionatlity.
 *
 * The userspace part transforms the logic expressions into an array
 * consisting of multiple sequences of interconnected ematches separated
 * by markers. Precedence is implemented by a special ematch kind
 * referencing a sequence beyond the marker of the current sequence
 * causing the current position in the sequence to be pushed onto a stack
 * to allow the current position to be overwritten by the position referenced
 * in the special ematch. Matching continues in the new sequence until a
 * marker is reached causing the position to be restored from the stack.
 *
 * Example:
 *          A AND (B1 OR B2) AND C AND D
 *
 *              ------->-PUSH-------
 *    -->--    /         -->--      \   -->--
 *   /     \  /         /     \      \ /     \
 * +-------+-------+-------+-------+-------+--------+
 * | A AND | B AND | C AND | D END | B1 OR | B2 END |
 * +-------+-------+-------+-------+-------+--------+
 *                    \                      /
 *                     --------<-POP---------
 *
 * where B is a virtual ematch referencing to sequence starting with B1.
 *
 * ==========================================================================
 *
 * How to write an ematch in 60 seconds
 * ------------------------------------
 *
 *   1) Provide a matcher function:
 *      static int my_match(struct sk_buff *skb, struct tcf_ematch *m,
 *                          struct tcf_pkt_info *info)
 *      {
 *              struct mydata *d = (struct mydata *) m->data;
 *
 *              if (...matching goes here...)
 *                      return 1;
 *              else
 *                      return 0;
 *      }
 *
 *   2) Fill out a struct tcf_ematch_ops:
 *      static struct tcf_ematch_ops my_ops = {
 *              .kind = unique id,
 *              .datalen = sizeof(struct mydata),
 *              .match = my_match,
 *              .owner = THIS_MODULE,
 *      };
 *
 *   3) Register/Unregister your ematch:
 *      static int __init init_my_ematch(void)
 *      {
 *              return tcf_em_register(&my_ops);
 *      }
 *
 *      static void __exit exit_my_ematch(void)
 *      {
 *              tcf_em_unregister(&my_ops);
 *      }
 *
 *      module_init(init_my_ematch);
 *      module_exit(exit_my_ematch);
 *
 *   4) By now you should have two more seconds left, barely enough to
 *      open up a beer to watch the compilation going.
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>
#include <net/pkt_cls.h>

static LIST_HEAD(ematch_ops);
static DEFINE_RWLOCK(ematch_mod_lock);

static struct tcf_ematch_ops *tcf_em_lookup(u16 kind)
{
        struct tcf_ematch_ops *e = NULL;

        read_lock(&ematch_mod_lock);
        list_for_each_entry(e, &ematch_ops, link) {
                if (kind == e->kind) {
                        if (!try_module_get(e->owner))
                                e = NULL;
                        read_unlock(&ematch_mod_lock);
                        return e;
                }
        }
        read_unlock(&ematch_mod_lock);

        return NULL;
}

/**
 * tcf_em_register - register an extended match
 *
 * @ops: ematch operations lookup table
 *
 * This function must be called by ematches to announce their presence.
 * The given @ops must have kind set to a unique identifier and the
 * callback match() must be implemented. All other callbacks are optional
 * and a fallback implementation is used instead.
 *
 * Returns -EEXISTS if an ematch of the same kind has already registered.
 */
int tcf_em_register(struct tcf_ematch_ops *ops)
{
        int err = -EEXIST;
        struct tcf_ematch_ops *e;

        if (ops->match == NULL)
                return -EINVAL;

        write_lock(&ematch_mod_lock);
        list_for_each_entry(e, &ematch_ops, link)
                if (ops->kind == e->kind)
                        goto errout;

        list_add_tail(&ops->link, &ematch_ops);
        err = 0;
errout:
        write_unlock(&ematch_mod_lock);
        return err;
}
EXPORT_SYMBOL(tcf_em_register);

/**
 * tcf_em_unregister - unregister and extended match
 *
 * @ops: ematch operations lookup table
 *
 * This function must be called by ematches to announce their disappearance
 * for examples when the module gets unloaded. The @ops parameter must be
 * the same as the one used for registration.
 *
 * Returns -ENOENT if no matching ematch was found.
 */
void tcf_em_unregister(struct tcf_ematch_ops *ops)
{
        write_lock(&ematch_mod_lock);
        list_del(&ops->link);
        write_unlock(&ematch_mod_lock);
}
EXPORT_SYMBOL(tcf_em_unregister);

static inline struct tcf_ematch *tcf_em_get_match(struct tcf_ematch_tree *tree,
                                                  int index)
{
        return &tree->matches[index];
}


static int tcf_em_validate(struct tcf_proto *tp,
                           struct tcf_ematch_tree_hdr *tree_hdr,
                           struct tcf_ematch *em, struct nlattr *nla, int idx)
{
        int err = -EINVAL;
        struct tcf_ematch_hdr *em_hdr = nla_data(nla);
        int data_len = nla_len(nla) - sizeof(*em_hdr);
        void *data = (void *) em_hdr + sizeof(*em_hdr);
        struct net *net = tp->chain->block->net;

        if (!TCF_EM_REL_VALID(em_hdr->flags))
                goto errout;

        if (em_hdr->kind == TCF_EM_CONTAINER) {
                /* Special ematch called "container", carries an index
                 * referencing an external ematch sequence.
                 */
                u32 ref;

                if (data_len < sizeof(ref))
                        goto errout;
                ref = *(u32 *) data;

                if (ref >= tree_hdr->nmatches)
                        goto errout;

                /* We do not allow backward jumps to avoid loops and jumps
                 * to our own position are of course illegal.
                 */
                if (ref <= idx)
                        goto errout;


                em->data = ref;
        } else {
                /* Note: This lookup will increase the module refcnt
                 * of the ematch module referenced. In case of a failure,
                 * a destroy function is called by the underlying layer
                 * which automatically releases the reference again, therefore
                 * the module MUST not be given back under any circumstances
                 * here. Be aware, the destroy function assumes that the
                 * module is held if the ops field is non zero.
                 */
                em->ops = tcf_em_lookup(em_hdr->kind);

                if (em->ops == NULL) {
                        err = -ENOENT;
#ifdef CONFIG_MODULES
                        __rtnl_unlock();
                        request_module("ematch-kind-%u", em_hdr->kind);
                        rtnl_lock();
                        em->ops = tcf_em_lookup(em_hdr->kind);
                        if (em->ops) {
                                /* We dropped the RTNL mutex in order to
                                 * perform the module load. Tell the caller
                                 * to replay the request.
                                 */
                                module_put(em->ops->owner);
                                em->ops = NULL;
                                err = -EAGAIN;
                        }
#endif
                        goto errout;
                }

                /* ematch module provides expected length of data, so we
                 * can do a basic sanity check.
                 */
                if (em->ops->datalen && data_len < em->ops->datalen)
                        goto errout;

                if (em->ops->change) {
                        err = -EINVAL;
                        if (em_hdr->flags & TCF_EM_SIMPLE)
                                goto errout;
                        err = em->ops->change(net, data, data_len, em);
                        if (err < 0)
                                goto errout;
                } else if (data_len > 0) {
                        /* ematch module doesn't provide an own change
                         * procedure and expects us to allocate and copy
                         * the ematch data.
                         *
                         * TCF_EM_SIMPLE may be specified stating that the
                         * data only consists of a u32 integer and the module
                         * does not expected a memory reference but rather
                         * the value carried.
                         */
                        if (em_hdr->flags & TCF_EM_SIMPLE) {
                                if (em->ops->datalen > 0)
                                        goto errout;
                                if (data_len < sizeof(u32))
                                        goto errout;
                                em->data = *(u32 *) data;
                        } else {
                                void *v = kmemdup(data, data_len, GFP_KERNEL);
                                if (v == NULL) {
                                        err = -ENOBUFS;
                                        goto errout;
                                }
                                em->data = (unsigned long) v;
                        }
                        em->datalen = data_len;
                }
        }

        em->matchid = em_hdr->matchid;
        em->flags = em_hdr->flags;
        em->net = net;

        err = 0;
errout:
        return err;
}

static const struct nla_policy em_policy[TCA_EMATCH_TREE_MAX + 1] = {
        [TCA_EMATCH_TREE_HDR]        = { .len = sizeof(struct tcf_ematch_tree_hdr) },
        [TCA_EMATCH_TREE_LIST]        = { .type = NLA_NESTED },
};

/**
 * tcf_em_tree_validate - validate ematch config TLV and build ematch tree
 *
 * @tp: classifier kind handle
 * @nla: ematch tree configuration TLV
 * @tree: destination ematch tree variable to store the resulting
 *        ematch tree.
 *
 * This function validates the given configuration TLV @nla and builds an
 * ematch tree in @tree. The resulting tree must later be copied into
 * the private classifier data using tcf_em_tree_change(). You MUST NOT
 * provide the ematch tree variable of the private classifier data directly,
 * the changes would not be locked properly.
 *
 * Returns a negative error code if the configuration TLV contains errors.
 */
int tcf_em_tree_validate(struct tcf_proto *tp, struct nlattr *nla,
                         struct tcf_ematch_tree *tree)
{
        int idx, list_len, matches_len, err;
        struct nlattr *tb[TCA_EMATCH_TREE_MAX + 1];
        struct nlattr *rt_match, *rt_hdr, *rt_list;
        struct tcf_ematch_tree_hdr *tree_hdr;
        struct tcf_ematch *em;

        memset(tree, 0, sizeof(*tree));
        if (!nla)
                return 0;

        err = nla_parse_nested_deprecated(tb, TCA_EMATCH_TREE_MAX, nla,
                                          em_policy, NULL);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        rt_hdr = tb[TCA_EMATCH_TREE_HDR];
        rt_list = tb[TCA_EMATCH_TREE_LIST];

        if (rt_hdr == NULL || rt_list == NULL)
                goto errout;

        tree_hdr = nla_data(rt_hdr);
        memcpy(&tree->hdr, tree_hdr, sizeof(*tree_hdr));

        rt_match = nla_data(rt_list);
        list_len = nla_len(rt_list);
        matches_len = tree_hdr->nmatches * sizeof(*em);

        tree->matches = kzalloc(matches_len, GFP_KERNEL);
        if (tree->matches == NULL)
                goto errout;

        /* We do not use nla_parse_nested here because the maximum
         * number of attributes is unknown. This saves us the allocation
         * for a tb buffer which would serve no purpose at all.
         *
         * The array of rt attributes is parsed in the order as they are
         * provided, their type must be incremental from 1 to n. Even
         * if it does not serve any real purpose, a failure of sticking
         * to this policy will result in parsing failure.
         */
        for (idx = 0; nla_ok(rt_match, list_len); idx++) {
                err = -EINVAL;

                if (rt_match->nla_type != (idx + 1))
                        goto errout_abort;

                if (idx >= tree_hdr->nmatches)
                        goto errout_abort;

                if (nla_len(rt_match) < sizeof(struct tcf_ematch_hdr))
                        goto errout_abort;

                em = tcf_em_get_match(tree, idx);

                err = tcf_em_validate(tp, tree_hdr, em, rt_match, idx);
                if (err < 0)
                        goto errout_abort;

                rt_match = nla_next(rt_match, &list_len);
        }

        /* Check if the number of matches provided by userspace actually
         * complies with the array of matches. The number was used for
         * the validation of references and a mismatch could lead to
         * undefined references during the matching process.
         */
        if (idx != tree_hdr->nmatches) {
                err = -EINVAL;
                goto errout_abort;
        }

        err = 0;
errout:
        return err;

errout_abort:
        tcf_em_tree_destroy(tree);
        return err;
}
EXPORT_SYMBOL(tcf_em_tree_validate);

/**
 * tcf_em_tree_destroy - destroy an ematch tree
 *
 * @tree: ematch tree to be deleted
 *
 * This functions destroys an ematch tree previously created by
 * tcf_em_tree_validate()/tcf_em_tree_change(). You must ensure that
 * the ematch tree is not in use before calling this function.
 */
void tcf_em_tree_destroy(struct tcf_ematch_tree *tree)
{
        int i;

        if (tree->matches == NULL)
                return;

        for (i = 0; i < tree->hdr.nmatches; i++) {
                struct tcf_ematch *em = tcf_em_get_match(tree, i);

                if (em->ops) {
                        if (em->ops->destroy)
                                em->ops->destroy(em);
                        else if (!tcf_em_is_simple(em))
                                kfree((void *) em->data);
                        module_put(em->ops->owner);
                }
        }

        tree->hdr.nmatches = 0;
        kfree(tree->matches);
        tree->matches = NULL;
}
EXPORT_SYMBOL(tcf_em_tree_destroy);

/**
 * tcf_em_tree_dump - dump ematch tree into a rtnl message
 *
 * @skb: skb holding the rtnl message
 * @tree: ematch tree to be dumped
 * @tlv: TLV type to be used to encapsulate the tree
 *
 * This function dumps a ematch tree into a rtnl message. It is valid to
 * call this function while the ematch tree is in use.
 *
 * Returns -1 if the skb tailroom is insufficient.
 */
int tcf_em_tree_dump(struct sk_buff *skb, struct tcf_ematch_tree *tree, int tlv)
{
        int i;
        u8 *tail;
        struct nlattr *top_start;
        struct nlattr *list_start;

        top_start = nla_nest_start_noflag(skb, tlv);
        if (top_start == NULL)
                goto nla_put_failure;

        if (nla_put(skb, TCA_EMATCH_TREE_HDR, sizeof(tree->hdr), &tree->hdr))
                goto nla_put_failure;

        list_start = nla_nest_start_noflag(skb, TCA_EMATCH_TREE_LIST);
        if (list_start == NULL)
                goto nla_put_failure;

        tail = skb_tail_pointer(skb);
        for (i = 0; i < tree->hdr.nmatches; i++) {
                struct nlattr *match_start = (struct nlattr *)tail;
                struct tcf_ematch *em = tcf_em_get_match(tree, i);
                struct tcf_ematch_hdr em_hdr = {
                        .kind = em->ops ? em->ops->kind : TCF_EM_CONTAINER,
                        .matchid = em->matchid,
                        .flags = em->flags
                };

                if (nla_put(skb, i + 1, sizeof(em_hdr), &em_hdr))
                        goto nla_put_failure;

                if (em->ops && em->ops->dump) {
                        if (em->ops->dump(skb, em) < 0)
                                goto nla_put_failure;
                } else if (tcf_em_is_container(em) || tcf_em_is_simple(em)) {
                        u32 u = em->data;
                        nla_put_nohdr(skb, sizeof(u), &u);
                } else if (em->datalen > 0)
                        nla_put_nohdr(skb, em->datalen, (void *) em->data);

                tail = skb_tail_pointer(skb);
                match_start->nla_len = tail - (u8 *)match_start;
        }

        nla_nest_end(skb, list_start);
        nla_nest_end(skb, top_start);

        return 0;

nla_put_failure:
        return -1;
}
EXPORT_SYMBOL(tcf_em_tree_dump);

static inline int tcf_em_match(struct sk_buff *skb, struct tcf_ematch *em,
                               struct tcf_pkt_info *info)
{
        int r = em->ops->match(skb, em, info);

        return tcf_em_is_inverted(em) ? !r : r;
}

/* Do not use this function directly, use tcf_em_tree_match instead */
int __tcf_em_tree_match(struct sk_buff *skb, struct tcf_ematch_tree *tree,
                        struct tcf_pkt_info *info)
{
        int stackp = 0, match_idx = 0, res = 0;
        struct tcf_ematch *cur_match;
        int stack[CONFIG_NET_EMATCH_STACK];

proceed:
        while (match_idx < tree->hdr.nmatches) {
                cur_match = tcf_em_get_match(tree, match_idx);

                if (tcf_em_is_container(cur_match)) {
                        if (unlikely(stackp >= CONFIG_NET_EMATCH_STACK))
                                goto stack_overflow;

                        stack[stackp++] = match_idx;
                        match_idx = cur_match->data;
                        goto proceed;
                }

                res = tcf_em_match(skb, cur_match, info);

                if (tcf_em_early_end(cur_match, res))
                        break;

                match_idx++;
        }

pop_stack:
        if (stackp > 0) {
                match_idx = stack[--stackp];
                cur_match = tcf_em_get_match(tree, match_idx);

                if (tcf_em_is_inverted(cur_match))
                        res = !res;

                if (tcf_em_early_end(cur_match, res)) {
                        goto pop_stack;
                } else {
                        match_idx++;
                        goto proceed;
                }
        }

        return res;

stack_overflow:
        net_warn_ratelimited("tc ematch: local stack overflow, increase NET_EMATCH_STACK\n");
        return -1;
}
EXPORT_SYMBOL(__tcf_em_tree_match);















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   10 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
// SPDX-License-Identifier: GPL-2.0-only
/*
 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
 *
 * bitmap_create  - sets up the bitmap structure
 * bitmap_destroy - destroys the bitmap structure
 *
 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
 * - added disk storage for bitmap
 * - changes to allow various bitmap chunk sizes
 */

/*
 * Still to do:
 *
 * flush after percent set rather than just time based. (maybe both).
 */

#include <linux/blkdev.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/timer.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/file.h>
#include <linux/mount.h>
#include <linux/buffer_head.h>
#include <linux/seq_file.h>
#include <trace/events/block.h>
#include "md.h"
#include "md-bitmap.h"

#define BITMAP_MAJOR_LO 3
/* version 4 insists the bitmap is in little-endian order
 * with version 3, it is host-endian which is non-portable
 * Version 5 is currently set only for clustered devices
 */
#define BITMAP_MAJOR_HI 4
#define BITMAP_MAJOR_CLUSTERED 5
#define        BITMAP_MAJOR_HOSTENDIAN 3

/*
 * in-memory bitmap:
 *
 * Use 16 bit block counters to track pending writes to each "chunk".
 * The 2 high order bits are special-purpose, the first is a flag indicating
 * whether a resync is needed.  The second is a flag indicating whether a
 * resync is active.
 * This means that the counter is actually 14 bits:
 *
 * +--------+--------+------------------------------------------------+
 * | resync | resync |               counter                          |
 * | needed | active |                                                |
 * |  (0-1) |  (0-1) |              (0-16383)                         |
 * +--------+--------+------------------------------------------------+
 *
 * The "resync needed" bit is set when:
 *    a '1' bit is read from storage at startup.
 *    a write request fails on some drives
 *    a resync is aborted on a chunk with 'resync active' set
 * It is cleared (and resync-active set) when a resync starts across all drives
 * of the chunk.
 *
 *
 * The "resync active" bit is set when:
 *    a resync is started on all drives, and resync_needed is set.
 *       resync_needed will be cleared (as long as resync_active wasn't already set).
 * It is cleared when a resync completes.
 *
 * The counter counts pending write requests, plus the on-disk bit.
 * When the counter is '1' and the resync bits are clear, the on-disk
 * bit can be cleared as well, thus setting the counter to 0.
 * When we set a bit, or in the counter (to start a write), if the fields is
 * 0, we first set the disk bit and set the counter to 1.
 *
 * If the counter is 0, the on-disk bit is clear and the stripe is clean
 * Anything that dirties the stripe pushes the counter to 2 (at least)
 * and sets the on-disk bit (lazily).
 * If a periodic sweep find the counter at 2, it is decremented to 1.
 * If the sweep find the counter at 1, the on-disk bit is cleared and the
 * counter goes to zero.
 *
 * Also, we'll hijack the "map" pointer itself and use it as two 16 bit block
 * counters as a fallback when "page" memory cannot be allocated:
 *
 * Normal case (page memory allocated):
 *
 *     page pointer (32-bit)
 *
 *     [ ] ------+
 *               |
 *               +-------> [   ][   ]..[   ] (4096 byte page == 2048 counters)
 *                          c1   c2    c2048
 *
 * Hijacked case (page memory allocation failed):
 *
 *     hijacked page pointer (32-bit)
 *
 *     [                  ][                  ] (no page memory allocated)
 *      counter #1 (16-bit) counter #2 (16-bit)
 *
 */

#define PAGE_BITS (PAGE_SIZE << 3)
#define PAGE_BIT_SHIFT (PAGE_SHIFT + 3)

#define NEEDED(x) (((bitmap_counter_t) x) & NEEDED_MASK)
#define RESYNC(x) (((bitmap_counter_t) x) & RESYNC_MASK)
#define COUNTER(x) (((bitmap_counter_t) x) & COUNTER_MAX)

/* how many counters per page? */
#define PAGE_COUNTER_RATIO (PAGE_BITS / COUNTER_BITS)
/* same, except a shift value for more efficient bitops */
#define PAGE_COUNTER_SHIFT (PAGE_BIT_SHIFT - COUNTER_BIT_SHIFT)
/* same, except a mask value for more efficient bitops */
#define PAGE_COUNTER_MASK  (PAGE_COUNTER_RATIO - 1)

#define BITMAP_BLOCK_SHIFT 9

/*
 * bitmap structures:
 */

/* the in-memory bitmap is represented by bitmap_pages */
struct bitmap_page {
        /*
         * map points to the actual memory page
         */
        char *map;
        /*
         * in emergencies (when map cannot be alloced), hijack the map
         * pointer and use it as two counters itself
         */
        unsigned int hijacked:1;
        /*
         * If any counter in this page is '1' or '2' - and so could be
         * cleared then that page is marked as 'pending'
         */
        unsigned int pending:1;
        /*
         * count of dirty bits on the page
         */
        unsigned int  count:30;
};

/* the main bitmap structure - one per mddev */
struct bitmap {

        struct bitmap_counts {
                spinlock_t lock;
                struct bitmap_page *bp;
                /* total number of pages in the bitmap */
                unsigned long pages;
                /* number of pages not yet allocated */
                unsigned long missing_pages;
                /* chunksize = 2^chunkshift (for bitops) */
                unsigned long chunkshift;
                /* total number of data chunks for the array */
                unsigned long chunks;
        } counts;

        struct mddev *mddev; /* the md device that the bitmap is for */

        __u64        events_cleared;
        int need_sync;

        struct bitmap_storage {
                /* backing disk file */
                struct file *file;
                /* cached copy of the bitmap file superblock */
                struct page *sb_page;
                unsigned long sb_index;
                /* list of cache pages for the file */
                struct page **filemap;
                /* attributes associated filemap pages */
                unsigned long *filemap_attr;
                /* number of pages in the file */
                unsigned long file_pages;
                /* total bytes in the bitmap */
                unsigned long bytes;
        } storage;

        unsigned long flags;

        int allclean;

        atomic_t behind_writes;
        /* highest actual value at runtime */
        unsigned long behind_writes_used;

        /*
         * the bitmap daemon - periodically wakes up and sweeps the bitmap
         * file, cleaning up bits and flushing out pages to disk as necessary
         */
        unsigned long daemon_lastrun; /* jiffies of last run */
        /*
         * when we lasted called end_sync to update bitmap with resync
         * progress.
         */
        unsigned long last_end_sync;

        /* pending writes to the bitmap file */
        atomic_t pending_writes;
        wait_queue_head_t write_wait;
        wait_queue_head_t overflow_wait;
        wait_queue_head_t behind_wait;

        struct kernfs_node *sysfs_can_clear;
        /* slot offset for clustered env */
        int cluster_slot;
};

static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,
                           int chunksize, bool init);

static inline char *bmname(struct bitmap *bitmap)
{
        return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
}

static bool __bitmap_enabled(struct bitmap *bitmap)
{
        return bitmap->storage.filemap &&
               !test_bit(BITMAP_STALE, &bitmap->flags);
}

static bool bitmap_enabled(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return false;

        return __bitmap_enabled(bitmap);
}

/*
 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
 *
 * 1) check to see if this page is allocated, if it's not then try to alloc
 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
 *    page pointer directly as a counter
 *
 * if we find our page, we increment the page's refcount so that it stays
 * allocated while we're using it
 */
static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
                               unsigned long page, int create, int no_hijack)
__releases(bitmap->lock)
__acquires(bitmap->lock)
{
        unsigned char *mappage;

        WARN_ON_ONCE(page >= bitmap->pages);
        if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
                return 0;

        if (bitmap->bp[page].map) /* page is already allocated, just return */
                return 0;

        if (!create)
                return -ENOENT;

        /* this page has not been allocated yet */

        spin_unlock_irq(&bitmap->lock);
        /* It is possible that this is being called inside a
         * prepare_to_wait/finish_wait loop from raid5c:make_request().
         * In general it is not permitted to sleep in that context as it
         * can cause the loop to spin freely.
         * That doesn't apply here as we can only reach this point
         * once with any loop.
         * When this function completes, either bp[page].map or
         * bp[page].hijacked.  In either case, this function will
         * abort before getting to this point again.  So there is
         * no risk of a free-spin, and so it is safe to assert
         * that sleeping here is allowed.
         */
        sched_annotate_sleep();
        mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
        spin_lock_irq(&bitmap->lock);

        if (mappage == NULL) {
                pr_debug("md/bitmap: map page allocation failed, hijacking\n");
                /* We don't support hijack for cluster raid */
                if (no_hijack)
                        return -ENOMEM;
                /* failed - set the hijacked flag so that we can use the
                 * pointer as a counter */
                if (!bitmap->bp[page].map)
                        bitmap->bp[page].hijacked = 1;
        } else if (bitmap->bp[page].map ||
                   bitmap->bp[page].hijacked) {
                /* somebody beat us to getting the page */
                kfree(mappage);
        } else {

                /* no page was in place and we have one, so install it */

                bitmap->bp[page].map = mappage;
                bitmap->missing_pages--;
        }
        return 0;
}

/* if page is completely empty, put it back on the free list, or dealloc it */
/* if page was hijacked, unmark the flag so it might get alloced next time */
/* Note: lock should be held when calling this */
static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
{
        char *ptr;

        if (bitmap->bp[page].count) /* page is still busy */
                return;

        /* page is no longer in use, it can be released */

        if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
                bitmap->bp[page].hijacked = 0;
                bitmap->bp[page].map = NULL;
        } else {
                /* normal case, free the page */
                ptr = bitmap->bp[page].map;
                bitmap->bp[page].map = NULL;
                bitmap->missing_pages++;
                kfree(ptr);
        }
}

/*
 * bitmap file handling - read and write the bitmap file and its superblock
 */

/*
 * basic page I/O operations
 */

/* IO operations when bitmap is stored near all superblocks */

/* choose a good rdev and read the page from there */
static int read_sb_page(struct mddev *mddev, loff_t offset,
                struct page *page, unsigned long index, int size)
{

        sector_t sector = mddev->bitmap_info.offset + offset +
                index * (PAGE_SIZE / SECTOR_SIZE);
        struct md_rdev *rdev;

        rdev_for_each(rdev, mddev) {
                u32 iosize = roundup(size, bdev_logical_block_size(rdev->bdev));

                if (!test_bit(In_sync, &rdev->flags) ||
                    test_bit(Faulty, &rdev->flags) ||
                    test_bit(Bitmap_sync, &rdev->flags))
                        continue;

                if (sync_page_io(rdev, sector, iosize, page, REQ_OP_READ, true))
                        return 0;
        }
        return -EIO;
}

static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
{
        /* Iterate the disks of an mddev, using rcu to protect access to the
         * linked list, and raising the refcount of devices we return to ensure
         * they don't disappear while in use.
         * As devices are only added or removed when raid_disk is < 0 and
         * nr_pending is 0 and In_sync is clear, the entries we return will
         * still be in the same position on the list when we re-enter
         * list_for_each_entry_continue_rcu.
         *
         * Note that if entered with 'rdev == NULL' to start at the
         * beginning, we temporarily assign 'rdev' to an address which
         * isn't really an rdev, but which can be used by
         * list_for_each_entry_continue_rcu() to find the first entry.
         */
        rcu_read_lock();
        if (rdev == NULL)
                /* start at the beginning */
                rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
        else {
                /* release the previous rdev and start from there. */
                rdev_dec_pending(rdev, mddev);
        }
        list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
                if (rdev->raid_disk >= 0 &&
                    !test_bit(Faulty, &rdev->flags)) {
                        /* this is a usable devices */
                        atomic_inc(&rdev->nr_pending);
                        rcu_read_unlock();
                        return rdev;
                }
        }
        rcu_read_unlock();
        return NULL;
}

static unsigned int optimal_io_size(struct block_device *bdev,
                                    unsigned int last_page_size,
                                    unsigned int io_size)
{
        if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))
                return roundup(last_page_size, bdev_io_opt(bdev));
        return io_size;
}

static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,
                                   loff_t start, loff_t boundary)
{
        if (io_size != opt_size &&
            start + opt_size / SECTOR_SIZE <= boundary)
                return opt_size;
        if (start + io_size / SECTOR_SIZE <= boundary)
                return io_size;

        /* Overflows boundary */
        return 0;
}

static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
                           unsigned long pg_index, struct page *page)
{
        struct block_device *bdev;
        struct mddev *mddev = bitmap->mddev;
        struct bitmap_storage *store = &bitmap->storage;
        unsigned int bitmap_limit = (bitmap->storage.file_pages - pg_index) <<
                PAGE_SHIFT;
        loff_t sboff, offset = mddev->bitmap_info.offset;
        sector_t ps = pg_index * PAGE_SIZE / SECTOR_SIZE;
        unsigned int size = PAGE_SIZE;
        unsigned int opt_size = PAGE_SIZE;
        sector_t doff;

        bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
        /* we compare length (page numbers), not page offset. */
        if ((pg_index - store->sb_index) == store->file_pages - 1) {
                unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1);

                if (last_page_size == 0)
                        last_page_size = PAGE_SIZE;
                size = roundup(last_page_size, bdev_logical_block_size(bdev));
                opt_size = optimal_io_size(bdev, last_page_size, size);
        }

        sboff = rdev->sb_start + offset;
        doff = rdev->data_offset;

        /* Just make sure we aren't corrupting data or metadata */
        if (mddev->external) {
                /* Bitmap could be anywhere. */
                if (sboff + ps > doff &&
                    sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))
                        return -EINVAL;
        } else if (offset < 0) {
                /* DATA  BITMAP METADATA  */
                size = bitmap_io_size(size, opt_size, offset + ps, 0);
                if (size == 0)
                        /* bitmap runs in to metadata */
                        return -EINVAL;

                if (doff + mddev->dev_sectors > sboff)
                        /* data runs in to bitmap */
                        return -EINVAL;
        } else if (rdev->sb_start < rdev->data_offset) {
                /* METADATA BITMAP DATA */
                size = bitmap_io_size(size, opt_size, sboff + ps, doff);
                if (size == 0)
                        /* bitmap runs in to data */
                        return -EINVAL;
        }

        md_super_write(mddev, rdev, sboff + ps, (int)min(size, bitmap_limit), page);
        return 0;
}

static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,
                          struct page *page, bool wait)
{
        struct mddev *mddev = bitmap->mddev;

        do {
                struct md_rdev *rdev = NULL;

                while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
                        if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {
                                set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
                                return;
                        }
                }
        } while (wait && md_super_wait(mddev) < 0);
}

static void md_bitmap_file_kick(struct bitmap *bitmap);

#ifdef CONFIG_MD_BITMAP_FILE
static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
{
        struct buffer_head *bh = page_buffers(page);

        while (bh && bh->b_blocknr) {
                atomic_inc(&bitmap->pending_writes);
                set_buffer_locked(bh);
                set_buffer_mapped(bh);
                submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
                bh = bh->b_this_page;
        }

        if (wait)
                wait_event(bitmap->write_wait,
                           atomic_read(&bitmap->pending_writes) == 0);
}

static void end_bitmap_write(struct buffer_head *bh, int uptodate)
{
        struct bitmap *bitmap = bh->b_private;

        if (!uptodate)
                set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
        if (atomic_dec_and_test(&bitmap->pending_writes))
                wake_up(&bitmap->write_wait);
}

static void free_buffers(struct page *page)
{
        struct buffer_head *bh;

        if (!PagePrivate(page))
                return;

        bh = page_buffers(page);
        while (bh) {
                struct buffer_head *next = bh->b_this_page;
                free_buffer_head(bh);
                bh = next;
        }
        detach_page_private(page);
        put_page(page);
}

/* read a page from a file.
 * We both read the page, and attach buffers to the page to record the
 * address of each block (using bmap).  These addresses will be used
 * to write the block later, completely bypassing the filesystem.
 * This usage is similar to how swap files are handled, and allows us
 * to write to a file with no concerns of memory allocation failing.
 */
static int read_file_page(struct file *file, unsigned long index,
                struct bitmap *bitmap, unsigned long count, struct page *page)
{
        int ret = 0;
        struct inode *inode = file_inode(file);
        struct buffer_head *bh;
        sector_t block, blk_cur;
        unsigned long blocksize = i_blocksize(inode);

        pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
                 (unsigned long long)index << PAGE_SHIFT);

        bh = alloc_page_buffers(page, blocksize);
        if (!bh) {
                ret = -ENOMEM;
                goto out;
        }
        attach_page_private(page, bh);
        blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
        while (bh) {
                block = blk_cur;

                if (count == 0)
                        bh->b_blocknr = 0;
                else {
                        ret = bmap(inode, &block);
                        if (ret || !block) {
                                ret = -EINVAL;
                                bh->b_blocknr = 0;
                                goto out;
                        }

                        bh->b_blocknr = block;
                        bh->b_bdev = inode->i_sb->s_bdev;
                        if (count < blocksize)
                                count = 0;
                        else
                                count -= blocksize;

                        bh->b_end_io = end_bitmap_write;
                        bh->b_private = bitmap;
                        atomic_inc(&bitmap->pending_writes);
                        set_buffer_locked(bh);
                        set_buffer_mapped(bh);
                        submit_bh(REQ_OP_READ, bh);
                }
                blk_cur++;
                bh = bh->b_this_page;
        }

        wait_event(bitmap->write_wait,
                   atomic_read(&bitmap->pending_writes)==0);
        if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
                ret = -EIO;
out:
        if (ret)
                pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
                       (int)PAGE_SIZE,
                       (unsigned long long)index << PAGE_SHIFT,
                       ret);
        return ret;
}
#else /* CONFIG_MD_BITMAP_FILE */
static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
{
}
static int read_file_page(struct file *file, unsigned long index,
                struct bitmap *bitmap, unsigned long count, struct page *page)
{
        return -EIO;
}
static void free_buffers(struct page *page)
{
        put_page(page);
}
#endif /* CONFIG_MD_BITMAP_FILE */

/*
 * bitmap file superblock operations
 */

/*
 * write out a page to a file
 */
static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index,
                               bool wait)
{
        struct bitmap_storage *store = &bitmap->storage;
        struct page *page = store->filemap[pg_index];

        if (mddev_is_clustered(bitmap->mddev)) {
                /* go to node bitmap area starting point */
                pg_index += store->sb_index;
        }

        if (store->file)
                write_file_page(bitmap, page, wait);
        else
                write_sb_page(bitmap, pg_index, page, wait);
}

/*
 * md_bitmap_wait_writes() should be called before writing any bitmap
 * blocks, to ensure previous writes, particularly from
 * md_bitmap_daemon_work(), have completed.
 */
static void md_bitmap_wait_writes(struct bitmap *bitmap)
{
        if (bitmap->storage.file)
                wait_event(bitmap->write_wait,
                           atomic_read(&bitmap->pending_writes)==0);
        else
                /* Note that we ignore the return value.  The writes
                 * might have failed, but that would just mean that
                 * some bits which should be cleared haven't been,
                 * which is safe.  The relevant bitmap blocks will
                 * probably get written again, but there is no great
                 * loss if they aren't.
                 */
                md_super_wait(bitmap->mddev);
}


/* update the event counter and sync the superblock to disk */
static void bitmap_update_sb(void *data)
{
        bitmap_super_t *sb;
        struct bitmap *bitmap = data;

        if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
                return;
        if (bitmap->mddev->bitmap_info.external)
                return;
        if (!bitmap->storage.sb_page) /* no superblock */
                return;
        sb = kmap_local_page(bitmap->storage.sb_page);
        sb->events = cpu_to_le64(bitmap->mddev->events);
        if (bitmap->mddev->events < bitmap->events_cleared)
                /* rocking back to read-only */
                bitmap->events_cleared = bitmap->mddev->events;
        sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
        /*
         * clear BITMAP_WRITE_ERROR bit to protect against the case that
         * a bitmap write error occurred but the later writes succeeded.
         */
        sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
        /* Just in case these have been changed via sysfs: */
        sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
        sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
        /* This might have been changed by a reshape */
        sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
        sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
        sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
        sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
                                           bitmap_info.space);
        kunmap_local(sb);

        if (bitmap->storage.file)
                write_file_page(bitmap, bitmap->storage.sb_page, 1);
        else
                write_sb_page(bitmap, bitmap->storage.sb_index,
                              bitmap->storage.sb_page, 1);
}

static void bitmap_print_sb(struct bitmap *bitmap)
{
        bitmap_super_t *sb;

        if (!bitmap || !bitmap->storage.sb_page)
                return;
        sb = kmap_local_page(bitmap->storage.sb_page);
        pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
        pr_debug("         magic: %08x\n", le32_to_cpu(sb->magic));
        pr_debug("       version: %u\n", le32_to_cpu(sb->version));
        pr_debug("          uuid: %08x.%08x.%08x.%08x\n",
                 le32_to_cpu(*(__le32 *)(sb->uuid+0)),
                 le32_to_cpu(*(__le32 *)(sb->uuid+4)),
                 le32_to_cpu(*(__le32 *)(sb->uuid+8)),
                 le32_to_cpu(*(__le32 *)(sb->uuid+12)));
        pr_debug("        events: %llu\n",
                 (unsigned long long) le64_to_cpu(sb->events));
        pr_debug("events cleared: %llu\n",
                 (unsigned long long) le64_to_cpu(sb->events_cleared));
        pr_debug("         state: %08x\n", le32_to_cpu(sb->state));
        pr_debug("     chunksize: %u B\n", le32_to_cpu(sb->chunksize));
        pr_debug("  daemon sleep: %us\n", le32_to_cpu(sb->daemon_sleep));
        pr_debug("     sync size: %llu KB\n",
                 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
        pr_debug("max write behind: %u\n", le32_to_cpu(sb->write_behind));
        kunmap_local(sb);
}

/*
 * bitmap_new_disk_sb
 * @bitmap
 *
 * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
 * This function verifies 'bitmap_info' and populates the on-disk bitmap
 * structure, which is to be written to disk.
 *
 * Returns: 0 on success, -Exxx on error
 */
static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
{
        bitmap_super_t *sb;
        unsigned long chunksize, daemon_sleep, write_behind;

        bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
        if (bitmap->storage.sb_page == NULL)
                return -ENOMEM;
        bitmap->storage.sb_index = 0;

        sb = kmap_local_page(bitmap->storage.sb_page);

        sb->magic = cpu_to_le32(BITMAP_MAGIC);
        sb->version = cpu_to_le32(BITMAP_MAJOR_HI);

        chunksize = bitmap->mddev->bitmap_info.chunksize;
        BUG_ON(!chunksize);
        if (!is_power_of_2(chunksize)) {
                kunmap_local(sb);
                pr_warn("bitmap chunksize not a power of 2\n");
                return -EINVAL;
        }
        sb->chunksize = cpu_to_le32(chunksize);

        daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
        if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
                pr_debug("Choosing daemon_sleep default (5 sec)\n");
                daemon_sleep = 5 * HZ;
        }
        sb->daemon_sleep = cpu_to_le32(daemon_sleep);
        bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;

        /*
         * FIXME: write_behind for RAID1.  If not specified, what
         * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
         */
        write_behind = bitmap->mddev->bitmap_info.max_write_behind;
        if (write_behind > COUNTER_MAX)
                write_behind = COUNTER_MAX / 2;
        sb->write_behind = cpu_to_le32(write_behind);
        bitmap->mddev->bitmap_info.max_write_behind = write_behind;

        /* keep the array size field of the bitmap superblock up to date */
        sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

        memcpy(sb->uuid, bitmap->mddev->uuid, 16);

        set_bit(BITMAP_STALE, &bitmap->flags);
        sb->state = cpu_to_le32(bitmap->flags);
        bitmap->events_cleared = bitmap->mddev->events;
        sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
        bitmap->mddev->bitmap_info.nodes = 0;

        kunmap_local(sb);

        return 0;
}

/* read the superblock from the bitmap file and initialize some bitmap fields */
static int md_bitmap_read_sb(struct bitmap *bitmap)
{
        char *reason = NULL;
        bitmap_super_t *sb;
        unsigned long chunksize, daemon_sleep, write_behind;
        unsigned long long events;
        int nodes = 0;
        unsigned long sectors_reserved = 0;
        int err = -EINVAL;
        struct page *sb_page;
        loff_t offset = 0;

        if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
                chunksize = 128 * 1024 * 1024;
                daemon_sleep = 5 * HZ;
                write_behind = 0;
                set_bit(BITMAP_STALE, &bitmap->flags);
                err = 0;
                goto out_no_sb;
        }
        /* page 0 is the superblock, read it... */
        sb_page = alloc_page(GFP_KERNEL);
        if (!sb_page)
                return -ENOMEM;
        bitmap->storage.sb_page = sb_page;

re_read:
        /* If cluster_slot is set, the cluster is setup */
        if (bitmap->cluster_slot >= 0) {
                sector_t bm_blocks = bitmap->mddev->resync_max_sectors;

                bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
                           (bitmap->mddev->bitmap_info.chunksize >> 9));
                /* bits to bytes */
                bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
                /* to 4k blocks */
                bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
                offset = bitmap->cluster_slot * (bm_blocks << 3);
                pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
                        bitmap->cluster_slot, offset);
        }

        if (bitmap->storage.file) {
                loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
                int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;

                err = read_file_page(bitmap->storage.file, 0,
                                bitmap, bytes, sb_page);
        } else {
                err = read_sb_page(bitmap->mddev, offset, sb_page, 0,
                                   sizeof(bitmap_super_t));
        }
        if (err)
                return err;

        err = -EINVAL;
        sb = kmap_local_page(sb_page);

        chunksize = le32_to_cpu(sb->chunksize);
        daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
        write_behind = le32_to_cpu(sb->write_behind);
        sectors_reserved = le32_to_cpu(sb->sectors_reserved);

        /* verify that the bitmap-specific fields are valid */
        if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
                reason = "bad magic";
        else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
                 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
                reason = "unrecognized superblock version";
        else if (chunksize < 512)
                reason = "bitmap chunksize too small";
        else if (!is_power_of_2(chunksize))
                reason = "bitmap chunksize not a power of 2";
        else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
                reason = "daemon sleep period out of range";
        else if (write_behind > COUNTER_MAX)
                reason = "write-behind limit out of range (0 - 16383)";
        if (reason) {
                pr_warn("%s: invalid bitmap file superblock: %s\n",
                        bmname(bitmap), reason);
                goto out;
        }

        /*
         * Setup nodes/clustername only if bitmap version is
         * cluster-compatible
         */
        if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
                nodes = le32_to_cpu(sb->nodes);
                strscpy(bitmap->mddev->bitmap_info.cluster_name,
                                sb->cluster_name, 64);
        }

        /* keep the array size field of the bitmap superblock up to date */
        sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

        if (bitmap->mddev->persistent) {
                /*
                 * We have a persistent array superblock, so compare the
                 * bitmap's UUID and event counter to the mddev's
                 */
                if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
                        pr_warn("%s: bitmap superblock UUID mismatch\n",
                                bmname(bitmap));
                        goto out;
                }
                events = le64_to_cpu(sb->events);
                if (!nodes && (events < bitmap->mddev->events)) {
                        pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
                                bmname(bitmap), events,
                                (unsigned long long) bitmap->mddev->events);
                        set_bit(BITMAP_STALE, &bitmap->flags);
                }
        }

        /* assign fields using values from superblock */
        bitmap->flags |= le32_to_cpu(sb->state);
        if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
                set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
        bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
        err = 0;

out:
        kunmap_local(sb);
        if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
                /* Assigning chunksize is required for "re_read" */
                bitmap->mddev->bitmap_info.chunksize = chunksize;
                err = md_setup_cluster(bitmap->mddev, nodes);
                if (err) {
                        pr_warn("%s: Could not setup cluster service (%d)\n",
                                bmname(bitmap), err);
                        goto out_no_sb;
                }
                bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
                goto re_read;
        }

out_no_sb:
        if (err == 0) {
                if (test_bit(BITMAP_STALE, &bitmap->flags))
                        bitmap->events_cleared = bitmap->mddev->events;
                bitmap->mddev->bitmap_info.chunksize = chunksize;
                bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
                bitmap->mddev->bitmap_info.max_write_behind = write_behind;
                bitmap->mddev->bitmap_info.nodes = nodes;
                if (bitmap->mddev->bitmap_info.space == 0 ||
                        bitmap->mddev->bitmap_info.space > sectors_reserved)
                        bitmap->mddev->bitmap_info.space = sectors_reserved;
        } else {
                bitmap_print_sb(bitmap);
                if (bitmap->cluster_slot < 0)
                        md_cluster_stop(bitmap->mddev);
        }
        return err;
}

/*
 * general bitmap file operations
 */

/*
 * on-disk bitmap:
 *
 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
 * file a page at a time. There's a superblock at the start of the file.
 */
/* calculate the index of the page that contains this bit */
static inline unsigned long file_page_index(struct bitmap_storage *store,
                                            unsigned long chunk)
{
        if (store->sb_page)
                chunk += sizeof(bitmap_super_t) << 3;
        return chunk >> PAGE_BIT_SHIFT;
}

/* calculate the (bit) offset of this bit within a page */
static inline unsigned long file_page_offset(struct bitmap_storage *store,
                                             unsigned long chunk)
{
        if (store->sb_page)
                chunk += sizeof(bitmap_super_t) << 3;
        return chunk & (PAGE_BITS - 1);
}

/*
 * return a pointer to the page in the filemap that contains the given bit
 *
 */
static inline struct page *filemap_get_page(struct bitmap_storage *store,
                                            unsigned long chunk)
{
        if (file_page_index(store, chunk) >= store->file_pages)
                return NULL;
        return store->filemap[file_page_index(store, chunk)];
}

static int md_bitmap_storage_alloc(struct bitmap_storage *store,
                                   unsigned long chunks, int with_super,
                                   int slot_number)
{
        int pnum, offset = 0;
        unsigned long num_pages;
        unsigned long bytes;

        bytes = DIV_ROUND_UP(chunks, 8);
        if (with_super)
                bytes += sizeof(bitmap_super_t);

        num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
        offset = slot_number * num_pages;

        store->filemap = kmalloc_array(num_pages, sizeof(struct page *),
                                       GFP_KERNEL);
        if (!store->filemap)
                return -ENOMEM;

        if (with_super && !store->sb_page) {
                store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
                if (store->sb_page == NULL)
                        return -ENOMEM;
        }

        pnum = 0;
        if (store->sb_page) {
                store->filemap[0] = store->sb_page;
                pnum = 1;
                store->sb_index = offset;
        }

        for ( ; pnum < num_pages; pnum++) {
                store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
                if (!store->filemap[pnum]) {
                        store->file_pages = pnum;
                        return -ENOMEM;
                }
        }
        store->file_pages = pnum;

        /* We need 4 bits per page, rounded up to a multiple
         * of sizeof(unsigned long) */
        store->filemap_attr = kzalloc(
                roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
                GFP_KERNEL);
        if (!store->filemap_attr)
                return -ENOMEM;

        store->bytes = bytes;

        return 0;
}

static void md_bitmap_file_unmap(struct bitmap_storage *store)
{
        struct file *file = store->file;
        struct page *sb_page = store->sb_page;
        struct page **map = store->filemap;
        int pages = store->file_pages;

        while (pages--)
                if (map[pages] != sb_page) /* 0 is sb_page, release it below */
                        free_buffers(map[pages]);
        kfree(map);
        kfree(store->filemap_attr);

        if (sb_page)
                free_buffers(sb_page);

        if (file) {
                struct inode *inode = file_inode(file);
                invalidate_mapping_pages(inode->i_mapping, 0, -1);
                fput(file);
        }
}

/*
 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
 * then it is no longer reliable, so we stop using it and we mark the file
 * as failed in the superblock
 */
static void md_bitmap_file_kick(struct bitmap *bitmap)
{
        if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
                bitmap_update_sb(bitmap);

                if (bitmap->storage.file) {
                        pr_warn("%s: kicking failed bitmap file %pD4 from array!\n",
                                bmname(bitmap), bitmap->storage.file);

                } else
                        pr_warn("%s: disabling internal bitmap due to errors\n",
                                bmname(bitmap));
        }
}

enum bitmap_page_attr {
        BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
        BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
                                    * i.e. counter is 1 or 2. */
        BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
};

static inline void set_page_attr(struct bitmap *bitmap, int pnum,
                                 enum bitmap_page_attr attr)
{
        set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
}

static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
                                   enum bitmap_page_attr attr)
{
        clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
}

static inline int test_page_attr(struct bitmap *bitmap, int pnum,
                                 enum bitmap_page_attr attr)
{
        return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
}

static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
                                           enum bitmap_page_attr attr)
{
        return test_and_clear_bit((pnum<<2) + attr,
                                  bitmap->storage.filemap_attr);
}
/*
 * bitmap_file_set_bit -- called before performing a write to the md device
 * to set (and eventually sync) a particular bit in the bitmap file
 *
 * we set the bit immediately, then we record the page number so that
 * when an unplug occurs, we can flush the dirty pages out to disk
 */
static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
{
        unsigned long bit;
        struct page *page;
        void *kaddr;
        unsigned long chunk = block >> bitmap->counts.chunkshift;
        struct bitmap_storage *store = &bitmap->storage;
        unsigned long index = file_page_index(store, chunk);
        unsigned long node_offset = 0;

        index += store->sb_index;
        if (mddev_is_clustered(bitmap->mddev))
                node_offset = bitmap->cluster_slot * store->file_pages;

        page = filemap_get_page(&bitmap->storage, chunk);
        if (!page)
                return;
        bit = file_page_offset(&bitmap->storage, chunk);

        /* set the bit */
        kaddr = kmap_local_page(page);
        if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
                set_bit(bit, kaddr);
        else
                set_bit_le(bit, kaddr);
        kunmap_local(kaddr);
        pr_debug("set file bit %lu page %lu\n", bit, index);
        /* record page number so it gets flushed to disk when unplug occurs */
        set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY);
}

static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
{
        unsigned long bit;
        struct page *page;
        void *paddr;
        unsigned long chunk = block >> bitmap->counts.chunkshift;
        struct bitmap_storage *store = &bitmap->storage;
        unsigned long index = file_page_index(store, chunk);
        unsigned long node_offset = 0;

        index += store->sb_index;
        if (mddev_is_clustered(bitmap->mddev))
                node_offset = bitmap->cluster_slot * store->file_pages;

        page = filemap_get_page(&bitmap->storage, chunk);
        if (!page)
                return;
        bit = file_page_offset(&bitmap->storage, chunk);
        paddr = kmap_local_page(page);
        if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
                clear_bit(bit, paddr);
        else
                clear_bit_le(bit, paddr);
        kunmap_local(paddr);
        if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
                set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING);
                bitmap->allclean = 0;
        }
}

static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
{
        unsigned long bit;
        struct page *page;
        void *paddr;
        unsigned long chunk = block >> bitmap->counts.chunkshift;
        int set = 0;

        page = filemap_get_page(&bitmap->storage, chunk);
        if (!page)
                return -EINVAL;
        bit = file_page_offset(&bitmap->storage, chunk);
        paddr = kmap_local_page(page);
        if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
                set = test_bit(bit, paddr);
        else
                set = test_bit_le(bit, paddr);
        kunmap_local(paddr);
        return set;
}

/* this gets called when the md device is ready to unplug its underlying
 * (slave) device queues -- before we let any writes go down, we need to
 * sync the dirty pages of the bitmap file to disk */
static void __bitmap_unplug(struct bitmap *bitmap)
{
        unsigned long i;
        int dirty, need_write;
        int writing = 0;

        if (!__bitmap_enabled(bitmap))
                return;

        /* look at each page to see if there are any set bits that need to be
         * flushed out to disk */
        for (i = 0; i < bitmap->storage.file_pages; i++) {
                dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
                need_write = test_and_clear_page_attr(bitmap, i,
                                                      BITMAP_PAGE_NEEDWRITE);
                if (dirty || need_write) {
                        if (!writing) {
                                md_bitmap_wait_writes(bitmap);
                                mddev_add_trace_msg(bitmap->mddev,
                                        "md bitmap_unplug");
                        }
                        clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
                        filemap_write_page(bitmap, i, false);
                        writing = 1;
                }
        }
        if (writing)
                md_bitmap_wait_writes(bitmap);

        if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
                md_bitmap_file_kick(bitmap);
}

struct bitmap_unplug_work {
        struct work_struct work;
        struct bitmap *bitmap;
        struct completion *done;
};

static void md_bitmap_unplug_fn(struct work_struct *work)
{
        struct bitmap_unplug_work *unplug_work =
                container_of(work, struct bitmap_unplug_work, work);

        __bitmap_unplug(unplug_work->bitmap);
        complete(unplug_work->done);
}

static void bitmap_unplug_async(struct bitmap *bitmap)
{
        DECLARE_COMPLETION_ONSTACK(done);
        struct bitmap_unplug_work unplug_work;

        INIT_WORK_ONSTACK(&unplug_work.work, md_bitmap_unplug_fn);
        unplug_work.bitmap = bitmap;
        unplug_work.done = &done;

        queue_work(md_bitmap_wq, &unplug_work.work);
        wait_for_completion(&done);
        destroy_work_on_stack(&unplug_work.work);
}

static void bitmap_unplug(struct mddev *mddev, bool sync)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;

        if (sync)
                __bitmap_unplug(bitmap);
        else
                bitmap_unplug_async(bitmap);
}

static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);

/*
 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
 * mapping of the bitmap file.
 *
 * Special case: If there's no bitmap file, or if the bitmap file had been
 * previously kicked from the array, we mark all the bits as 1's in order to
 * cause a full resync.
 *
 * We ignore all bits for sectors that end earlier than 'start'.
 * This is used when reading an out-of-date bitmap.
 */
static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
{
        bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
        struct mddev *mddev = bitmap->mddev;
        unsigned long chunks = bitmap->counts.chunks;
        struct bitmap_storage *store = &bitmap->storage;
        struct file *file = store->file;
        unsigned long node_offset = 0;
        unsigned long bit_cnt = 0;
        unsigned long i;
        int ret;

        if (!file && !mddev->bitmap_info.offset) {
                /* No permanent bitmap - fill with '1s'. */
                store->filemap = NULL;
                store->file_pages = 0;
                for (i = 0; i < chunks ; i++) {
                        /* if the disk bit is set, set the memory bit */
                        int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
                                      >= start);
                        md_bitmap_set_memory_bits(bitmap,
                                                  (sector_t)i << bitmap->counts.chunkshift,
                                                  needed);
                }
                return 0;
        }

        if (file && i_size_read(file->f_mapping->host) < store->bytes) {
                pr_warn("%s: bitmap file too short %lu < %lu\n",
                        bmname(bitmap),
                        (unsigned long) i_size_read(file->f_mapping->host),
                        store->bytes);
                ret = -ENOSPC;
                goto err;
        }

        if (mddev_is_clustered(mddev))
                node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));

        for (i = 0; i < store->file_pages; i++) {
                struct page *page = store->filemap[i];
                int count;

                /* unmap the old page, we're done with it */
                if (i == store->file_pages - 1)
                        count = store->bytes - i * PAGE_SIZE;
                else
                        count = PAGE_SIZE;

                if (file)
                        ret = read_file_page(file, i, bitmap, count, page);
                else
                        ret = read_sb_page(mddev, 0, page, i + node_offset,
                                           count);
                if (ret)
                        goto err;
        }

        if (outofdate) {
                pr_warn("%s: bitmap file is out of date, doing full recovery\n",
                        bmname(bitmap));

                for (i = 0; i < store->file_pages; i++) {
                        struct page *page = store->filemap[i];
                        unsigned long offset = 0;
                        void *paddr;

                        if (i == 0 && !mddev->bitmap_info.external)
                                offset = sizeof(bitmap_super_t);

                        /*
                         * If the bitmap is out of date, dirty the whole page
                         * and write it out
                         */
                        paddr = kmap_local_page(page);
                        memset(paddr + offset, 0xff, PAGE_SIZE - offset);
                        kunmap_local(paddr);

                        filemap_write_page(bitmap, i, true);
                        if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) {
                                ret = -EIO;
                                goto err;
                        }
                }
        }

        for (i = 0; i < chunks; i++) {
                struct page *page = filemap_get_page(&bitmap->storage, i);
                unsigned long bit = file_page_offset(&bitmap->storage, i);
                void *paddr;
                bool was_set;

                paddr = kmap_local_page(page);
                if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
                        was_set = test_bit(bit, paddr);
                else
                        was_set = test_bit_le(bit, paddr);
                kunmap_local(paddr);

                if (was_set) {
                        /* if the disk bit is set, set the memory bit */
                        int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
                                      >= start);
                        md_bitmap_set_memory_bits(bitmap,
                                                  (sector_t)i << bitmap->counts.chunkshift,
                                                  needed);
                        bit_cnt++;
                }
        }

        pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
                 bmname(bitmap), store->file_pages,
                 bit_cnt, chunks);

        return 0;

 err:
        pr_warn("%s: bitmap initialisation failed: %d\n",
                bmname(bitmap), ret);
        return ret;
}

/* just flag bitmap pages as needing to be written. */
static void bitmap_write_all(struct mddev *mddev)
{
        int i;
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap || !bitmap->storage.filemap)
                return;

        /* Only one copy, so nothing needed */
        if (bitmap->storage.file)
                return;

        for (i = 0; i < bitmap->storage.file_pages; i++)
                set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
        bitmap->allclean = 0;
}

static void md_bitmap_count_page(struct bitmap_counts *bitmap,
                                 sector_t offset, int inc)
{
        sector_t chunk = offset >> bitmap->chunkshift;
        unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
        bitmap->bp[page].count += inc;
        md_bitmap_checkfree(bitmap, page);
}

static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
{
        sector_t chunk = offset >> bitmap->chunkshift;
        unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
        struct bitmap_page *bp = &bitmap->bp[page];

        if (!bp->pending)
                bp->pending = 1;
}

static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
                                               sector_t offset, sector_t *blocks,
                                               int create);

static void mddev_set_timeout(struct mddev *mddev, unsigned long timeout,
                              bool force)
{
        struct md_thread *thread;

        rcu_read_lock();
        thread = rcu_dereference(mddev->thread);

        if (!thread)
                goto out;

        if (force || thread->timeout < MAX_SCHEDULE_TIMEOUT)
                thread->timeout = timeout;

out:
        rcu_read_unlock();
}

/*
 * bitmap daemon -- periodically wakes up to clean bits and flush pages
 *                        out to disk
 */
static void bitmap_daemon_work(struct mddev *mddev)
{
        struct bitmap *bitmap;
        unsigned long j;
        unsigned long nextpage;
        sector_t blocks;
        struct bitmap_counts *counts;

        /* Use a mutex to guard daemon_work against
         * bitmap_destroy.
         */
        mutex_lock(&mddev->bitmap_info.mutex);
        bitmap = mddev->bitmap;
        if (bitmap == NULL) {
                mutex_unlock(&mddev->bitmap_info.mutex);
                return;
        }
        if (time_before(jiffies, bitmap->daemon_lastrun
                        + mddev->bitmap_info.daemon_sleep))
                goto done;

        bitmap->daemon_lastrun = jiffies;
        if (bitmap->allclean) {
                mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);
                goto done;
        }
        bitmap->allclean = 1;

        mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work");

        /* Any file-page which is PENDING now needs to be written.
         * So set NEEDWRITE now, then after we make any last-minute changes
         * we will write it.
         */
        for (j = 0; j < bitmap->storage.file_pages; j++)
                if (test_and_clear_page_attr(bitmap, j,
                                             BITMAP_PAGE_PENDING))
                        set_page_attr(bitmap, j,
                                      BITMAP_PAGE_NEEDWRITE);

        if (bitmap->need_sync &&
            mddev->bitmap_info.external == 0) {
                /* Arrange for superblock update as well as
                 * other changes */
                bitmap_super_t *sb;
                bitmap->need_sync = 0;
                if (bitmap->storage.filemap) {
                        sb = kmap_local_page(bitmap->storage.sb_page);
                        sb->events_cleared =
                                cpu_to_le64(bitmap->events_cleared);
                        kunmap_local(sb);
                        set_page_attr(bitmap, 0,
                                      BITMAP_PAGE_NEEDWRITE);
                }
        }
        /* Now look at the bitmap counters and if any are '2' or '1',
         * decrement and handle accordingly.
         */
        counts = &bitmap->counts;
        spin_lock_irq(&counts->lock);
        nextpage = 0;
        for (j = 0; j < counts->chunks; j++) {
                bitmap_counter_t *bmc;
                sector_t  block = (sector_t)j << counts->chunkshift;

                if (j == nextpage) {
                        nextpage += PAGE_COUNTER_RATIO;
                        if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
                                j |= PAGE_COUNTER_MASK;
                                continue;
                        }
                        counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
                }

                bmc = md_bitmap_get_counter(counts, block, &blocks, 0);
                if (!bmc) {
                        j |= PAGE_COUNTER_MASK;
                        continue;
                }
                if (*bmc == 1 && !bitmap->need_sync) {
                        /* We can clear the bit */
                        *bmc = 0;
                        md_bitmap_count_page(counts, block, -1);
                        md_bitmap_file_clear_bit(bitmap, block);
                } else if (*bmc && *bmc <= 2) {
                        *bmc = 1;
                        md_bitmap_set_pending(counts, block);
                        bitmap->allclean = 0;
                }
        }
        spin_unlock_irq(&counts->lock);

        md_bitmap_wait_writes(bitmap);
        /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
         * DIRTY pages need to be written by bitmap_unplug so it can wait
         * for them.
         * If we find any DIRTY page we stop there and let bitmap_unplug
         * handle all the rest.  This is important in the case where
         * the first blocking holds the superblock and it has been updated.
         * We mustn't write any other blocks before the superblock.
         */
        for (j = 0;
             j < bitmap->storage.file_pages
                     && !test_bit(BITMAP_STALE, &bitmap->flags);
             j++) {
                if (test_page_attr(bitmap, j,
                                   BITMAP_PAGE_DIRTY))
                        /* bitmap_unplug will handle the rest */
                        break;
                if (bitmap->storage.filemap &&
                    test_and_clear_page_attr(bitmap, j,
                                             BITMAP_PAGE_NEEDWRITE))
                        filemap_write_page(bitmap, j, false);
        }

 done:
        if (bitmap->allclean == 0)
                mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
        mutex_unlock(&mddev->bitmap_info.mutex);
}

static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
                                               sector_t offset, sector_t *blocks,
                                               int create)
__releases(bitmap->lock)
__acquires(bitmap->lock)
{
        /* If 'create', we might release the lock and reclaim it.
         * The lock must have been taken with interrupts enabled.
         * If !create, we don't release the lock.
         */
        sector_t chunk = offset >> bitmap->chunkshift;
        unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
        unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
        sector_t csize = ((sector_t)1) << bitmap->chunkshift;
        int err;

        if (page >= bitmap->pages) {
                /*
                 * This can happen if bitmap_start_sync goes beyond
                 * End-of-device while looking for a whole page or
                 * user set a huge number to sysfs bitmap_set_bits.
                 */
                *blocks = csize - (offset & (csize - 1));
                return NULL;
        }
        err = md_bitmap_checkpage(bitmap, page, create, 0);

        if (bitmap->bp[page].hijacked ||
            bitmap->bp[page].map == NULL)
                csize = ((sector_t)1) << (bitmap->chunkshift +
                                          PAGE_COUNTER_SHIFT);

        *blocks = csize - (offset & (csize - 1));

        if (err < 0)
                return NULL;

        /* now locked ... */

        if (bitmap->bp[page].hijacked) { /* hijacked pointer */
                /* should we use the first or second counter field
                 * of the hijacked pointer? */
                int hi = (pageoff > PAGE_COUNTER_MASK);
                return  &((bitmap_counter_t *)
                          &bitmap->bp[page].map)[hi];
        } else /* page is allocated */
                return (bitmap_counter_t *)
                        &(bitmap->bp[page].map[pageoff]);
}

static int bitmap_startwrite(struct mddev *mddev, sector_t offset,
                             unsigned long sectors)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return 0;

        while (sectors) {
                sector_t blocks;
                bitmap_counter_t *bmc;

                spin_lock_irq(&bitmap->counts.lock);
                bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
                if (!bmc) {
                        spin_unlock_irq(&bitmap->counts.lock);
                        return 0;
                }

                if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
                        DEFINE_WAIT(__wait);
                        /* note that it is safe to do the prepare_to_wait
                         * after the test as long as we do it before dropping
                         * the spinlock.
                         */
                        prepare_to_wait(&bitmap->overflow_wait, &__wait,
                                        TASK_UNINTERRUPTIBLE);
                        spin_unlock_irq(&bitmap->counts.lock);
                        schedule();
                        finish_wait(&bitmap->overflow_wait, &__wait);
                        continue;
                }

                switch (*bmc) {
                case 0:
                        md_bitmap_file_set_bit(bitmap, offset);
                        md_bitmap_count_page(&bitmap->counts, offset, 1);
                        fallthrough;
                case 1:
                        *bmc = 2;
                }

                (*bmc)++;

                spin_unlock_irq(&bitmap->counts.lock);

                offset += blocks;
                if (sectors > blocks)
                        sectors -= blocks;
                else
                        sectors = 0;
        }
        return 0;
}

static void bitmap_endwrite(struct mddev *mddev, sector_t offset,
                            unsigned long sectors)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;

        while (sectors) {
                sector_t blocks;
                unsigned long flags;
                bitmap_counter_t *bmc;

                spin_lock_irqsave(&bitmap->counts.lock, flags);
                bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
                if (!bmc) {
                        spin_unlock_irqrestore(&bitmap->counts.lock, flags);
                        return;
                }

                if (!bitmap->mddev->degraded) {
                        if (bitmap->events_cleared < bitmap->mddev->events) {
                                bitmap->events_cleared = bitmap->mddev->events;
                                bitmap->need_sync = 1;
                                sysfs_notify_dirent_safe(
                                                bitmap->sysfs_can_clear);
                        }
                } else if (!NEEDED(*bmc)) {
                        *bmc |= NEEDED_MASK;
                }

                if (COUNTER(*bmc) == COUNTER_MAX)
                        wake_up(&bitmap->overflow_wait);

                (*bmc)--;
                if (*bmc <= 2) {
                        md_bitmap_set_pending(&bitmap->counts, offset);
                        bitmap->allclean = 0;
                }
                spin_unlock_irqrestore(&bitmap->counts.lock, flags);
                offset += blocks;
                if (sectors > blocks)
                        sectors -= blocks;
                else
                        sectors = 0;
        }
}

static bool __bitmap_start_sync(struct bitmap *bitmap, sector_t offset,
                                sector_t *blocks, bool degraded)
{
        bitmap_counter_t *bmc;
        bool rv;

        if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
                *blocks = 1024;
                return true; /* always resync if no bitmap */
        }
        spin_lock_irq(&bitmap->counts.lock);

        rv = false;
        bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
        if (bmc) {
                /* locked */
                if (RESYNC(*bmc)) {
                        rv = true;
                } else if (NEEDED(*bmc)) {
                        rv = true;
                        if (!degraded) { /* don't set/clear bits if degraded */
                                *bmc |= RESYNC_MASK;
                                *bmc &= ~NEEDED_MASK;
                        }
                }
        }
        spin_unlock_irq(&bitmap->counts.lock);

        return rv;
}

static bool bitmap_start_sync(struct mddev *mddev, sector_t offset,
                              sector_t *blocks, bool degraded)
{
        /* bitmap_start_sync must always report on multiples of whole
         * pages, otherwise resync (which is very PAGE_SIZE based) will
         * get confused.
         * So call __bitmap_start_sync repeatedly (if needed) until
         * At least PAGE_SIZE>>9 blocks are covered.
         * Return the 'or' of the result.
         */
        bool rv = false;
        sector_t blocks1;

        *blocks = 0;
        while (*blocks < (PAGE_SIZE>>9)) {
                rv |= __bitmap_start_sync(mddev->bitmap, offset,
                                          &blocks1, degraded);
                offset += blocks1;
                *blocks += blocks1;
        }

        return rv;
}

static void __bitmap_end_sync(struct bitmap *bitmap, sector_t offset,
                              sector_t *blocks, bool aborted)
{
        bitmap_counter_t *bmc;
        unsigned long flags;

        if (bitmap == NULL) {
                *blocks = 1024;
                return;
        }
        spin_lock_irqsave(&bitmap->counts.lock, flags);
        bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
        if (bmc == NULL)
                goto unlock;
        /* locked */
        if (RESYNC(*bmc)) {
                *bmc &= ~RESYNC_MASK;

                if (!NEEDED(*bmc) && aborted)
                        *bmc |= NEEDED_MASK;
                else {
                        if (*bmc <= 2) {
                                md_bitmap_set_pending(&bitmap->counts, offset);
                                bitmap->allclean = 0;
                        }
                }
        }
 unlock:
        spin_unlock_irqrestore(&bitmap->counts.lock, flags);
}

static void bitmap_end_sync(struct mddev *mddev, sector_t offset,
                            sector_t *blocks)
{
        __bitmap_end_sync(mddev->bitmap, offset, blocks, true);
}

static void bitmap_close_sync(struct mddev *mddev)
{
        /* Sync has finished, and any bitmap chunks that weren't synced
         * properly have been aborted.  It remains to us to clear the
         * RESYNC bit wherever it is still on
         */
        sector_t sector = 0;
        sector_t blocks;
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;

        while (sector < bitmap->mddev->resync_max_sectors) {
                __bitmap_end_sync(bitmap, sector, &blocks, false);
                sector += blocks;
        }
}

static void bitmap_cond_end_sync(struct mddev *mddev, sector_t sector,
                                 bool force)
{
        sector_t s = 0;
        sector_t blocks;
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;
        if (sector == 0) {
                bitmap->last_end_sync = jiffies;
                return;
        }
        if (!force && time_before(jiffies, (bitmap->last_end_sync
                                  + bitmap->mddev->bitmap_info.daemon_sleep)))
                return;
        wait_event(bitmap->mddev->recovery_wait,
                   atomic_read(&bitmap->mddev->recovery_active) == 0);

        bitmap->mddev->curr_resync_completed = sector;
        set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
        sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
        s = 0;
        while (s < sector && s < bitmap->mddev->resync_max_sectors) {
                __bitmap_end_sync(bitmap, s, &blocks, false);
                s += blocks;
        }
        bitmap->last_end_sync = jiffies;
        sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
}

static void bitmap_sync_with_cluster(struct mddev *mddev,
                                     sector_t old_lo, sector_t old_hi,
                                     sector_t new_lo, sector_t new_hi)
{
        struct bitmap *bitmap = mddev->bitmap;
        sector_t sector, blocks = 0;

        for (sector = old_lo; sector < new_lo; ) {
                __bitmap_end_sync(bitmap, sector, &blocks, false);
                sector += blocks;
        }
        WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");

        for (sector = old_hi; sector < new_hi; ) {
                bitmap_start_sync(mddev, sector, &blocks, false);
                sector += blocks;
        }
        WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
}

static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
{
        /* For each chunk covered by any of these sectors, set the
         * counter to 2 and possibly set resync_needed.  They should all
         * be 0 at this point
         */

        sector_t secs;
        bitmap_counter_t *bmc;
        spin_lock_irq(&bitmap->counts.lock);
        bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
        if (!bmc) {
                spin_unlock_irq(&bitmap->counts.lock);
                return;
        }
        if (!*bmc) {
                *bmc = 2;
                md_bitmap_count_page(&bitmap->counts, offset, 1);
                md_bitmap_set_pending(&bitmap->counts, offset);
                bitmap->allclean = 0;
        }
        if (needed)
                *bmc |= NEEDED_MASK;
        spin_unlock_irq(&bitmap->counts.lock);
}

/* dirty the memory and file bits for bitmap chunks "s" to "e" */
static void bitmap_dirty_bits(struct mddev *mddev, unsigned long s,
                              unsigned long e)
{
        unsigned long chunk;
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;

        for (chunk = s; chunk <= e; chunk++) {
                sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;

                md_bitmap_set_memory_bits(bitmap, sec, 1);
                md_bitmap_file_set_bit(bitmap, sec);
                if (sec < bitmap->mddev->recovery_cp)
                        /* We are asserting that the array is dirty,
                         * so move the recovery_cp address back so
                         * that it is obvious that it is dirty
                         */
                        bitmap->mddev->recovery_cp = sec;
        }
}

static void bitmap_flush(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;
        long sleep;

        if (!bitmap) /* there was no bitmap */
                return;

        /* run the daemon_work three time to ensure everything is flushed
         * that can be
         */
        sleep = mddev->bitmap_info.daemon_sleep * 2;
        bitmap->daemon_lastrun -= sleep;
        bitmap_daemon_work(mddev);
        bitmap->daemon_lastrun -= sleep;
        bitmap_daemon_work(mddev);
        bitmap->daemon_lastrun -= sleep;
        bitmap_daemon_work(mddev);
        if (mddev->bitmap_info.external)
                md_super_wait(mddev);
        bitmap_update_sb(bitmap);
}

static void md_bitmap_free(void *data)
{
        unsigned long k, pages;
        struct bitmap_page *bp;
        struct bitmap *bitmap = data;

        if (!bitmap) /* there was no bitmap */
                return;

        if (bitmap->sysfs_can_clear)
                sysfs_put(bitmap->sysfs_can_clear);

        if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
                bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
                md_cluster_stop(bitmap->mddev);

        /* Shouldn't be needed - but just in case.... */
        wait_event(bitmap->write_wait,
                   atomic_read(&bitmap->pending_writes) == 0);

        /* release the bitmap file  */
        md_bitmap_file_unmap(&bitmap->storage);

        bp = bitmap->counts.bp;
        pages = bitmap->counts.pages;

        /* free all allocated memory */

        if (bp) /* deallocate the page memory */
                for (k = 0; k < pages; k++)
                        if (bp[k].map && !bp[k].hijacked)
                                kfree(bp[k].map);
        kfree(bp);
        kfree(bitmap);
}

static void bitmap_start_behind_write(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;
        int bw;

        if (!bitmap)
                return;

        atomic_inc(&bitmap->behind_writes);
        bw = atomic_read(&bitmap->behind_writes);
        if (bw > bitmap->behind_writes_used)
                bitmap->behind_writes_used = bw;

        pr_debug("inc write-behind count %d/%lu\n",
                 bw, bitmap->mddev->bitmap_info.max_write_behind);
}

static void bitmap_end_behind_write(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return;

        if (atomic_dec_and_test(&bitmap->behind_writes))
                wake_up(&bitmap->behind_wait);
        pr_debug("dec write-behind count %d/%lu\n",
                 atomic_read(&bitmap->behind_writes),
                 bitmap->mddev->bitmap_info.max_write_behind);
}

static void bitmap_wait_behind_writes(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;

        /* wait for behind writes to complete */
        if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
                pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
                         mdname(mddev));
                /* need to kick something here to make sure I/O goes? */
                wait_event(bitmap->behind_wait,
                           atomic_read(&bitmap->behind_writes) == 0);
        }
}

static void bitmap_destroy(struct mddev *mddev)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap) /* there was no bitmap */
                return;

        bitmap_wait_behind_writes(mddev);
        if (!mddev->serialize_policy)
                mddev_destroy_serial_pool(mddev, NULL);

        mutex_lock(&mddev->bitmap_info.mutex);
        spin_lock(&mddev->lock);
        mddev->bitmap = NULL; /* disconnect from the md device */
        spin_unlock(&mddev->lock);
        mutex_unlock(&mddev->bitmap_info.mutex);
        mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);

        md_bitmap_free(bitmap);
}

/*
 * initialize the bitmap structure
 * if this returns an error, bitmap_destroy must be called to do clean up
 * once mddev->bitmap is set
 */
static struct bitmap *__bitmap_create(struct mddev *mddev, int slot)
{
        struct bitmap *bitmap;
        sector_t blocks = mddev->resync_max_sectors;
        struct file *file = mddev->bitmap_info.file;
        int err;
        struct kernfs_node *bm = NULL;

        BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);

        BUG_ON(file && mddev->bitmap_info.offset);

        if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
                pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
                          mdname(mddev));
                return ERR_PTR(-EBUSY);
        }

        bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
        if (!bitmap)
                return ERR_PTR(-ENOMEM);

        spin_lock_init(&bitmap->counts.lock);
        atomic_set(&bitmap->pending_writes, 0);
        init_waitqueue_head(&bitmap->write_wait);
        init_waitqueue_head(&bitmap->overflow_wait);
        init_waitqueue_head(&bitmap->behind_wait);

        bitmap->mddev = mddev;
        bitmap->cluster_slot = slot;

        if (mddev->kobj.sd)
                bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
        if (bm) {
                bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
                sysfs_put(bm);
        } else
                bitmap->sysfs_can_clear = NULL;

        bitmap->storage.file = file;
        if (file) {
                get_file(file);
                /* As future accesses to this file will use bmap,
                 * and bypass the page cache, we must sync the file
                 * first.
                 */
                vfs_fsync(file, 1);
        }
        /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
        if (!mddev->bitmap_info.external) {
                /*
                 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
                 * instructing us to create a new on-disk bitmap instance.
                 */
                if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
                        err = md_bitmap_new_disk_sb(bitmap);
                else
                        err = md_bitmap_read_sb(bitmap);
        } else {
                err = 0;
                if (mddev->bitmap_info.chunksize == 0 ||
                    mddev->bitmap_info.daemon_sleep == 0)
                        /* chunksize and time_base need to be
                         * set first. */
                        err = -EINVAL;
        }
        if (err)
                goto error;

        bitmap->daemon_lastrun = jiffies;
        err = __bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize,
                              true);
        if (err)
                goto error;

        pr_debug("created bitmap (%lu pages) for device %s\n",
                 bitmap->counts.pages, bmname(bitmap));

        err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
        if (err)
                goto error;

        return bitmap;
 error:
        md_bitmap_free(bitmap);
        return ERR_PTR(err);
}

static int bitmap_create(struct mddev *mddev, int slot)
{
        struct bitmap *bitmap = __bitmap_create(mddev, slot);

        if (IS_ERR(bitmap))
                return PTR_ERR(bitmap);

        mddev->bitmap = bitmap;
        return 0;
}

static int bitmap_load(struct mddev *mddev)
{
        int err = 0;
        sector_t start = 0;
        sector_t sector = 0;
        struct bitmap *bitmap = mddev->bitmap;
        struct md_rdev *rdev;

        if (!bitmap)
                goto out;

        rdev_for_each(rdev, mddev)
                mddev_create_serial_pool(mddev, rdev);

        if (mddev_is_clustered(mddev))
                md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);

        /* Clear out old bitmap info first:  Either there is none, or we
         * are resuming after someone else has possibly changed things,
         * so we should forget old cached info.
         * All chunks should be clean, but some might need_sync.
         */
        while (sector < mddev->resync_max_sectors) {
                sector_t blocks;
                bitmap_start_sync(mddev, sector, &blocks, false);
                sector += blocks;
        }
        bitmap_close_sync(mddev);

        if (mddev->degraded == 0
            || bitmap->events_cleared == mddev->events)
                /* no need to keep dirty bits to optimise a
                 * re-add of a missing device */
                start = mddev->recovery_cp;

        mutex_lock(&mddev->bitmap_info.mutex);
        err = md_bitmap_init_from_disk(bitmap, start);
        mutex_unlock(&mddev->bitmap_info.mutex);

        if (err)
                goto out;
        clear_bit(BITMAP_STALE, &bitmap->flags);

        /* Kick recovery in case any bits were set */
        set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);

        mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
        md_wakeup_thread(mddev->thread);

        bitmap_update_sb(bitmap);

        if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
                err = -EIO;
out:
        return err;
}

/* caller need to free returned bitmap with md_bitmap_free() */
static void *bitmap_get_from_slot(struct mddev *mddev, int slot)
{
        int rv = 0;
        struct bitmap *bitmap;

        bitmap = __bitmap_create(mddev, slot);
        if (IS_ERR(bitmap)) {
                rv = PTR_ERR(bitmap);
                return ERR_PTR(rv);
        }

        rv = md_bitmap_init_from_disk(bitmap, 0);
        if (rv) {
                md_bitmap_free(bitmap);
                return ERR_PTR(rv);
        }

        return bitmap;
}

/* Loads the bitmap associated with slot and copies the resync information
 * to our bitmap
 */
static int bitmap_copy_from_slot(struct mddev *mddev, int slot, sector_t *low,
                                 sector_t *high, bool clear_bits)
{
        int rv = 0, i, j;
        sector_t block, lo = 0, hi = 0;
        struct bitmap_counts *counts;
        struct bitmap *bitmap;

        bitmap = bitmap_get_from_slot(mddev, slot);
        if (IS_ERR(bitmap)) {
                pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
                return -1;
        }

        counts = &bitmap->counts;
        for (j = 0; j < counts->chunks; j++) {
                block = (sector_t)j << counts->chunkshift;
                if (md_bitmap_file_test_bit(bitmap, block)) {
                        if (!lo)
                                lo = block;
                        hi = block;
                        md_bitmap_file_clear_bit(bitmap, block);
                        md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
                        md_bitmap_file_set_bit(mddev->bitmap, block);
                }
        }

        if (clear_bits) {
                bitmap_update_sb(bitmap);
                /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
                 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
                for (i = 0; i < bitmap->storage.file_pages; i++)
                        if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
                                set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
                __bitmap_unplug(bitmap);
        }
        __bitmap_unplug(mddev->bitmap);
        *low = lo;
        *high = hi;
        md_bitmap_free(bitmap);

        return rv;
}

static void bitmap_set_pages(void *data, unsigned long pages)
{
        struct bitmap *bitmap = data;

        bitmap->counts.pages = pages;
}

static int bitmap_get_stats(void *data, struct md_bitmap_stats *stats)
{
        struct bitmap_storage *storage;
        struct bitmap_counts *counts;
        struct bitmap *bitmap = data;
        bitmap_super_t *sb;

        if (!bitmap)
                return -ENOENT;
        if (bitmap->mddev->bitmap_info.external)
                return -ENOENT;
        if (!bitmap->storage.sb_page) /* no superblock */
                return -EINVAL;
        sb = kmap_local_page(bitmap->storage.sb_page);
        stats->sync_size = le64_to_cpu(sb->sync_size);
        kunmap_local(sb);

        counts = &bitmap->counts;
        stats->missing_pages = counts->missing_pages;
        stats->pages = counts->pages;

        storage = &bitmap->storage;
        stats->file_pages = storage->file_pages;
        stats->file = storage->file;

        stats->behind_writes = atomic_read(&bitmap->behind_writes);
        stats->behind_wait = wq_has_sleeper(&bitmap->behind_wait);
        stats->events_cleared = bitmap->events_cleared;
        return 0;
}

static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks,
                           int chunksize, bool init)
{
        /* If chunk_size is 0, choose an appropriate chunk size.
         * Then possibly allocate new storage space.
         * Then quiesce, copy bits, replace bitmap, and re-start
         *
         * This function is called both to set up the initial bitmap
         * and to resize the bitmap while the array is active.
         * If this happens as a result of the array being resized,
         * chunksize will be zero, and we need to choose a suitable
         * chunksize, otherwise we use what we are given.
         */
        struct bitmap_storage store;
        struct bitmap_counts old_counts;
        unsigned long chunks;
        sector_t block;
        sector_t old_blocks, new_blocks;
        int chunkshift;
        int ret = 0;
        long pages;
        struct bitmap_page *new_bp;

        if (bitmap->storage.file && !init) {
                pr_info("md: cannot resize file-based bitmap\n");
                return -EINVAL;
        }

        if (chunksize == 0) {
                /* If there is enough space, leave the chunk size unchanged,
                 * else increase by factor of two until there is enough space.
                 */
                long bytes;
                long space = bitmap->mddev->bitmap_info.space;

                if (space == 0) {
                        /* We don't know how much space there is, so limit
                         * to current size - in sectors.
                         */
                        bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
                        if (!bitmap->mddev->bitmap_info.external)
                                bytes += sizeof(bitmap_super_t);
                        space = DIV_ROUND_UP(bytes, 512);
                        bitmap->mddev->bitmap_info.space = space;
                }
                chunkshift = bitmap->counts.chunkshift;
                chunkshift--;
                do {
                        /* 'chunkshift' is shift from block size to chunk size */
                        chunkshift++;
                        chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
                        bytes = DIV_ROUND_UP(chunks, 8);
                        if (!bitmap->mddev->bitmap_info.external)
                                bytes += sizeof(bitmap_super_t);
                } while (bytes > (space << 9) && (chunkshift + BITMAP_BLOCK_SHIFT) <
                        (BITS_PER_BYTE * sizeof(((bitmap_super_t *)0)->chunksize) - 1));
        } else
                chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;

        chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
        memset(&store, 0, sizeof(store));
        if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
                ret = md_bitmap_storage_alloc(&store, chunks,
                                              !bitmap->mddev->bitmap_info.external,
                                              mddev_is_clustered(bitmap->mddev)
                                              ? bitmap->cluster_slot : 0);
        if (ret) {
                md_bitmap_file_unmap(&store);
                goto err;
        }

        pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);

        new_bp = kcalloc(pages, sizeof(*new_bp), GFP_KERNEL);
        ret = -ENOMEM;
        if (!new_bp) {
                md_bitmap_file_unmap(&store);
                goto err;
        }

        if (!init)
                bitmap->mddev->pers->quiesce(bitmap->mddev, 1);

        store.file = bitmap->storage.file;
        bitmap->storage.file = NULL;

        if (store.sb_page && bitmap->storage.sb_page)
                memcpy(page_address(store.sb_page),
                       page_address(bitmap->storage.sb_page),
                       sizeof(bitmap_super_t));
        spin_lock_irq(&bitmap->counts.lock);
        md_bitmap_file_unmap(&bitmap->storage);
        bitmap->storage = store;

        old_counts = bitmap->counts;
        bitmap->counts.bp = new_bp;
        bitmap->counts.pages = pages;
        bitmap->counts.missing_pages = pages;
        bitmap->counts.chunkshift = chunkshift;
        bitmap->counts.chunks = chunks;
        bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
                                                     BITMAP_BLOCK_SHIFT);

        blocks = min(old_counts.chunks << old_counts.chunkshift,
                     chunks << chunkshift);

        /* For cluster raid, need to pre-allocate bitmap */
        if (mddev_is_clustered(bitmap->mddev)) {
                unsigned long page;
                for (page = 0; page < pages; page++) {
                        ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
                        if (ret) {
                                unsigned long k;

                                /* deallocate the page memory */
                                for (k = 0; k < page; k++) {
                                        kfree(new_bp[k].map);
                                }
                                kfree(new_bp);

                                /* restore some fields from old_counts */
                                bitmap->counts.bp = old_counts.bp;
                                bitmap->counts.pages = old_counts.pages;
                                bitmap->counts.missing_pages = old_counts.pages;
                                bitmap->counts.chunkshift = old_counts.chunkshift;
                                bitmap->counts.chunks = old_counts.chunks;
                                bitmap->mddev->bitmap_info.chunksize =
                                        1UL << (old_counts.chunkshift + BITMAP_BLOCK_SHIFT);
                                blocks = old_counts.chunks << old_counts.chunkshift;
                                pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
                                break;
                        } else
                                bitmap->counts.bp[page].count += 1;
                }
        }

        for (block = 0; block < blocks; ) {
                bitmap_counter_t *bmc_old, *bmc_new;
                int set;

                bmc_old = md_bitmap_get_counter(&old_counts, block, &old_blocks, 0);
                set = bmc_old && NEEDED(*bmc_old);

                if (set) {
                        bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
                        if (bmc_new) {
                                if (*bmc_new == 0) {
                                        /* need to set on-disk bits too. */
                                        sector_t end = block + new_blocks;
                                        sector_t start = block >> chunkshift;

                                        start <<= chunkshift;
                                        while (start < end) {
                                                md_bitmap_file_set_bit(bitmap, block);
                                                start += 1 << chunkshift;
                                        }
                                        *bmc_new = 2;
                                        md_bitmap_count_page(&bitmap->counts, block, 1);
                                        md_bitmap_set_pending(&bitmap->counts, block);
                                }
                                *bmc_new |= NEEDED_MASK;
                        }
                        if (new_blocks < old_blocks)
                                old_blocks = new_blocks;
                }
                block += old_blocks;
        }

        if (bitmap->counts.bp != old_counts.bp) {
                unsigned long k;
                for (k = 0; k < old_counts.pages; k++)
                        if (!old_counts.bp[k].hijacked)
                                kfree(old_counts.bp[k].map);
                kfree(old_counts.bp);
        }

        if (!init) {
                int i;
                while (block < (chunks << chunkshift)) {
                        bitmap_counter_t *bmc;
                        bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
                        if (bmc) {
                                /* new space.  It needs to be resynced, so
                                 * we set NEEDED_MASK.
                                 */
                                if (*bmc == 0) {
                                        *bmc = NEEDED_MASK | 2;
                                        md_bitmap_count_page(&bitmap->counts, block, 1);
                                        md_bitmap_set_pending(&bitmap->counts, block);
                                }
                        }
                        block += new_blocks;
                }
                for (i = 0; i < bitmap->storage.file_pages; i++)
                        set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
        }
        spin_unlock_irq(&bitmap->counts.lock);

        if (!init) {
                __bitmap_unplug(bitmap);
                bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
        }
        ret = 0;
err:
        return ret;
}

static int bitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize,
                         bool init)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return 0;

        return __bitmap_resize(bitmap, blocks, chunksize, init);
}

static ssize_t
location_show(struct mddev *mddev, char *page)
{
        ssize_t len;
        if (mddev->bitmap_info.file)
                len = sprintf(page, "file");
        else if (mddev->bitmap_info.offset)
                len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
        else
                len = sprintf(page, "none");
        len += sprintf(page+len, "\n");
        return len;
}

static ssize_t
location_store(struct mddev *mddev, const char *buf, size_t len)
{
        int rv;

        rv = mddev_suspend_and_lock(mddev);
        if (rv)
                return rv;

        if (mddev->pers) {
                if (mddev->recovery || mddev->sync_thread) {
                        rv = -EBUSY;
                        goto out;
                }
        }

        if (mddev->bitmap || mddev->bitmap_info.file ||
            mddev->bitmap_info.offset) {
                /* bitmap already configured.  Only option is to clear it */
                if (strncmp(buf, "none", 4) != 0) {
                        rv = -EBUSY;
                        goto out;
                }

                bitmap_destroy(mddev);
                mddev->bitmap_info.offset = 0;
                if (mddev->bitmap_info.file) {
                        struct file *f = mddev->bitmap_info.file;
                        mddev->bitmap_info.file = NULL;
                        fput(f);
                }
        } else {
                /* No bitmap, OK to set a location */
                long long offset;

                if (strncmp(buf, "none", 4) == 0)
                        /* nothing to be done */;
                else if (strncmp(buf, "file:", 5) == 0) {
                        /* Not supported yet */
                        rv = -EINVAL;
                        goto out;
                } else {
                        if (buf[0] == '+')
                                rv = kstrtoll(buf+1, 10, &offset);
                        else
                                rv = kstrtoll(buf, 10, &offset);
                        if (rv)
                                goto out;
                        if (offset == 0) {
                                rv = -EINVAL;
                                goto out;
                        }
                        if (mddev->bitmap_info.external == 0 &&
                            mddev->major_version == 0 &&
                            offset != mddev->bitmap_info.default_offset) {
                                rv = -EINVAL;
                                goto out;
                        }

                        mddev->bitmap_info.offset = offset;
                        rv = bitmap_create(mddev, -1);
                        if (rv)
                                goto out;

                        rv = bitmap_load(mddev);
                        if (rv) {
                                mddev->bitmap_info.offset = 0;
                                bitmap_destroy(mddev);
                                goto out;
                        }
                }
        }
        if (!mddev->external) {
                /* Ensure new bitmap info is stored in
                 * metadata promptly.
                 */
                set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
                md_wakeup_thread(mddev->thread);
        }
        rv = 0;
out:
        mddev_unlock_and_resume(mddev);
        if (rv)
                return rv;
        return len;
}

static struct md_sysfs_entry bitmap_location =
__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);

/* 'bitmap/space' is the space available at 'location' for the
 * bitmap.  This allows the kernel to know when it is safe to
 * resize the bitmap to match a resized array.
 */
static ssize_t
space_show(struct mddev *mddev, char *page)
{
        return sprintf(page, "%lu\n", mddev->bitmap_info.space);
}

static ssize_t
space_store(struct mddev *mddev, const char *buf, size_t len)
{
        struct bitmap *bitmap;
        unsigned long sectors;
        int rv;

        rv = kstrtoul(buf, 10, &sectors);
        if (rv)
                return rv;

        if (sectors == 0)
                return -EINVAL;

        bitmap = mddev->bitmap;
        if (bitmap && sectors < (bitmap->storage.bytes + 511) >> 9)
                return -EFBIG; /* Bitmap is too big for this small space */

        /* could make sure it isn't too big, but that isn't really
         * needed - user-space should be careful.
         */
        mddev->bitmap_info.space = sectors;
        return len;
}

static struct md_sysfs_entry bitmap_space =
__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);

static ssize_t
timeout_show(struct mddev *mddev, char *page)
{
        ssize_t len;
        unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
        unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;

        len = sprintf(page, "%lu", secs);
        if (jifs)
                len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
        len += sprintf(page+len, "\n");
        return len;
}

static ssize_t
timeout_store(struct mddev *mddev, const char *buf, size_t len)
{
        /* timeout can be set at any time */
        unsigned long timeout;
        int rv = strict_strtoul_scaled(buf, &timeout, 4);
        if (rv)
                return rv;

        /* just to make sure we don't overflow... */
        if (timeout >= LONG_MAX / HZ)
                return -EINVAL;

        timeout = timeout * HZ / 10000;

        if (timeout >= MAX_SCHEDULE_TIMEOUT)
                timeout = MAX_SCHEDULE_TIMEOUT-1;
        if (timeout < 1)
                timeout = 1;

        mddev->bitmap_info.daemon_sleep = timeout;
        mddev_set_timeout(mddev, timeout, false);
        md_wakeup_thread(mddev->thread);

        return len;
}

static struct md_sysfs_entry bitmap_timeout =
__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);

static ssize_t
backlog_show(struct mddev *mddev, char *page)
{
        return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
}

static ssize_t
backlog_store(struct mddev *mddev, const char *buf, size_t len)
{
        unsigned long backlog;
        unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
        struct md_rdev *rdev;
        bool has_write_mostly = false;
        int rv = kstrtoul(buf, 10, &backlog);
        if (rv)
                return rv;
        if (backlog > COUNTER_MAX)
                return -EINVAL;

        rv = mddev_suspend_and_lock(mddev);
        if (rv)
                return rv;

        /*
         * Without write mostly device, it doesn't make sense to set
         * backlog for max_write_behind.
         */
        rdev_for_each(rdev, mddev) {
                if (test_bit(WriteMostly, &rdev->flags)) {
                        has_write_mostly = true;
                        break;
                }
        }
        if (!has_write_mostly) {
                pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
                                    mdname(mddev));
                mddev_unlock(mddev);
                return -EINVAL;
        }

        mddev->bitmap_info.max_write_behind = backlog;
        if (!backlog && mddev->serial_info_pool) {
                /* serial_info_pool is not needed if backlog is zero */
                if (!mddev->serialize_policy)
                        mddev_destroy_serial_pool(mddev, NULL);
        } else if (backlog && !mddev->serial_info_pool) {
                /* serial_info_pool is needed since backlog is not zero */
                rdev_for_each(rdev, mddev)
                        mddev_create_serial_pool(mddev, rdev);
        }
        if (old_mwb != backlog)
                bitmap_update_sb(mddev->bitmap);

        mddev_unlock_and_resume(mddev);
        return len;
}

static struct md_sysfs_entry bitmap_backlog =
__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);

static ssize_t
chunksize_show(struct mddev *mddev, char *page)
{
        return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
}

static ssize_t
chunksize_store(struct mddev *mddev, const char *buf, size_t len)
{
        /* Can only be changed when no bitmap is active */
        int rv;
        unsigned long csize;
        if (mddev->bitmap)
                return -EBUSY;
        rv = kstrtoul(buf, 10, &csize);
        if (rv)
                return rv;
        if (csize < 512 ||
            !is_power_of_2(csize))
                return -EINVAL;
        if (BITS_PER_LONG > 32 && csize >= (1ULL << (BITS_PER_BYTE *
                sizeof(((bitmap_super_t *)0)->chunksize))))
                return -EOVERFLOW;
        mddev->bitmap_info.chunksize = csize;
        return len;
}

static struct md_sysfs_entry bitmap_chunksize =
__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);

static ssize_t metadata_show(struct mddev *mddev, char *page)
{
        if (mddev_is_clustered(mddev))
                return sprintf(page, "clustered\n");
        return sprintf(page, "%s\n", (mddev->bitmap_info.external
                                      ? "external" : "internal"));
}

static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
{
        if (mddev->bitmap ||
            mddev->bitmap_info.file ||
            mddev->bitmap_info.offset)
                return -EBUSY;
        if (strncmp(buf, "external", 8) == 0)
                mddev->bitmap_info.external = 1;
        else if ((strncmp(buf, "internal", 8) == 0) ||
                        (strncmp(buf, "clustered", 9) == 0))
                mddev->bitmap_info.external = 0;
        else
                return -EINVAL;
        return len;
}

static struct md_sysfs_entry bitmap_metadata =
__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);

static ssize_t can_clear_show(struct mddev *mddev, char *page)
{
        int len;
        struct bitmap *bitmap;

        spin_lock(&mddev->lock);
        bitmap = mddev->bitmap;
        if (bitmap)
                len = sprintf(page, "%s\n", (bitmap->need_sync ? "false" :
                                                                 "true"));
        else
                len = sprintf(page, "\n");
        spin_unlock(&mddev->lock);
        return len;
}

static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (!bitmap)
                return -ENOENT;

        if (strncmp(buf, "false", 5) == 0) {
                bitmap->need_sync = 1;
                return len;
        }

        if (strncmp(buf, "true", 4) == 0) {
                if (mddev->degraded)
                        return -EBUSY;
                bitmap->need_sync = 0;
                return len;
        }

        return -EINVAL;
}

static struct md_sysfs_entry bitmap_can_clear =
__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);

static ssize_t
behind_writes_used_show(struct mddev *mddev, char *page)
{
        ssize_t ret;
        struct bitmap *bitmap;

        spin_lock(&mddev->lock);
        bitmap = mddev->bitmap;
        if (!bitmap)
                ret = sprintf(page, "0\n");
        else
                ret = sprintf(page, "%lu\n", bitmap->behind_writes_used);
        spin_unlock(&mddev->lock);

        return ret;
}

static ssize_t
behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
{
        struct bitmap *bitmap = mddev->bitmap;

        if (bitmap)
                bitmap->behind_writes_used = 0;
        return len;
}

static struct md_sysfs_entry max_backlog_used =
__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
       behind_writes_used_show, behind_writes_used_reset);

static struct attribute *md_bitmap_attrs[] = {
        &bitmap_location.attr,
        &bitmap_space.attr,
        &bitmap_timeout.attr,
        &bitmap_backlog.attr,
        &bitmap_chunksize.attr,
        &bitmap_metadata.attr,
        &bitmap_can_clear.attr,
        &max_backlog_used.attr,
        NULL
};
const struct attribute_group md_bitmap_group = {
        .name = "bitmap",
        .attrs = md_bitmap_attrs,
};

static struct bitmap_operations bitmap_ops = {
        .enabled                = bitmap_enabled,
        .create                        = bitmap_create,
        .resize                        = bitmap_resize,
        .load                        = bitmap_load,
        .destroy                = bitmap_destroy,
        .flush                        = bitmap_flush,
        .write_all                = bitmap_write_all,
        .dirty_bits                = bitmap_dirty_bits,
        .unplug                        = bitmap_unplug,
        .daemon_work                = bitmap_daemon_work,

        .start_behind_write        = bitmap_start_behind_write,
        .end_behind_write        = bitmap_end_behind_write,
        .wait_behind_writes        = bitmap_wait_behind_writes,

        .startwrite                = bitmap_startwrite,
        .endwrite                = bitmap_endwrite,
        .start_sync                = bitmap_start_sync,
        .end_sync                = bitmap_end_sync,
        .cond_end_sync                = bitmap_cond_end_sync,
        .close_sync                = bitmap_close_sync,

        .update_sb                = bitmap_update_sb,
        .get_stats                = bitmap_get_stats,

        .sync_with_cluster        = bitmap_sync_with_cluster,
        .get_from_slot                = bitmap_get_from_slot,
        .copy_from_slot                = bitmap_copy_from_slot,
        .set_pages                = bitmap_set_pages,
        .free                        = md_bitmap_free,
};

void mddev_set_bitmap_ops(struct mddev *mddev)
{
        mddev->bitmap_ops = &bitmap_ops;
}




















































































































    1 

















    5 
    5 








































    1 




    1 



    1 
    1 
    1 


    1 
















    1 





































    8 




    8 
    8 
    8 

    1 

    4 










    1 

    3 



    5 











































































    1 















    1 









    3 


    2 




    1 


    1 
    1 
    1 






    1 
    1 



    1 










    2 


























































































































































































































































































































































    1 
    1 
    1 












    5 













































































































































    1 


















    1 
























































































































    1 











    1 



    1 
    1 

    1 


    1 

    1 




















    1 



















    6 
    1 
    1 








    8 






    8 
































    8 


    8 
    8 


    3 




    1 

    1 


    1 













    1 






































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  Copyright (c) 1999-2021 Petko Manolov (petkan@nucleusys.com)
 *
 */

#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
#include <linux/usb.h>
#include <linux/module.h>
#include <asm/byteorder.h>
#include <linux/uaccess.h>
#include "pegasus.h"

/*
 * Version Information
 */
#define DRIVER_AUTHOR "Petko Manolov <petkan@nucleusys.com>"
#define DRIVER_DESC "Pegasus/Pegasus II USB Ethernet driver"

static const char driver_name[] = "pegasus";

#undef        PEGASUS_WRITE_EEPROM
#define        BMSR_MEDIA        (BMSR_10HALF | BMSR_10FULL | BMSR_100HALF | \
                        BMSR_100FULL | BMSR_ANEGCAPABLE)
#define CARRIER_CHECK_DELAY (2 * HZ)

static bool loopback;
static bool mii_mode;
static char *devid;

static struct usb_eth_dev usb_dev_id[] = {
#define        PEGASUS_DEV(pn, vid, pid, flags)        \
        {.name = pn, .vendor = vid, .device = pid, .private = flags},
#define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \
        PEGASUS_DEV(pn, vid, pid, flags)
#include "pegasus.h"
#undef        PEGASUS_DEV
#undef        PEGASUS_DEV_CLASS
        {NULL, 0, 0, 0},
        {NULL, 0, 0, 0}
};

static struct usb_device_id pegasus_ids[] = {
#define        PEGASUS_DEV(pn, vid, pid, flags) \
        {.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid},
/*
 * The Belkin F8T012xx1 bluetooth adaptor has the same vendor and product
 * IDs as the Belkin F5D5050, so we need to teach the pegasus driver to
 * ignore adaptors belonging to the "Wireless" class 0xE0. For this one
 * case anyway, seeing as the pegasus is for "Wired" adaptors.
 */
#define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \
        {.match_flags = (USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_DEV_CLASS), \
        .idVendor = vid, .idProduct = pid, .bDeviceClass = dclass},
#include "pegasus.h"
#undef        PEGASUS_DEV
#undef        PEGASUS_DEV_CLASS
        {},
        {}
};

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");
module_param(loopback, bool, 0);
module_param(mii_mode, bool, 0);
module_param(devid, charp, 0);
MODULE_PARM_DESC(loopback, "Enable MAC loopback mode (bit 0)");
MODULE_PARM_DESC(mii_mode, "Enable HomePNA mode (bit 0),default=MII mode = 0");
MODULE_PARM_DESC(devid, "The format is: 'DEV_name:VendorID:DeviceID:Flags'");

/* use ethtool to change the level for any given device */
static int msg_level = -1;
module_param(msg_level, int, 0);
MODULE_PARM_DESC(msg_level, "Override default message level");

MODULE_DEVICE_TABLE(usb, pegasus_ids);
static const struct net_device_ops pegasus_netdev_ops;

/*****/

static void async_ctrl_callback(struct urb *urb)
{
        struct usb_ctrlrequest *req = (struct usb_ctrlrequest *)urb->context;
        int status = urb->status;

        if (status < 0)
                dev_dbg(&urb->dev->dev, "%s failed with %d", __func__, status);
        kfree(req);
        usb_free_urb(urb);
}

static int get_registers(pegasus_t *pegasus, __u16 indx, __u16 size, void *data)
{
        return usb_control_msg_recv(pegasus->usb, 0, PEGASUS_REQ_GET_REGS,
                                   PEGASUS_REQT_READ, 0, indx, data, size,
                                   1000, GFP_NOIO);
}

static int set_registers(pegasus_t *pegasus, __u16 indx, __u16 size,
                         const void *data)
{
        int ret;

        ret = usb_control_msg_send(pegasus->usb, 0, PEGASUS_REQ_SET_REGS,
                                    PEGASUS_REQT_WRITE, 0, indx, data, size,
                                    1000, GFP_NOIO);
        if (ret < 0)
                netif_dbg(pegasus, drv, pegasus->net, "%s failed with %d\n", __func__, ret);

        return ret;
}

/*
 * There is only one way to write to a single ADM8511 register and this is via
 * specific control request.  'data' is ignored by the device, but it is here to
 * not break the API.
 */
static int set_register(pegasus_t *pegasus, __u16 indx, __u8 data)
{
        void *buf = &data;
        int ret;

        ret = usb_control_msg_send(pegasus->usb, 0, PEGASUS_REQ_SET_REG,
                                    PEGASUS_REQT_WRITE, data, indx, buf, 1,
                                    1000, GFP_NOIO);
        if (ret < 0)
                netif_dbg(pegasus, drv, pegasus->net, "%s failed with %d\n", __func__, ret);

        return ret;
}

static int update_eth_regs_async(pegasus_t *pegasus)
{
        int ret = -ENOMEM;
        struct urb *async_urb;
        struct usb_ctrlrequest *req;

        req = kmalloc(sizeof(struct usb_ctrlrequest), GFP_ATOMIC);
        if (req == NULL)
                return ret;

        async_urb = usb_alloc_urb(0, GFP_ATOMIC);
        if (async_urb == NULL) {
                kfree(req);
                return ret;
        }
        req->bRequestType = PEGASUS_REQT_WRITE;
        req->bRequest = PEGASUS_REQ_SET_REGS;
        req->wValue = cpu_to_le16(0);
        req->wIndex = cpu_to_le16(EthCtrl0);
        req->wLength = cpu_to_le16(3);

        usb_fill_control_urb(async_urb, pegasus->usb,
                             usb_sndctrlpipe(pegasus->usb, 0), (void *)req,
                             pegasus->eth_regs, 3, async_ctrl_callback, req);

        ret = usb_submit_urb(async_urb, GFP_ATOMIC);
        if (ret) {
                if (ret == -ENODEV)
                        netif_device_detach(pegasus->net);
                netif_err(pegasus, drv, pegasus->net,
                          "%s returned %d\n", __func__, ret);
        }
        return ret;
}

static int __mii_op(pegasus_t *p, __u8 phy, __u8 indx, __u16 *regd, __u8 cmd)
{
        int i, ret;
        __le16 regdi;
        __u8 data[4] = { phy, 0, 0, indx };

        if (cmd & PHY_WRITE) {
                __le16 *t = (__le16 *) & data[1];
                *t = cpu_to_le16(*regd);
        }
        set_register(p, PhyCtrl, 0);
        set_registers(p, PhyAddr, sizeof(data), data);
        set_register(p, PhyCtrl, (indx | cmd));
        for (i = 0; i < REG_TIMEOUT; i++) {
                ret = get_registers(p, PhyCtrl, 1, data);
                if (ret < 0)
                        goto fail;
                if (data[0] & PHY_DONE)
                        break;
        }
        if (i >= REG_TIMEOUT) {
                ret = -ETIMEDOUT;
                goto fail;
        }
        if (cmd & PHY_READ) {
                ret = get_registers(p, PhyData, 2, &regdi);
                if (ret < 0)
                        goto fail;
                *regd = le16_to_cpu(regdi);
        }
        return 0;
fail:
        netif_dbg(p, drv, p->net, "%s failed\n", __func__);
        return ret;
}

/* Returns non-negative int on success, error on failure */
static int read_mii_word(pegasus_t *pegasus, __u8 phy, __u8 indx, __u16 *regd)
{
        return __mii_op(pegasus, phy, indx, regd, PHY_READ);
}

/* Returns zero on success, error on failure */
static int write_mii_word(pegasus_t *pegasus, __u8 phy, __u8 indx, __u16 *regd)
{
        return __mii_op(pegasus, phy, indx, regd, PHY_WRITE);
}

static int mdio_read(struct net_device *dev, int phy_id, int loc)
{
        pegasus_t *pegasus = netdev_priv(dev);
        int ret;
        u16 res;

        ret = read_mii_word(pegasus, phy_id, loc, &res);
        if (ret < 0)
                return ret;

        return (int)res;
}

static void mdio_write(struct net_device *dev, int phy_id, int loc, int val)
{
        pegasus_t *pegasus = netdev_priv(dev);
        u16 data = val;

        write_mii_word(pegasus, phy_id, loc, &data);
}

static int read_eprom_word(pegasus_t *pegasus, __u8 index, __u16 *retdata)
{
        int ret, i;
        __le16 retdatai;
        __u8 tmp = 0;

        set_register(pegasus, EpromCtrl, 0);
        set_register(pegasus, EpromOffset, index);
        set_register(pegasus, EpromCtrl, EPROM_READ);

        for (i = 0; i < REG_TIMEOUT; i++) {
                ret = get_registers(pegasus, EpromCtrl, 1, &tmp);
                if (ret < 0)
                        goto fail;
                if (tmp & EPROM_DONE)
                        break;
        }
        if (i >= REG_TIMEOUT) {
                ret = -ETIMEDOUT;
                goto fail;
        }

        ret = get_registers(pegasus, EpromData, 2, &retdatai);
        if (ret < 0)
                goto fail;
        *retdata = le16_to_cpu(retdatai);
        return ret;

fail:
        netif_dbg(pegasus, drv, pegasus->net, "%s failed\n", __func__);
        return ret;
}

#ifdef        PEGASUS_WRITE_EEPROM
static inline void enable_eprom_write(pegasus_t *pegasus)
{
        __u8 tmp;

        get_registers(pegasus, EthCtrl2, 1, &tmp);
        set_register(pegasus, EthCtrl2, tmp | EPROM_WR_ENABLE);
}

static inline void disable_eprom_write(pegasus_t *pegasus)
{
        __u8 tmp;

        get_registers(pegasus, EthCtrl2, 1, &tmp);
        set_register(pegasus, EpromCtrl, 0);
        set_register(pegasus, EthCtrl2, tmp & ~EPROM_WR_ENABLE);
}

static int write_eprom_word(pegasus_t *pegasus, __u8 index, __u16 data)
{
        int i;
        __u8 tmp, d[4] = { 0x3f, 0, 0, EPROM_WRITE };
        int ret;
        __le16 le_data = cpu_to_le16(data);

        set_registers(pegasus, EpromOffset, 4, d);
        enable_eprom_write(pegasus);
        set_register(pegasus, EpromOffset, index);
        set_registers(pegasus, EpromData, 2, &le_data);
        set_register(pegasus, EpromCtrl, EPROM_WRITE);

        for (i = 0; i < REG_TIMEOUT; i++) {
                ret = get_registers(pegasus, EpromCtrl, 1, &tmp);
                if (ret == -ESHUTDOWN)
                        goto fail;
                if (tmp & EPROM_DONE)
                        break;
        }
        disable_eprom_write(pegasus);
        if (i >= REG_TIMEOUT)
                goto fail;

        return ret;

fail:
        netif_dbg(pegasus, drv, pegasus->net, "%s failed\n", __func__);
        return -ETIMEDOUT;
}
#endif        /* PEGASUS_WRITE_EEPROM */

static inline int get_node_id(pegasus_t *pegasus, u8 *id)
{
        int i, ret;
        u16 w16;

        for (i = 0; i < 3; i++) {
                ret = read_eprom_word(pegasus, i, &w16);
                if (ret < 0)
                        return ret;
                ((__le16 *) id)[i] = cpu_to_le16(w16);
        }

        return 0;
}

static void set_ethernet_addr(pegasus_t *pegasus)
{
        int ret;
        u8 node_id[6];

        if (pegasus->features & PEGASUS_II) {
                ret = get_registers(pegasus, 0x10, sizeof(node_id), node_id);
                if (ret < 0)
                        goto err;
        } else {
                ret = get_node_id(pegasus, node_id);
                if (ret < 0)
                        goto err;
                ret = set_registers(pegasus, EthID, sizeof(node_id), node_id);
                if (ret < 0)
                        goto err;
        }

        eth_hw_addr_set(pegasus->net, node_id);

        return;
err:
        eth_hw_addr_random(pegasus->net);
        netif_dbg(pegasus, drv, pegasus->net, "software assigned MAC address.\n");

        return;
}

static inline int reset_mac(pegasus_t *pegasus)
{
        int ret, i;
        __u8 data = 0x8;

        set_register(pegasus, EthCtrl1, data);
        for (i = 0; i < REG_TIMEOUT; i++) {
                ret = get_registers(pegasus, EthCtrl1, 1, &data);
                if (ret < 0)
                        goto fail;
                if (~data & 0x08) {
                        if (loopback)
                                break;
                        if (mii_mode && (pegasus->features & HAS_HOME_PNA))
                                set_register(pegasus, Gpio1, 0x34);
                        else
                                set_register(pegasus, Gpio1, 0x26);
                        set_register(pegasus, Gpio0, pegasus->features);
                        set_register(pegasus, Gpio0, DEFAULT_GPIO_SET);
                        break;
                }
        }
        if (i == REG_TIMEOUT)
                return -ETIMEDOUT;

        if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS ||
            usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) {
                set_register(pegasus, Gpio0, 0x24);
                set_register(pegasus, Gpio0, 0x26);
        }
        if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_ELCON) {
                __u16 auxmode;
                ret = read_mii_word(pegasus, 3, 0x1b, &auxmode);
                if (ret < 0)
                        goto fail;
                auxmode |= 4;
                write_mii_word(pegasus, 3, 0x1b, &auxmode);
        }

        return 0;
fail:
        netif_dbg(pegasus, drv, pegasus->net, "%s failed\n", __func__);
        return ret;
}

static int enable_net_traffic(struct net_device *dev, struct usb_device *usb)
{
        pegasus_t *pegasus = netdev_priv(dev);
        int ret;
        __u16 linkpart;
        __u8 data[4];

        ret = read_mii_word(pegasus, pegasus->phy, MII_LPA, &linkpart);
        if (ret < 0)
                goto fail;
        data[0] = 0xc8; /* TX & RX enable, append status, no CRC */
        data[1] = 0;
        if (linkpart & (ADVERTISE_100FULL | ADVERTISE_10FULL))
                data[1] |= 0x20;        /* set full duplex */
        if (linkpart & (ADVERTISE_100FULL | ADVERTISE_100HALF))
                data[1] |= 0x10;        /* set 100 Mbps */
        if (mii_mode)
                data[1] = 0;
        data[2] = loopback ? 0x09 : 0x01;

        memcpy(pegasus->eth_regs, data, sizeof(data));
        ret = set_registers(pegasus, EthCtrl0, 3, data);

        if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS ||
            usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS2 ||
            usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) {
                u16 auxmode;
                ret = read_mii_word(pegasus, 0, 0x1b, &auxmode);
                if (ret < 0)
                        goto fail;
                auxmode |= 4;
                write_mii_word(pegasus, 0, 0x1b, &auxmode);
        }

        return ret;
fail:
        netif_dbg(pegasus, drv, pegasus->net, "%s failed\n", __func__);
        return ret;
}

static void read_bulk_callback(struct urb *urb)
{
        pegasus_t *pegasus = urb->context;
        struct net_device *net;
        u8 *buf = urb->transfer_buffer;
        int rx_status, count = urb->actual_length;
        int status = urb->status;
        __u16 pkt_len;

        if (!pegasus)
                return;

        net = pegasus->net;
        if (!netif_device_present(net) || !netif_running(net))
                return;

        switch (status) {
        case 0:
                break;
        case -ETIME:
                netif_dbg(pegasus, rx_err, net, "reset MAC\n");
                pegasus->flags &= ~PEGASUS_RX_BUSY;
                break;
        case -EPIPE:                /* stall, or disconnect from TT */
                /* FIXME schedule work to clear the halt */
                netif_warn(pegasus, rx_err, net, "no rx stall recovery\n");
                return;
        case -ENOENT:
        case -ECONNRESET:
        case -ESHUTDOWN:
                netif_dbg(pegasus, ifdown, net, "rx unlink, %d\n", status);
                return;
        default:
                netif_dbg(pegasus, rx_err, net, "RX status %d\n", status);
                goto goon;
        }

        if (count < 4)
                goto goon;

        rx_status = buf[count - 2];
        if (rx_status & 0x1c) {
                netif_dbg(pegasus, rx_err, net,
                          "RX packet error %x\n", rx_status);
                net->stats.rx_errors++;
                if (rx_status & 0x04)        /* runt        */
                        net->stats.rx_length_errors++;
                if (rx_status & 0x08)
                        net->stats.rx_crc_errors++;
                if (rx_status & 0x10)        /* extra bits        */
                        net->stats.rx_frame_errors++;
                goto goon;
        }
        if (pegasus->chip == 0x8513) {
                pkt_len = le32_to_cpu(*(__le32 *)urb->transfer_buffer);
                pkt_len &= 0x0fff;
                pegasus->rx_skb->data += 2;
        } else {
                pkt_len = buf[count - 3] << 8;
                pkt_len += buf[count - 4];
                pkt_len &= 0xfff;
                pkt_len -= 4;
        }

        /*
         * If the packet is unreasonably long, quietly drop it rather than
         * kernel panicing by calling skb_put.
         */
        if (pkt_len > PEGASUS_MTU)
                goto goon;

        /*
         * at this point we are sure pegasus->rx_skb != NULL
         * so we go ahead and pass up the packet.
         */
        skb_put(pegasus->rx_skb, pkt_len);
        pegasus->rx_skb->protocol = eth_type_trans(pegasus->rx_skb, net);
        netif_rx(pegasus->rx_skb);
        net->stats.rx_packets++;
        net->stats.rx_bytes += pkt_len;

        if (pegasus->flags & PEGASUS_UNPLUG)
                return;

        pegasus->rx_skb = __netdev_alloc_skb_ip_align(pegasus->net, PEGASUS_MTU,
                                                      GFP_ATOMIC);

        if (pegasus->rx_skb == NULL)
                goto tl_sched;
goon:
        usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb,
                          usb_rcvbulkpipe(pegasus->usb, 1),
                          pegasus->rx_skb->data, PEGASUS_MTU,
                          read_bulk_callback, pegasus);
        rx_status = usb_submit_urb(pegasus->rx_urb, GFP_ATOMIC);
        if (rx_status == -ENODEV)
                netif_device_detach(pegasus->net);
        else if (rx_status) {
                pegasus->flags |= PEGASUS_RX_URB_FAIL;
                goto tl_sched;
        } else {
                pegasus->flags &= ~PEGASUS_RX_URB_FAIL;
        }

        return;

tl_sched:
        tasklet_schedule(&pegasus->rx_tl);
}

static void rx_fixup(struct tasklet_struct *t)
{
        pegasus_t *pegasus = from_tasklet(pegasus, t, rx_tl);
        int status;

        if (pegasus->flags & PEGASUS_UNPLUG)
                return;

        if (pegasus->flags & PEGASUS_RX_URB_FAIL)
                if (pegasus->rx_skb)
                        goto try_again;
        if (pegasus->rx_skb == NULL)
                pegasus->rx_skb = __netdev_alloc_skb_ip_align(pegasus->net,
                                                              PEGASUS_MTU,
                                                              GFP_ATOMIC);
        if (pegasus->rx_skb == NULL) {
                netif_warn(pegasus, rx_err, pegasus->net, "low on memory\n");
                tasklet_schedule(&pegasus->rx_tl);
                return;
        }
        usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb,
                          usb_rcvbulkpipe(pegasus->usb, 1),
                          pegasus->rx_skb->data, PEGASUS_MTU,
                          read_bulk_callback, pegasus);
try_again:
        status = usb_submit_urb(pegasus->rx_urb, GFP_ATOMIC);
        if (status == -ENODEV)
                netif_device_detach(pegasus->net);
        else if (status) {
                pegasus->flags |= PEGASUS_RX_URB_FAIL;
                tasklet_schedule(&pegasus->rx_tl);
        } else {
                pegasus->flags &= ~PEGASUS_RX_URB_FAIL;
        }
}

static void write_bulk_callback(struct urb *urb)
{
        pegasus_t *pegasus = urb->context;
        struct net_device *net;
        int status = urb->status;

        if (!pegasus)
                return;

        net = pegasus->net;

        if (!netif_device_present(net) || !netif_running(net))
                return;

        switch (status) {
        case -EPIPE:
                /* FIXME schedule_work() to clear the tx halt */
                netif_stop_queue(net);
                netif_warn(pegasus, tx_err, net, "no tx stall recovery\n");
                return;
        case -ENOENT:
        case -ECONNRESET:
        case -ESHUTDOWN:
                netif_dbg(pegasus, ifdown, net, "tx unlink, %d\n", status);
                return;
        default:
                netif_info(pegasus, tx_err, net, "TX status %d\n", status);
                fallthrough;
        case 0:
                break;
        }

        netif_trans_update(net); /* prevent tx timeout */
        netif_wake_queue(net);
}

static void intr_callback(struct urb *urb)
{
        pegasus_t *pegasus = urb->context;
        struct net_device *net;
        int res, status = urb->status;

        if (!pegasus)
                return;
        net = pegasus->net;

        switch (status) {
        case 0:
                break;
        case -ECONNRESET:        /* unlink */
        case -ENOENT:
        case -ESHUTDOWN:
                return;
        default:
                /* some Pegasus-I products report LOTS of data
                 * toggle errors... avoid log spamming
                 */
                netif_dbg(pegasus, timer, net, "intr status %d\n", status);
        }

        if (urb->actual_length >= 6) {
                u8 *d = urb->transfer_buffer;

                /* byte 0 == tx_status1, reg 2B */
                if (d[0] & (TX_UNDERRUN|EXCESSIVE_COL
                                        |LATE_COL|JABBER_TIMEOUT)) {
                        net->stats.tx_errors++;
                        if (d[0] & TX_UNDERRUN)
                                net->stats.tx_fifo_errors++;
                        if (d[0] & (EXCESSIVE_COL | JABBER_TIMEOUT))
                                net->stats.tx_aborted_errors++;
                        if (d[0] & LATE_COL)
                                net->stats.tx_window_errors++;
                }

                /* d[5].LINK_STATUS lies on some adapters.
                 * d[0].NO_CARRIER kicks in only with failed TX.
                 * ... so monitoring with MII may be safest.
                 */

                /* bytes 3-4 == rx_lostpkt, reg 2E/2F */
                net->stats.rx_missed_errors += ((d[3] & 0x7f) << 8) | d[4];
        }

        res = usb_submit_urb(urb, GFP_ATOMIC);
        if (res == -ENODEV)
                netif_device_detach(pegasus->net);
        if (res)
                netif_err(pegasus, timer, net,
                          "can't resubmit interrupt urb, %d\n", res);
}

static void pegasus_tx_timeout(struct net_device *net, unsigned int txqueue)
{
        pegasus_t *pegasus = netdev_priv(net);
        netif_warn(pegasus, timer, net, "tx timeout\n");
        usb_unlink_urb(pegasus->tx_urb);
        net->stats.tx_errors++;
}

static netdev_tx_t pegasus_start_xmit(struct sk_buff *skb,
                                            struct net_device *net)
{
        pegasus_t *pegasus = netdev_priv(net);
        int count = ((skb->len + 2) & 0x3f) ? skb->len + 2 : skb->len + 3;
        int res;
        __u16 l16 = skb->len;

        netif_stop_queue(net);

        ((__le16 *) pegasus->tx_buff)[0] = cpu_to_le16(l16);
        skb_copy_from_linear_data(skb, pegasus->tx_buff + 2, skb->len);
        usb_fill_bulk_urb(pegasus->tx_urb, pegasus->usb,
                          usb_sndbulkpipe(pegasus->usb, 2),
                          pegasus->tx_buff, count,
                          write_bulk_callback, pegasus);
        if ((res = usb_submit_urb(pegasus->tx_urb, GFP_ATOMIC))) {
                netif_warn(pegasus, tx_err, net, "fail tx, %d\n", res);
                switch (res) {
                case -EPIPE:                /* stall, or disconnect from TT */
                        /* cleanup should already have been scheduled */
                        break;
                case -ENODEV:                /* disconnect() upcoming */
                case -EPERM:
                        netif_device_detach(pegasus->net);
                        break;
                default:
                        net->stats.tx_errors++;
                        netif_start_queue(net);
                }
        } else {
                net->stats.tx_packets++;
                net->stats.tx_bytes += skb->len;
        }
        dev_kfree_skb(skb);

        return NETDEV_TX_OK;
}

static inline void disable_net_traffic(pegasus_t *pegasus)
{
        __le16 tmp = cpu_to_le16(0);

        set_registers(pegasus, EthCtrl0, sizeof(tmp), &tmp);
}

static inline int get_interrupt_interval(pegasus_t *pegasus)
{
        u16 data;
        u8 interval;
        int ret;

        ret = read_eprom_word(pegasus, 4, &data);
        if (ret < 0)
                return ret;

        interval = data >> 8;
        if (pegasus->usb->speed != USB_SPEED_HIGH) {
                if (interval < 0x80) {
                        netif_info(pegasus, timer, pegasus->net,
                                   "intr interval changed from %ums to %ums\n",
                                   interval, 0x80);
                        interval = 0x80;
                        data = (data & 0x00FF) | ((u16)interval << 8);
#ifdef PEGASUS_WRITE_EEPROM
                        write_eprom_word(pegasus, 4, data);
#endif
                }
        }
        pegasus->intr_interval = interval;

        return 0;
}

static void set_carrier(struct net_device *net)
{
        pegasus_t *pegasus = netdev_priv(net);
        u16 tmp;

        if (read_mii_word(pegasus, pegasus->phy, MII_BMSR, &tmp))
                return;

        if (tmp & BMSR_LSTATUS)
                netif_carrier_on(net);
        else
                netif_carrier_off(net);
}

static void free_all_urbs(pegasus_t *pegasus)
{
        usb_free_urb(pegasus->intr_urb);
        usb_free_urb(pegasus->tx_urb);
        usb_free_urb(pegasus->rx_urb);
}

static void unlink_all_urbs(pegasus_t *pegasus)
{
        usb_kill_urb(pegasus->intr_urb);
        usb_kill_urb(pegasus->tx_urb);
        usb_kill_urb(pegasus->rx_urb);
}

static int alloc_urbs(pegasus_t *pegasus)
{
        int res = -ENOMEM;

        pegasus->rx_urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!pegasus->rx_urb) {
                return res;
        }
        pegasus->tx_urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!pegasus->tx_urb) {
                usb_free_urb(pegasus->rx_urb);
                return res;
        }
        pegasus->intr_urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!pegasus->intr_urb) {
                usb_free_urb(pegasus->tx_urb);
                usb_free_urb(pegasus->rx_urb);
                return res;
        }

        return 0;
}

static int pegasus_open(struct net_device *net)
{
        pegasus_t *pegasus = netdev_priv(net);
        int res=-ENOMEM;

        if (pegasus->rx_skb == NULL)
                pegasus->rx_skb = __netdev_alloc_skb_ip_align(pegasus->net,
                                                              PEGASUS_MTU,
                                                              GFP_KERNEL);
        if (!pegasus->rx_skb)
                goto exit;

        set_registers(pegasus, EthID, 6, net->dev_addr);

        usb_fill_bulk_urb(pegasus->rx_urb, pegasus->usb,
                          usb_rcvbulkpipe(pegasus->usb, 1),
                          pegasus->rx_skb->data, PEGASUS_MTU,
                          read_bulk_callback, pegasus);
        if ((res = usb_submit_urb(pegasus->rx_urb, GFP_KERNEL))) {
                if (res == -ENODEV)
                        netif_device_detach(pegasus->net);
                netif_dbg(pegasus, ifup, net, "failed rx_urb, %d\n", res);
                goto exit;
        }

        usb_fill_int_urb(pegasus->intr_urb, pegasus->usb,
                         usb_rcvintpipe(pegasus->usb, 3),
                         pegasus->intr_buff, sizeof(pegasus->intr_buff),
                         intr_callback, pegasus, pegasus->intr_interval);
        if ((res = usb_submit_urb(pegasus->intr_urb, GFP_KERNEL))) {
                if (res == -ENODEV)
                        netif_device_detach(pegasus->net);
                netif_dbg(pegasus, ifup, net, "failed intr_urb, %d\n", res);
                usb_kill_urb(pegasus->rx_urb);
                goto exit;
        }
        res = enable_net_traffic(net, pegasus->usb);
        if (res < 0) {
                netif_dbg(pegasus, ifup, net,
                          "can't enable_net_traffic() - %d\n", res);
                res = -EIO;
                usb_kill_urb(pegasus->rx_urb);
                usb_kill_urb(pegasus->intr_urb);
                goto exit;
        }
        set_carrier(net);
        netif_start_queue(net);
        netif_dbg(pegasus, ifup, net, "open\n");
        res = 0;
exit:
        return res;
}

static int pegasus_close(struct net_device *net)
{
        pegasus_t *pegasus = netdev_priv(net);

        netif_stop_queue(net);
        if (!(pegasus->flags & PEGASUS_UNPLUG))
                disable_net_traffic(pegasus);
        tasklet_kill(&pegasus->rx_tl);
        unlink_all_urbs(pegasus);

        return 0;
}

static void pegasus_get_drvinfo(struct net_device *dev,
                                struct ethtool_drvinfo *info)
{
        pegasus_t *pegasus = netdev_priv(dev);

        strscpy(info->driver, driver_name, sizeof(info->driver));
        usb_make_path(pegasus->usb, info->bus_info, sizeof(info->bus_info));
}

/* also handles three patterns of some kind in hardware */
#define        WOL_SUPPORTED        (WAKE_MAGIC|WAKE_PHY)

static void
pegasus_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
{
        pegasus_t        *pegasus = netdev_priv(dev);

        wol->supported = WAKE_MAGIC | WAKE_PHY;
        wol->wolopts = pegasus->wolopts;
}

static int
pegasus_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
{
        pegasus_t        *pegasus = netdev_priv(dev);
        u8                reg78 = 0x04;
        int                ret;

        if (wol->wolopts & ~WOL_SUPPORTED)
                return -EINVAL;

        if (wol->wolopts & WAKE_MAGIC)
                reg78 |= 0x80;
        if (wol->wolopts & WAKE_PHY)
                reg78 |= 0x40;
        /* FIXME this 0x10 bit still needs to get set in the chip... */
        if (wol->wolopts)
                pegasus->eth_regs[0] |= 0x10;
        else
                pegasus->eth_regs[0] &= ~0x10;
        pegasus->wolopts = wol->wolopts;

        ret = set_register(pegasus, WakeupControl, reg78);
        if (!ret)
                ret = device_set_wakeup_enable(&pegasus->usb->dev,
                                                wol->wolopts);
        return ret;
}

static inline void pegasus_reset_wol(struct net_device *dev)
{
        struct ethtool_wolinfo wol;

        memset(&wol, 0, sizeof wol);
        (void) pegasus_set_wol(dev, &wol);
}

static int
pegasus_get_link_ksettings(struct net_device *dev,
                           struct ethtool_link_ksettings *ecmd)
{
        pegasus_t *pegasus;

        pegasus = netdev_priv(dev);
        mii_ethtool_get_link_ksettings(&pegasus->mii, ecmd);
        return 0;
}

static int
pegasus_set_link_ksettings(struct net_device *dev,
                           const struct ethtool_link_ksettings *ecmd)
{
        pegasus_t *pegasus = netdev_priv(dev);
        return mii_ethtool_set_link_ksettings(&pegasus->mii, ecmd);
}

static int pegasus_nway_reset(struct net_device *dev)
{
        pegasus_t *pegasus = netdev_priv(dev);
        return mii_nway_restart(&pegasus->mii);
}

static u32 pegasus_get_link(struct net_device *dev)
{
        pegasus_t *pegasus = netdev_priv(dev);
        return mii_link_ok(&pegasus->mii);
}

static u32 pegasus_get_msglevel(struct net_device *dev)
{
        pegasus_t *pegasus = netdev_priv(dev);
        return pegasus->msg_enable;
}

static void pegasus_set_msglevel(struct net_device *dev, u32 v)
{
        pegasus_t *pegasus = netdev_priv(dev);
        pegasus->msg_enable = v;
}

static const struct ethtool_ops ops = {
        .get_drvinfo = pegasus_get_drvinfo,
        .nway_reset = pegasus_nway_reset,
        .get_link = pegasus_get_link,
        .get_msglevel = pegasus_get_msglevel,
        .set_msglevel = pegasus_set_msglevel,
        .get_wol = pegasus_get_wol,
        .set_wol = pegasus_set_wol,
        .get_link_ksettings = pegasus_get_link_ksettings,
        .set_link_ksettings = pegasus_set_link_ksettings,
};

static int pegasus_siocdevprivate(struct net_device *net, struct ifreq *rq,
                                  void __user *udata, int cmd)
{
        __u16 *data = (__u16 *) &rq->ifr_ifru;
        pegasus_t *pegasus = netdev_priv(net);
        int res;

        switch (cmd) {
        case SIOCDEVPRIVATE:
                data[0] = pegasus->phy;
                fallthrough;
        case SIOCDEVPRIVATE + 1:
                res = read_mii_word(pegasus, data[0], data[1] & 0x1f, &data[3]);
                break;
        case SIOCDEVPRIVATE + 2:
                if (!capable(CAP_NET_ADMIN))
                        return -EPERM;
                write_mii_word(pegasus, pegasus->phy, data[1] & 0x1f, &data[2]);
                res = 0;
                break;
        default:
                res = -EOPNOTSUPP;
        }
        return res;
}

static void pegasus_set_multicast(struct net_device *net)
{
        pegasus_t *pegasus = netdev_priv(net);

        if (net->flags & IFF_PROMISC) {
                pegasus->eth_regs[EthCtrl2] |= RX_PROMISCUOUS;
                netif_info(pegasus, link, net, "Promiscuous mode enabled\n");
        } else if (!netdev_mc_empty(net) || (net->flags & IFF_ALLMULTI)) {
                pegasus->eth_regs[EthCtrl0] |= RX_MULTICAST;
                pegasus->eth_regs[EthCtrl2] &= ~RX_PROMISCUOUS;
                netif_dbg(pegasus, link, net, "set allmulti\n");
        } else {
                pegasus->eth_regs[EthCtrl0] &= ~RX_MULTICAST;
                pegasus->eth_regs[EthCtrl2] &= ~RX_PROMISCUOUS;
        }
        update_eth_regs_async(pegasus);
}

static __u8 mii_phy_probe(pegasus_t *pegasus)
{
        int i, ret;
        __u16 tmp;

        for (i = 0; i < 32; i++) {
                ret = read_mii_word(pegasus, i, MII_BMSR, &tmp);
                if (ret < 0)
                        goto fail;
                if (tmp == 0 || tmp == 0xffff || (tmp & BMSR_MEDIA) == 0)
                        continue;
                else
                        return i;
        }
fail:
        return 0xff;
}

static inline void setup_pegasus_II(pegasus_t *pegasus)
{
        int ret;
        __u8 data = 0xa5;

        set_register(pegasus, Reg1d, 0);
        set_register(pegasus, Reg7b, 1);
        msleep(100);
        if ((pegasus->features & HAS_HOME_PNA) && mii_mode)
                set_register(pegasus, Reg7b, 0);
        else
                set_register(pegasus, Reg7b, 2);

        set_register(pegasus, 0x83, data);
        ret = get_registers(pegasus, 0x83, 1, &data);
        if (ret < 0)
                goto fail;

        if (data == 0xa5)
                pegasus->chip = 0x8513;
        else
                pegasus->chip = 0;

        set_register(pegasus, 0x80, 0xc0);
        set_register(pegasus, 0x83, 0xff);
        set_register(pegasus, 0x84, 0x01);

        if (pegasus->features & HAS_HOME_PNA && mii_mode)
                set_register(pegasus, Reg81, 6);
        else
                set_register(pegasus, Reg81, 2);

        return;
fail:
        netif_dbg(pegasus, drv, pegasus->net, "%s failed\n", __func__);
}

static void check_carrier(struct work_struct *work)
{
        pegasus_t *pegasus = container_of(work, pegasus_t, carrier_check.work);
        set_carrier(pegasus->net);
        if (!(pegasus->flags & PEGASUS_UNPLUG)) {
                queue_delayed_work(system_long_wq, &pegasus->carrier_check,
                        CARRIER_CHECK_DELAY);
        }
}

static int pegasus_blacklisted(struct usb_device *udev)
{
        struct usb_device_descriptor *udd = &udev->descriptor;

        /* Special quirk to keep the driver from handling the Belkin Bluetooth
         * dongle which happens to have the same ID.
         */
        if ((udd->idVendor == cpu_to_le16(VENDOR_BELKIN)) &&
            (udd->idProduct == cpu_to_le16(0x0121)) &&
            (udd->bDeviceClass == USB_CLASS_WIRELESS_CONTROLLER) &&
            (udd->bDeviceProtocol == 1))
                return 1;

        return 0;
}

static int pegasus_probe(struct usb_interface *intf,
                         const struct usb_device_id *id)
{
        struct usb_device *dev = interface_to_usbdev(intf);
        struct net_device *net;
        pegasus_t *pegasus;
        int dev_index = id - pegasus_ids;
        int res = -ENOMEM;

        if (pegasus_blacklisted(dev))
                return -ENODEV;

        net = alloc_etherdev(sizeof(struct pegasus));
        if (!net)
                goto out;

        pegasus = netdev_priv(net);
        pegasus->dev_index = dev_index;

        res = alloc_urbs(pegasus);
        if (res < 0) {
                dev_err(&intf->dev, "can't allocate %s\n", "urbs");
                goto out1;
        }

        tasklet_setup(&pegasus->rx_tl, rx_fixup);

        INIT_DELAYED_WORK(&pegasus->carrier_check, check_carrier);

        pegasus->intf = intf;
        pegasus->usb = dev;
        pegasus->net = net;


        net->watchdog_timeo = PEGASUS_TX_TIMEOUT;
        net->netdev_ops = &pegasus_netdev_ops;
        net->ethtool_ops = &ops;
        pegasus->mii.dev = net;
        pegasus->mii.mdio_read = mdio_read;
        pegasus->mii.mdio_write = mdio_write;
        pegasus->mii.phy_id_mask = 0x1f;
        pegasus->mii.reg_num_mask = 0x1f;
        pegasus->msg_enable = netif_msg_init(msg_level, NETIF_MSG_DRV
                                | NETIF_MSG_PROBE | NETIF_MSG_LINK);

        pegasus->features = usb_dev_id[dev_index].private;
        res = get_interrupt_interval(pegasus);
        if (res)
                goto out2;
        if (reset_mac(pegasus)) {
                dev_err(&intf->dev, "can't reset MAC\n");
                res = -EIO;
                goto out2;
        }
        set_ethernet_addr(pegasus);
        if (pegasus->features & PEGASUS_II) {
                dev_info(&intf->dev, "setup Pegasus II specific registers\n");
                setup_pegasus_II(pegasus);
        }
        pegasus->phy = mii_phy_probe(pegasus);
        if (pegasus->phy == 0xff) {
                dev_warn(&intf->dev, "can't locate MII phy, using default\n");
                pegasus->phy = 1;
        }
        pegasus->mii.phy_id = pegasus->phy;
        usb_set_intfdata(intf, pegasus);
        SET_NETDEV_DEV(net, &intf->dev);
        pegasus_reset_wol(net);
        res = register_netdev(net);
        if (res)
                goto out3;
        queue_delayed_work(system_long_wq, &pegasus->carrier_check,
                           CARRIER_CHECK_DELAY);
        dev_info(&intf->dev, "%s, %s, %pM\n", net->name,
                 usb_dev_id[dev_index].name, net->dev_addr);
        return 0;

out3:
        usb_set_intfdata(intf, NULL);
out2:
        free_all_urbs(pegasus);
out1:
        free_netdev(net);
out:
        return res;
}

static void pegasus_disconnect(struct usb_interface *intf)
{
        struct pegasus *pegasus = usb_get_intfdata(intf);

        usb_set_intfdata(intf, NULL);
        if (!pegasus) {
                dev_dbg(&intf->dev, "unregistering non-bound device?\n");
                return;
        }

        pegasus->flags |= PEGASUS_UNPLUG;
        cancel_delayed_work_sync(&pegasus->carrier_check);
        unregister_netdev(pegasus->net);
        unlink_all_urbs(pegasus);
        free_all_urbs(pegasus);
        if (pegasus->rx_skb != NULL) {
                dev_kfree_skb(pegasus->rx_skb);
                pegasus->rx_skb = NULL;
        }
        free_netdev(pegasus->net);
}

static int pegasus_suspend(struct usb_interface *intf, pm_message_t message)
{
        struct pegasus *pegasus = usb_get_intfdata(intf);

        netif_device_detach(pegasus->net);
        cancel_delayed_work_sync(&pegasus->carrier_check);
        if (netif_running(pegasus->net)) {
                usb_kill_urb(pegasus->rx_urb);
                usb_kill_urb(pegasus->intr_urb);
        }
        return 0;
}

static int pegasus_resume(struct usb_interface *intf)
{
        struct pegasus *pegasus = usb_get_intfdata(intf);

        netif_device_attach(pegasus->net);
        if (netif_running(pegasus->net)) {
                pegasus->rx_urb->status = 0;
                pegasus->rx_urb->actual_length = 0;
                read_bulk_callback(pegasus->rx_urb);

                pegasus->intr_urb->status = 0;
                pegasus->intr_urb->actual_length = 0;
                intr_callback(pegasus->intr_urb);
        }
        queue_delayed_work(system_long_wq, &pegasus->carrier_check,
                                CARRIER_CHECK_DELAY);
        return 0;
}

static const struct net_device_ops pegasus_netdev_ops = {
        .ndo_open =                        pegasus_open,
        .ndo_stop =                        pegasus_close,
        .ndo_siocdevprivate =                pegasus_siocdevprivate,
        .ndo_start_xmit =                pegasus_start_xmit,
        .ndo_set_rx_mode =                pegasus_set_multicast,
        .ndo_tx_timeout =                pegasus_tx_timeout,
        .ndo_set_mac_address =                eth_mac_addr,
        .ndo_validate_addr =                eth_validate_addr,
};

static struct usb_driver pegasus_driver = {
        .name = driver_name,
        .probe = pegasus_probe,
        .disconnect = pegasus_disconnect,
        .id_table = pegasus_ids,
        .suspend = pegasus_suspend,
        .resume = pegasus_resume,
        .disable_hub_initiated_lpm = 1,
};

static void __init parse_id(char *id)
{
        unsigned int vendor_id = 0, device_id = 0, flags = 0, i = 0;
        char *token, *name = NULL;

        if ((token = strsep(&id, ":")) != NULL)
                name = token;
        /* name now points to a null terminated string*/
        if ((token = strsep(&id, ":")) != NULL)
                vendor_id = simple_strtoul(token, NULL, 16);
        if ((token = strsep(&id, ":")) != NULL)
                device_id = simple_strtoul(token, NULL, 16);
        flags = simple_strtoul(id, NULL, 16);
        pr_info("%s: new device %s, vendor ID 0x%04x, device ID 0x%04x, flags: 0x%x\n",
                driver_name, name, vendor_id, device_id, flags);

        if (vendor_id > 0x10000 || vendor_id == 0)
                return;
        if (device_id > 0x10000 || device_id == 0)
                return;

        for (i = 0; usb_dev_id[i].name; i++);
        usb_dev_id[i].name = name;
        usb_dev_id[i].vendor = vendor_id;
        usb_dev_id[i].device = device_id;
        usb_dev_id[i].private = flags;
        pegasus_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE;
        pegasus_ids[i].idVendor = vendor_id;
        pegasus_ids[i].idProduct = device_id;
}

static int __init pegasus_init(void)
{
        pr_info("%s: " DRIVER_DESC "\n", driver_name);
        if (devid)
                parse_id(devid);
        return usb_register(&pegasus_driver);
}

static void __exit pegasus_exit(void)
{
        usb_deregister(&pegasus_driver);
}

module_init(pegasus_init);
module_exit(pegasus_exit);



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    8 








    8 




    8 




   11 











   10 







   10 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/fs/vfat/namei.c
 *
 *  Written 1992,1993 by Werner Almesberger
 *
 *  Windows95/Windows NT compatible extended MSDOS filesystem
 *    by Gordon Chaffee Copyright (C) 1995.  Send bug reports for the
 *    VFAT filesystem to <chaffee@cs.berkeley.edu>.  Specify
 *    what file operation caused you trouble and if you can duplicate
 *    the problem, send a script that demonstrates it.
 *
 *  Short name translation 1999, 2001 by Wolfram Pienkoss <wp@bszh.de>
 *
 *  Support Multibyte characters and cleanup by
 *                                OGAWA Hirofumi <hirofumi@mail.parknet.co.jp>
 */

#include <linux/module.h>
#include <linux/ctype.h>
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/kernel.h>
#include <linux/iversion.h>
#include "fat.h"

static inline unsigned long vfat_d_version(struct dentry *dentry)
{
        return (unsigned long) dentry->d_fsdata;
}

static inline void vfat_d_version_set(struct dentry *dentry,
                                      unsigned long version)
{
        dentry->d_fsdata = (void *) version;
}

/*
 * If new entry was created in the parent, it could create the 8.3
 * alias (the shortname of logname).  So, the parent may have the
 * negative-dentry which matches the created 8.3 alias.
 *
 * If it happened, the negative dentry isn't actually negative
 * anymore.  So, drop it.
 */
static bool vfat_revalidate_shortname(struct dentry *dentry, struct inode *dir)
{
        return inode_eq_iversion(dir, vfat_d_version(dentry));
}

static int vfat_revalidate(struct inode *dir, const struct qstr *name,
                           struct dentry *dentry, unsigned int flags)
{
        if (flags & LOOKUP_RCU)
                return -ECHILD;

        /* This is not negative dentry. Always valid. */
        if (d_really_is_positive(dentry))
                return 1;
        return vfat_revalidate_shortname(dentry, dir);
}

static int vfat_revalidate_ci(struct inode *dir, const struct qstr *name,
                              struct dentry *dentry, unsigned int flags)
{
        if (flags & LOOKUP_RCU)
                return -ECHILD;

        /*
         * This is not negative dentry. Always valid.
         *
         * Note, rename() to existing directory entry will have ->d_inode,
         * and will use existing name which isn't specified name by user.
         *
         * We may be able to drop this positive dentry here. But dropping
         * positive dentry isn't good idea. So it's unsupported like
         * rename("filename", "FILENAME") for now.
         */
        if (d_really_is_positive(dentry))
                return 1;

        /*
         * This may be nfsd (or something), anyway, we can't see the
         * intent of this. So, since this can be for creation, drop it.
         */
        if (!flags)
                return 0;

        /*
         * Drop the negative dentry, in order to make sure to use the
         * case sensitive name which is specified by user if this is
         * for creation.
         */
        if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET))
                return 0;

        return vfat_revalidate_shortname(dentry, dir);
}

/* returns the length of a struct qstr, ignoring trailing dots */
static unsigned int __vfat_striptail_len(unsigned int len, const char *name)
{
        while (len && name[len - 1] == '.')
                len--;
        return len;
}

static unsigned int vfat_striptail_len(const struct qstr *qstr)
{
        return __vfat_striptail_len(qstr->len, qstr->name);
}

/*
 * Compute the hash for the vfat name corresponding to the dentry.
 * Note: if the name is invalid, we leave the hash code unchanged so
 * that the existing dentry can be used. The vfat fs routines will
 * return ENOENT or EINVAL as appropriate.
 */
static int vfat_hash(const struct dentry *dentry, struct qstr *qstr)
{
        qstr->hash = full_name_hash(dentry, qstr->name, vfat_striptail_len(qstr));
        return 0;
}

/*
 * Compute the hash for the vfat name corresponding to the dentry.
 * Note: if the name is invalid, we leave the hash code unchanged so
 * that the existing dentry can be used. The vfat fs routines will
 * return ENOENT or EINVAL as appropriate.
 */
static int vfat_hashi(const struct dentry *dentry, struct qstr *qstr)
{
        struct nls_table *t = MSDOS_SB(dentry->d_sb)->nls_io;
        const unsigned char *name;
        unsigned int len;
        unsigned long hash;

        name = qstr->name;
        len = vfat_striptail_len(qstr);

        hash = init_name_hash(dentry);
        while (len--)
                hash = partial_name_hash(nls_tolower(t, *name++), hash);
        qstr->hash = end_name_hash(hash);

        return 0;
}

/*
 * Case insensitive compare of two vfat names.
 */
static int vfat_cmpi(const struct dentry *dentry,
                unsigned int len, const char *str, const struct qstr *name)
{
        struct nls_table *t = MSDOS_SB(dentry->d_sb)->nls_io;
        unsigned int alen, blen;

        /* A filename cannot end in '.' or we treat it like it has none */
        alen = vfat_striptail_len(name);
        blen = __vfat_striptail_len(len, str);
        if (alen == blen) {
                if (nls_strnicmp(t, name->name, str, alen) == 0)
                        return 0;
        }
        return 1;
}

/*
 * Case sensitive compare of two vfat names.
 */
static int vfat_cmp(const struct dentry *dentry,
                unsigned int len, const char *str, const struct qstr *name)
{
        unsigned int alen, blen;

        /* A filename cannot end in '.' or we treat it like it has none */
        alen = vfat_striptail_len(name);
        blen = __vfat_striptail_len(len, str);
        if (alen == blen) {
                if (strncmp(name->name, str, alen) == 0)
                        return 0;
        }
        return 1;
}

static const struct dentry_operations vfat_ci_dentry_ops = {
        .d_revalidate        = vfat_revalidate_ci,
        .d_hash                = vfat_hashi,
        .d_compare        = vfat_cmpi,
};

static const struct dentry_operations vfat_dentry_ops = {
        .d_revalidate        = vfat_revalidate,
        .d_hash                = vfat_hash,
        .d_compare        = vfat_cmp,
};

/* Characters that are undesirable in an MS-DOS file name */

static inline bool vfat_bad_char(wchar_t w)
{
        return (w < 0x0020)
            || (w == '*') || (w == '?') || (w == '<') || (w == '>')
            || (w == '|') || (w == '"') || (w == ':') || (w == '/')
            || (w == '\\');
}

static inline bool vfat_replace_char(wchar_t w)
{
        return (w == '[') || (w == ']') || (w == ';') || (w == ',')
            || (w == '+') || (w == '=');
}

static wchar_t vfat_skip_char(wchar_t w)
{
        return (w == '.') || (w == ' ');
}

static inline int vfat_is_used_badchars(const wchar_t *s, int len)
{
        int i;

        for (i = 0; i < len; i++)
                if (vfat_bad_char(s[i]))
                        return -EINVAL;

        if (s[i - 1] == ' ') /* last character cannot be space */
                return -EINVAL;

        return 0;
}

static int vfat_find_form(struct inode *dir, unsigned char *name)
{
        struct fat_slot_info sinfo;
        int err = fat_scan(dir, name, &sinfo);
        if (err)
                return -ENOENT;
        brelse(sinfo.bh);
        return 0;
}

/*
 * 1) Valid characters for the 8.3 format alias are any combination of
 * letters, uppercase alphabets, digits, any of the
 * following special characters:
 *     $ % ' ` - @ { } ~ ! # ( ) & _ ^
 * In this case Longfilename is not stored in disk.
 *
 * WinNT's Extension:
 * File name and extension name is contain uppercase/lowercase
 * only. And it is expressed by CASE_LOWER_BASE and CASE_LOWER_EXT.
 *
 * 2) File name is 8.3 format, but it contain the uppercase and
 * lowercase char, muliti bytes char, etc. In this case numtail is not
 * added, but Longfilename is stored.
 *
 * 3) When the one except for the above, or the following special
 * character are contained:
 *        .   [ ] ; , + =
 * numtail is added, and Longfilename must be stored in disk .
 */
struct shortname_info {
        unsigned char lower:1,
                      upper:1,
                      valid:1;
};
#define INIT_SHORTNAME_INFO(x)        do {                \
        (x)->lower = 1;                                \
        (x)->upper = 1;                                \
        (x)->valid = 1;                                \
} while (0)

static inline int to_shortname_char(struct nls_table *nls,
                                    unsigned char *buf, int buf_size,
                                    wchar_t *src, struct shortname_info *info)
{
        int len;

        if (vfat_skip_char(*src)) {
                info->valid = 0;
                return 0;
        }
        if (vfat_replace_char(*src)) {
                info->valid = 0;
                buf[0] = '_';
                return 1;
        }

        len = nls->uni2char(*src, buf, buf_size);
        if (len <= 0) {
                info->valid = 0;
                buf[0] = '_';
                len = 1;
        } else if (len == 1) {
                unsigned char prev = buf[0];

                if (buf[0] >= 0x7F) {
                        info->lower = 0;
                        info->upper = 0;
                }

                buf[0] = nls_toupper(nls, buf[0]);
                if (isalpha(buf[0])) {
                        if (buf[0] == prev)
                                info->lower = 0;
                        else
                                info->upper = 0;
                }
        } else {
                info->lower = 0;
                info->upper = 0;
        }

        return len;
}

/*
 * Given a valid longname, create a unique shortname.  Make sure the
 * shortname does not exist
 * Returns negative number on error, 0 for a normal
 * return, and 1 for valid shortname
 */
static int vfat_create_shortname(struct inode *dir, struct nls_table *nls,
                                 wchar_t *uname, int ulen,
                                 unsigned char *name_res, unsigned char *lcase)
{
        struct fat_mount_options *opts = &MSDOS_SB(dir->i_sb)->options;
        wchar_t *ip, *ext_start, *end, *name_start;
        unsigned char base[9], ext[4], buf[5], *p;
        unsigned char charbuf[NLS_MAX_CHARSET_SIZE];
        int chl, chi;
        int sz = 0, extlen, baselen, i, numtail_baselen, numtail2_baselen;
        int is_shortname;
        struct shortname_info base_info, ext_info;

        is_shortname = 1;
        INIT_SHORTNAME_INFO(&base_info);
        INIT_SHORTNAME_INFO(&ext_info);

        /* Now, we need to create a shortname from the long name */
        ext_start = end = &uname[ulen];
        while (--ext_start >= uname) {
                if (*ext_start == 0x002E) {        /* is `.' */
                        if (ext_start == end - 1) {
                                sz = ulen;
                                ext_start = NULL;
                        }
                        break;
                }
        }

        if (ext_start == uname - 1) {
                sz = ulen;
                ext_start = NULL;
        } else if (ext_start) {
                /*
                 * Names which start with a dot could be just
                 * an extension eg. "...test".  In this case Win95
                 * uses the extension as the name and sets no extension.
                 */
                name_start = &uname[0];
                while (name_start < ext_start) {
                        if (!vfat_skip_char(*name_start))
                                break;
                        name_start++;
                }
                if (name_start != ext_start) {
                        sz = ext_start - uname;
                        ext_start++;
                } else {
                        sz = ulen;
                        ext_start = NULL;
                }
        }

        numtail_baselen = 6;
        numtail2_baselen = 2;
        for (baselen = i = 0, p = base, ip = uname; i < sz; i++, ip++) {
                chl = to_shortname_char(nls, charbuf, sizeof(charbuf),
                                        ip, &base_info);
                if (chl == 0)
                        continue;

                if (baselen < 2 && (baselen + chl) > 2)
                        numtail2_baselen = baselen;
                if (baselen < 6 && (baselen + chl) > 6)
                        numtail_baselen = baselen;
                for (chi = 0; chi < chl; chi++) {
                        *p++ = charbuf[chi];
                        baselen++;
                        if (baselen >= 8)
                                break;
                }
                if (baselen >= 8) {
                        if ((chi < chl - 1) || (ip + 1) - uname < sz)
                                is_shortname = 0;
                        break;
                }
        }
        if (baselen == 0) {
                return -EINVAL;
        }

        extlen = 0;
        if (ext_start) {
                for (p = ext, ip = ext_start; extlen < 3 && ip < end; ip++) {
                        chl = to_shortname_char(nls, charbuf, sizeof(charbuf),
                                                ip, &ext_info);
                        if (chl == 0)
                                continue;

                        if ((extlen + chl) > 3) {
                                is_shortname = 0;
                                break;
                        }
                        for (chi = 0; chi < chl; chi++) {
                                *p++ = charbuf[chi];
                                extlen++;
                        }
                        if (extlen >= 3) {
                                if (ip + 1 != end)
                                        is_shortname = 0;
                                break;
                        }
                }
        }
        ext[extlen] = '\0';
        base[baselen] = '\0';

        /* Yes, it can happen. ".\xe5" would do it. */
        if (base[0] == DELETED_FLAG)
                base[0] = 0x05;

        /* OK, at this point we know that base is not longer than 8 symbols,
         * ext is not longer than 3, base is nonempty, both don't contain
         * any bad symbols (lowercase transformed to uppercase).
         */

        memset(name_res, ' ', MSDOS_NAME);
        memcpy(name_res, base, baselen);
        memcpy(name_res + 8, ext, extlen);
        *lcase = 0;
        if (is_shortname && base_info.valid && ext_info.valid) {
                if (vfat_find_form(dir, name_res) == 0)
                        return -EEXIST;

                if (opts->shortname & VFAT_SFN_CREATE_WIN95) {
                        return (base_info.upper && ext_info.upper);
                } else if (opts->shortname & VFAT_SFN_CREATE_WINNT) {
                        if ((base_info.upper || base_info.lower) &&
                            (ext_info.upper || ext_info.lower)) {
                                if (!base_info.upper && base_info.lower)
                                        *lcase |= CASE_LOWER_BASE;
                                if (!ext_info.upper && ext_info.lower)
                                        *lcase |= CASE_LOWER_EXT;
                                return 1;
                        }
                        return 0;
                } else {
                        BUG();
                }
        }

        if (opts->numtail == 0)
                if (vfat_find_form(dir, name_res) < 0)
                        return 0;

        /*
         * Try to find a unique extension.  This used to
         * iterate through all possibilities sequentially,
         * but that gave extremely bad performance.  Windows
         * only tries a few cases before using random
         * values for part of the base.
         */

        if (baselen > 6) {
                baselen = numtail_baselen;
                name_res[7] = ' ';
        }
        name_res[baselen] = '~';
        for (i = 1; i < 10; i++) {
                name_res[baselen + 1] = i + '0';
                if (vfat_find_form(dir, name_res) < 0)
                        return 0;
        }

        i = jiffies;
        sz = (jiffies >> 16) & 0x7;
        if (baselen > 2) {
                baselen = numtail2_baselen;
                name_res[7] = ' ';
        }
        name_res[baselen + 4] = '~';
        name_res[baselen + 5] = '1' + sz;
        while (1) {
                snprintf(buf, sizeof(buf), "%04X", i & 0xffff);
                memcpy(&name_res[baselen], buf, 4);
                if (vfat_find_form(dir, name_res) < 0)
                        break;
                i -= 11;
        }
        return 0;
}

/* Translate a string, including coded sequences into Unicode */
static int
xlate_to_uni(const unsigned char *name, int len, unsigned char *outname,
             int *longlen, int *outlen, int escape, int utf8,
             struct nls_table *nls)
{
        const unsigned char *ip;
        unsigned char *op;
        int i, fill;
        int charlen;

        if (utf8) {
                *outlen = utf8s_to_utf16s(name, len, UTF16_HOST_ENDIAN,
                                (wchar_t *) outname, FAT_LFN_LEN + 2);
                if (*outlen < 0)
                        return *outlen;
                else if (*outlen > FAT_LFN_LEN)
                        return -ENAMETOOLONG;

                op = &outname[*outlen * sizeof(wchar_t)];
        } else {
                for (i = 0, ip = name, op = outname, *outlen = 0;
                         i < len && *outlen < FAT_LFN_LEN;
                         *outlen += 1) {
                        if (escape && (*ip == ':')) {
                                u8 uc[2];

                                if (i > len - 5)
                                        return -EINVAL;

                                if (hex2bin(uc, ip + 1, 2) < 0)
                                        return -EINVAL;

                                *(wchar_t *)op = uc[0] << 8 | uc[1];

                                op += 2;
                                ip += 5;
                                i += 5;
                        } else {
                                charlen = nls->char2uni(ip, len - i,
                                                        (wchar_t *)op);
                                if (charlen < 0)
                                        return -EINVAL;
                                ip += charlen;
                                i += charlen;
                                op += 2;
                        }
                }
                if (i < len)
                        return -ENAMETOOLONG;
        }

        *longlen = *outlen;
        if (*outlen % 13) {
                *op++ = 0;
                *op++ = 0;
                *outlen += 1;
                if (*outlen % 13) {
                        fill = 13 - (*outlen % 13);
                        for (i = 0; i < fill; i++) {
                                *op++ = 0xff;
                                *op++ = 0xff;
                        }
                        *outlen += fill;
                }
        }

        return 0;
}

static int vfat_build_slots(struct inode *dir, const unsigned char *name,
                            int len, int is_dir, int cluster,
                            struct timespec64 *ts,
                            struct msdos_dir_slot *slots, int *nr_slots)
{
        struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb);
        struct fat_mount_options *opts = &sbi->options;
        struct msdos_dir_slot *ps;
        struct msdos_dir_entry *de;
        unsigned char cksum, lcase;
        unsigned char msdos_name[MSDOS_NAME];
        wchar_t *uname;
        __le16 time, date;
        u8 time_cs;
        int err, ulen, usize, i;
        loff_t offset;

        *nr_slots = 0;

        uname = __getname();
        if (!uname)
                return -ENOMEM;

        err = xlate_to_uni(name, len, (unsigned char *)uname, &ulen, &usize,
                           opts->unicode_xlate, opts->utf8, sbi->nls_io);
        if (err)
                goto out_free;

        err = vfat_is_used_badchars(uname, ulen);
        if (err)
                goto out_free;

        err = vfat_create_shortname(dir, sbi->nls_disk, uname, ulen,
                                    msdos_name, &lcase);
        if (err < 0)
                goto out_free;
        else if (err == 1) {
                de = (struct msdos_dir_entry *)slots;
                err = 0;
                goto shortname;
        }

        /* build the entry of long file name */
        cksum = fat_checksum(msdos_name);

        *nr_slots = usize / 13;
        for (ps = slots, i = *nr_slots; i > 0; i--, ps++) {
                ps->id = i;
                ps->attr = ATTR_EXT;
                ps->reserved = 0;
                ps->alias_checksum = cksum;
                ps->start = 0;
                offset = (i - 1) * 13;
                fatwchar_to16(ps->name0_4, uname + offset, 5);
                fatwchar_to16(ps->name5_10, uname + offset + 5, 6);
                fatwchar_to16(ps->name11_12, uname + offset + 11, 2);
        }
        slots[0].id |= 0x40;
        de = (struct msdos_dir_entry *)ps;

shortname:
        /* build the entry of 8.3 alias name */
        (*nr_slots)++;
        memcpy(de->name, msdos_name, MSDOS_NAME);
        de->attr = is_dir ? ATTR_DIR : ATTR_ARCH;
        de->lcase = lcase;
        fat_time_unix2fat(sbi, ts, &time, &date, &time_cs);
        de->time = de->ctime = time;
        de->date = de->cdate = de->adate = date;
        de->ctime_cs = time_cs;
        fat_set_start(de, cluster);
        de->size = 0;
out_free:
        __putname(uname);
        return err;
}

static int vfat_add_entry(struct inode *dir, const struct qstr *qname,
                          int is_dir, int cluster, struct timespec64 *ts,
                          struct fat_slot_info *sinfo)
{
        struct msdos_dir_slot *slots;
        unsigned int len;
        int err, nr_slots;

        len = vfat_striptail_len(qname);
        if (len == 0)
                return -ENOENT;

        slots = kmalloc_array(MSDOS_SLOTS, sizeof(*slots), GFP_NOFS);
        if (slots == NULL)
                return -ENOMEM;

        err = vfat_build_slots(dir, qname->name, len, is_dir, cluster, ts,
                               slots, &nr_slots);
        if (err)
                goto cleanup;

        err = fat_add_entries(dir, slots, nr_slots, sinfo);
        if (err)
                goto cleanup;

        /* update timestamp */
        fat_truncate_time(dir, ts, S_CTIME|S_MTIME);
        if (IS_DIRSYNC(dir))
                (void)fat_sync_inode(dir);
        else
                mark_inode_dirty(dir);
cleanup:
        kfree(slots);
        return err;
}

static int vfat_find(struct inode *dir, const struct qstr *qname,
                     struct fat_slot_info *sinfo)
{
        unsigned int len = vfat_striptail_len(qname);
        if (len == 0)
                return -ENOENT;
        return fat_search_long(dir, qname->name, len, sinfo);
}

static struct dentry *vfat_lookup(struct inode *dir, struct dentry *dentry,
                                  unsigned int flags)
{
        struct super_block *sb = dir->i_sb;
        struct fat_slot_info sinfo;
        struct inode *inode;
        struct dentry *alias;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = vfat_find(dir, &dentry->d_name, &sinfo);
        if (err) {
                if (err == -ENOENT) {
                        inode = NULL;
                        goto out;
                }
                goto error;
        }

        inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
        brelse(sinfo.bh);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                goto error;
        }

        alias = d_find_alias(inode);
        /*
         * Checking "alias->d_parent == dentry->d_parent" to make sure
         * FS is not corrupted (especially double linked dir).
         */
        if (alias && alias->d_parent == dentry->d_parent) {
                /*
                 * This inode has non anonymous-DCACHE_DISCONNECTED
                 * dentry. This means, the user did ->lookup() by an
                 * another name (longname vs 8.3 alias of it) in past.
                 *
                 * Switch to new one for reason of locality if possible.
                 */
                if (!S_ISDIR(inode->i_mode))
                        d_move(alias, dentry);
                iput(inode);
                mutex_unlock(&MSDOS_SB(sb)->s_lock);
                return alias;
        } else
                dput(alias);

out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        if (!inode)
                vfat_d_version_set(dentry, inode_query_iversion(dir));
        return d_splice_alias(inode, dentry);
error:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return ERR_PTR(err);
}

static int vfat_create(struct mnt_idmap *idmap, struct inode *dir,
                       struct dentry *dentry, umode_t mode, bool excl)
{
        struct super_block *sb = dir->i_sb;
        struct inode *inode;
        struct fat_slot_info sinfo;
        struct timespec64 ts;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        ts = current_time(dir);
        err = vfat_add_entry(dir, &dentry->d_name, 0, 0, &ts, &sinfo);
        if (err)
                goto out;
        inode_inc_iversion(dir);

        inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
        brelse(sinfo.bh);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                goto out;
        }
        inode_inc_iversion(inode);

        d_instantiate(dentry, inode);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return err;
}

static int vfat_rmdir(struct inode *dir, struct dentry *dentry)
{
        struct inode *inode = d_inode(dentry);
        struct super_block *sb = dir->i_sb;
        struct fat_slot_info sinfo;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = fat_dir_empty(inode);
        if (err)
                goto out;
        err = vfat_find(dir, &dentry->d_name, &sinfo);
        if (err)
                goto out;

        err = fat_remove_entries(dir, &sinfo);        /* and releases bh */
        if (err)
                goto out;
        drop_nlink(dir);

        clear_nlink(inode);
        fat_truncate_time(inode, NULL, S_ATIME|S_MTIME);
        fat_detach(inode);
        vfat_d_version_set(dentry, inode_query_iversion(dir));
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);

        return err;
}

static int vfat_unlink(struct inode *dir, struct dentry *dentry)
{
        struct inode *inode = d_inode(dentry);
        struct super_block *sb = dir->i_sb;
        struct fat_slot_info sinfo;
        int err;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        err = vfat_find(dir, &dentry->d_name, &sinfo);
        if (err)
                goto out;

        err = fat_remove_entries(dir, &sinfo);        /* and releases bh */
        if (err)
                goto out;
        clear_nlink(inode);
        fat_truncate_time(inode, NULL, S_ATIME|S_MTIME);
        fat_detach(inode);
        vfat_d_version_set(dentry, inode_query_iversion(dir));
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);

        return err;
}

static int vfat_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                      struct dentry *dentry, umode_t mode)
{
        struct super_block *sb = dir->i_sb;
        struct inode *inode;
        struct fat_slot_info sinfo;
        struct timespec64 ts;
        int err, cluster;

        mutex_lock(&MSDOS_SB(sb)->s_lock);

        ts = current_time(dir);
        cluster = fat_alloc_new_dir(dir, &ts);
        if (cluster < 0) {
                err = cluster;
                goto out;
        }
        err = vfat_add_entry(dir, &dentry->d_name, 1, cluster, &ts, &sinfo);
        if (err)
                goto out_free;
        inode_inc_iversion(dir);
        inc_nlink(dir);

        inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos);
        brelse(sinfo.bh);
        if (IS_ERR(inode)) {
                err = PTR_ERR(inode);
                /* the directory was completed, just return a error */
                goto out;
        }
        inode_inc_iversion(inode);
        set_nlink(inode, 2);

        d_instantiate(dentry, inode);

        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return 0;

out_free:
        fat_free_clusters(dir, cluster);
out:
        mutex_unlock(&MSDOS_SB(sb)->s_lock);
        return err;
}

static int vfat_get_dotdot_de(struct inode *inode, struct buffer_head **bh,
                              struct msdos_dir_entry **de)
{
        if (S_ISDIR(inode->i_mode)) {
                if (fat_get_dotdot_entry(inode, bh, de))
                        return -EIO;
        }
        return 0;
}

static int vfat_sync_ipos(struct inode *dir, struct inode *inode)
{
        if (IS_DIRSYNC(dir))
                return fat_sync_inode(inode);
        mark_inode_dirty(inode);
        return 0;
}

static int vfat_update_dotdot_de(struct inode *dir, struct inode *inode,
                                 struct buffer_head *dotdot_bh,
                                 struct msdos_dir_entry *dotdot_de)
{
        fat_set_start(dotdot_de, MSDOS_I(dir)->i_logstart);
        mark_buffer_dirty_inode(dotdot_bh, inode);
        if (IS_DIRSYNC(dir))
                return sync_dirty_buffer(dotdot_bh);
        return 0;
}

static void vfat_update_dir_metadata(struct inode *dir, struct timespec64 *ts)
{
        inode_inc_iversion(dir);
        fat_truncate_time(dir, ts, S_CTIME | S_MTIME);
        if (IS_DIRSYNC(dir))
                (void)fat_sync_inode(dir);
        else
                mark_inode_dirty(dir);
}

static int vfat_rename(struct inode *old_dir, struct dentry *old_dentry,
                       struct inode *new_dir, struct dentry *new_dentry)
{
        struct buffer_head *dotdot_bh;
        struct msdos_dir_entry *dotdot_de = NULL;
        struct inode *old_inode, *new_inode;
        struct fat_slot_info old_sinfo, sinfo;
        struct timespec64 ts;
        loff_t new_i_pos;
        int err, is_dir, corrupt = 0;
        struct super_block *sb = old_dir->i_sb;

        old_sinfo.bh = sinfo.bh = dotdot_bh = NULL;
        old_inode = d_inode(old_dentry);
        new_inode = d_inode(new_dentry);
        mutex_lock(&MSDOS_SB(sb)->s_lock);
        err = vfat_find(old_dir, &old_dentry->d_name, &old_sinfo);
        if (err)
                goto out;

        if (old_dir != new_dir) {
                err = vfat_get_dotdot_de(old_inode, &dotdot_bh, &dotdot_de);
                if (err)
                        goto out;
        }

        is_dir = S_ISDIR(old_inode->i_mode);
        ts = current_time(old_dir);
        if (new_inode) {
                if (is_dir) {
                        err = fat_dir_empty(new_inode);
                        if (err)
                                goto out;
                }
                new_i_pos = MSDOS_I(new_inode)->i_pos;
                fat_detach(new_inode);
        } else {
                err = vfat_add_entry(new_dir, &new_dentry->d_name, is_dir, 0,
                                     &ts, &sinfo);
                if (err)
                        goto out;
                new_i_pos = sinfo.i_pos;
        }
        inode_inc_iversion(new_dir);

        fat_detach(old_inode);
        fat_attach(old_inode, new_i_pos);
        err = vfat_sync_ipos(new_dir, old_inode);
        if (err)
                goto error_inode;

        if (dotdot_de) {
                err = vfat_update_dotdot_de(new_dir, old_inode, dotdot_bh,
                                            dotdot_de);
                if (err)
                        goto error_dotdot;
                drop_nlink(old_dir);
                if (!new_inode)
                         inc_nlink(new_dir);
        }

        err = fat_remove_entries(old_dir, &old_sinfo);        /* and releases bh */
        old_sinfo.bh = NULL;
        if (err)
                goto error_dotdot;
        vfat_update_dir_metadata(old_dir, &ts);

        if (new_inode) {
                drop_nlink(new_inode);
                if (is_dir)
                        drop_nlink(new_inode);
                fat_truncate_time(new_inode, &ts, S_CTIME);
        }
out:
        brelse(sinfo.bh);
        brelse(dotdot_bh);
        brelse(old_sinfo.bh);
        mutex_unlock(&MSDOS_SB(sb)->s_lock);

        return err;

error_dotdot:
        /* data cluster is shared, serious corruption */
        corrupt = 1;

        if (dotdot_de) {
                corrupt |= vfat_update_dotdot_de(old_dir, old_inode, dotdot_bh,
                                                 dotdot_de);
        }
error_inode:
        fat_detach(old_inode);
        fat_attach(old_inode, old_sinfo.i_pos);
        if (new_inode) {
                fat_attach(new_inode, new_i_pos);
                if (corrupt)
                        corrupt |= fat_sync_inode(new_inode);
        } else {
                /*
                 * If new entry was not sharing the data cluster, it
                 * shouldn't be serious corruption.
                 */
                int err2 = fat_remove_entries(new_dir, &sinfo);
                if (corrupt)
                        corrupt |= err2;
                sinfo.bh = NULL;
        }
        if (corrupt < 0) {
                fat_fs_error(new_dir->i_sb,
                             "%s: Filesystem corrupted (i_pos %lld)",
                             __func__, new_i_pos);
        }
        goto out;
}

static void vfat_exchange_ipos(struct inode *old_inode, struct inode *new_inode,
                               loff_t old_i_pos, loff_t new_i_pos)
{
        fat_detach(old_inode);
        fat_detach(new_inode);
        fat_attach(old_inode, new_i_pos);
        fat_attach(new_inode, old_i_pos);
}

static void vfat_move_nlink(struct inode *src, struct inode *dst)
{
        drop_nlink(src);
        inc_nlink(dst);
}

static int vfat_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
                                struct inode *new_dir, struct dentry *new_dentry)
{
        struct buffer_head *old_dotdot_bh = NULL, *new_dotdot_bh = NULL;
        struct msdos_dir_entry *old_dotdot_de = NULL, *new_dotdot_de = NULL;
        struct inode *old_inode, *new_inode;
        struct timespec64 ts = current_time(old_dir);
        loff_t old_i_pos, new_i_pos;
        int err, corrupt = 0;
        struct super_block *sb = old_dir->i_sb;

        old_inode = d_inode(old_dentry);
        new_inode = d_inode(new_dentry);

        /* Acquire super block lock for the operation to be atomic */
        mutex_lock(&MSDOS_SB(sb)->s_lock);

        /* if directories are not the same, get ".." info to update */
        if (old_dir != new_dir) {
                err = vfat_get_dotdot_de(old_inode, &old_dotdot_bh,
                                         &old_dotdot_de);
                if (err)
                        goto out;

                err = vfat_get_dotdot_de(new_inode, &new_dotdot_bh,
                                         &new_dotdot_de);
                if (err)
                        goto out;
        }

        old_i_pos = MSDOS_I(old_inode)->i_pos;
        new_i_pos = MSDOS_I(new_inode)->i_pos;

        vfat_exchange_ipos(old_inode, new_inode, old_i_pos, new_i_pos);

        err = vfat_sync_ipos(old_dir, new_inode);
        if (err)
                goto error_exchange;
        err = vfat_sync_ipos(new_dir, old_inode);
        if (err)
                goto error_exchange;

        /* update ".." directory entry info */
        if (old_dotdot_de) {
                err = vfat_update_dotdot_de(new_dir, old_inode, old_dotdot_bh,
                                            old_dotdot_de);
                if (err)
                        goto error_old_dotdot;
        }
        if (new_dotdot_de) {
                err = vfat_update_dotdot_de(old_dir, new_inode, new_dotdot_bh,
                                            new_dotdot_de);
                if (err)
                        goto error_new_dotdot;
        }

        /* if cross directory and only one is a directory, adjust nlink */
        if (!old_dotdot_de != !new_dotdot_de) {
                if (old_dotdot_de)
                        vfat_move_nlink(old_dir, new_dir);
                else
                        vfat_move_nlink(new_dir, old_dir);
        }

        vfat_update_dir_metadata(old_dir, &ts);
        /* if directories are not the same, update new_dir as well */
        if (old_dir != new_dir)
                vfat_update_dir_metadata(new_dir, &ts);

out:
        brelse(old_dotdot_bh);
        brelse(new_dotdot_bh);
        mutex_unlock(&MSDOS_SB(sb)->s_lock);

        return err;

error_new_dotdot:
        if (new_dotdot_de) {
                corrupt |= vfat_update_dotdot_de(new_dir, new_inode,
                                                 new_dotdot_bh, new_dotdot_de);
        }

error_old_dotdot:
        if (old_dotdot_de) {
                corrupt |= vfat_update_dotdot_de(old_dir, old_inode,
                                                 old_dotdot_bh, old_dotdot_de);
        }

error_exchange:
        vfat_exchange_ipos(old_inode, new_inode, new_i_pos, old_i_pos);
        corrupt |= vfat_sync_ipos(new_dir, new_inode);
        corrupt |= vfat_sync_ipos(old_dir, old_inode);

        if (corrupt < 0) {
                fat_fs_error(new_dir->i_sb,
                             "%s: Filesystem corrupted (i_pos %lld, %lld)",
                             __func__, old_i_pos, new_i_pos);
        }
        goto out;
}

static int vfat_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
                        struct dentry *old_dentry, struct inode *new_dir,
                        struct dentry *new_dentry, unsigned int flags)
{
        if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE))
                return -EINVAL;

        if (flags & RENAME_EXCHANGE) {
                return vfat_rename_exchange(old_dir, old_dentry,
                                            new_dir, new_dentry);
        }

        /* VFS already handled RENAME_NOREPLACE, handle it as a normal rename */
        return vfat_rename(old_dir, old_dentry, new_dir, new_dentry);
}

static const struct inode_operations vfat_dir_inode_operations = {
        .create                = vfat_create,
        .lookup                = vfat_lookup,
        .unlink                = vfat_unlink,
        .mkdir                = vfat_mkdir,
        .rmdir                = vfat_rmdir,
        .rename                = vfat_rename2,
        .setattr        = fat_setattr,
        .getattr        = fat_getattr,
        .update_time        = fat_update_time,
};

static void setup(struct super_block *sb)
{
        MSDOS_SB(sb)->dir_ops = &vfat_dir_inode_operations;
        if (MSDOS_SB(sb)->options.name_check != 's')
                sb->s_d_op = &vfat_ci_dentry_ops;
        else
                sb->s_d_op = &vfat_dentry_ops;
}

static int vfat_fill_super(struct super_block *sb, struct fs_context *fc)
{
        return fat_fill_super(sb, fc, setup);
}

static int vfat_get_tree(struct fs_context *fc)
{
        return get_tree_bdev(fc, vfat_fill_super);
}

static int vfat_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        return fat_parse_param(fc, param, true);
}

static const struct fs_context_operations vfat_context_ops = {
        .parse_param        = vfat_parse_param,
        .get_tree        = vfat_get_tree,
        .reconfigure        = fat_reconfigure,
        .free                = fat_free_fc,
};

static int vfat_init_fs_context(struct fs_context *fc)
{
        int err;

        /* Initialize with is_vfat == true */
        err = fat_init_fs_context(fc, true);
        if (err)
                return err;

        fc->ops = &vfat_context_ops;
        return 0;
}

static struct file_system_type vfat_fs_type = {
        .owner                = THIS_MODULE,
        .name                = "vfat",
        .kill_sb        = kill_block_super,
        .fs_flags        = FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
        .init_fs_context = vfat_init_fs_context,
        .parameters     = fat_param_spec,
};
MODULE_ALIAS_FS("vfat");

static int __init init_vfat_fs(void)
{
        return register_filesystem(&vfat_fs_type);
}

static void __exit exit_vfat_fs(void)
{
        unregister_filesystem(&vfat_fs_type);
}

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("VFAT filesystem support");
MODULE_AUTHOR("Gordon Chaffee");

module_init(init_vfat_fs)
module_exit(exit_vfat_fs)
















































































































































































































































































































    8 
    8 
    8 





































































































































































































































































    2 




















 2153 



 2149 






   75 

   75 
    3 
























































































































































   23 





   23 
   23 










   23 



























   73 










    1 


   66 


    7 







   14 





    2 


    2 


    2 


    1 
    3 


   14 





    2 
    2 

    2 
    2 

    2 
    2 

    2 
    2 


   14 





    3 
    1 

    2 
    2 

    2 
    2 

    2 
    2 


   11 





    4 
    2 

    2 
    4 

    4 
    2 

    3 
    3 




   43 





    4 


    6 


    4 



   44 





    5 
    2 

    3 
    4 

    4 
    3 


   34 





   11 
    6 

    8 
    9 

   14 
    3 


   35 





    8 
    8 

   12 
    4 

    6 
   10 




   17 
   23 


   11 










    9 



    5 

    4 


   19 


   13 










    5 



    2 

    2 


   23 
    5 









   39 
    6 





   29 






    6 














   46 











    1 







    1 



   35 


    8 
    1 


    1 
















    2 





















    3 


    1 




    1 


    1 



    2 





    2 



    2 








    1 



    1 

    1 








   32 












    1 







    1 



   28 



    1 
    1 





    1 


    1 












    1 




    1 













   85 



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/* gw.c - CAN frame Gateway/Router/Bridge with netlink interface
 *
 * Copyright (c) 2019 Volkswagen Group Electronic Research
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Volkswagen nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * Alternatively, provided that this notice is retained in full, this
 * software may be distributed under the terms of the GNU General
 * Public License ("GPL") version 2, in which case the provisions of the
 * GPL apply INSTEAD OF those given above.
 *
 * The provided data structures and external interfaces from this code
 * are not restricted to be used by modules with a GPL compatible license.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/net.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/skbuff.h>
#include <linux/can.h>
#include <linux/can/core.h>
#include <linux/can/skb.h>
#include <linux/can/gw.h>
#include <net/rtnetlink.h>
#include <net/net_namespace.h>
#include <net/sock.h>

#define CAN_GW_NAME "can-gw"

MODULE_DESCRIPTION("PF_CAN netlink gateway");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Oliver Hartkopp <oliver.hartkopp@volkswagen.de>");
MODULE_ALIAS(CAN_GW_NAME);

#define CGW_MIN_HOPS 1
#define CGW_MAX_HOPS 6
#define CGW_DEFAULT_HOPS 1

static unsigned int max_hops __read_mostly = CGW_DEFAULT_HOPS;
module_param(max_hops, uint, 0444);
MODULE_PARM_DESC(max_hops,
                 "maximum " CAN_GW_NAME " routing hops for CAN frames "
                 "(valid values: " __stringify(CGW_MIN_HOPS) "-"
                 __stringify(CGW_MAX_HOPS) " hops, "
                 "default: " __stringify(CGW_DEFAULT_HOPS) ")");

static struct notifier_block notifier;
static struct kmem_cache *cgw_cache __read_mostly;

/* structure that contains the (on-the-fly) CAN frame modifications */
struct cf_mod {
        struct {
                struct canfd_frame and;
                struct canfd_frame or;
                struct canfd_frame xor;
                struct canfd_frame set;
        } modframe;
        struct {
                u8 and;
                u8 or;
                u8 xor;
                u8 set;
        } modtype;
        void (*modfunc[MAX_MODFUNCTIONS])(struct canfd_frame *cf,
                                          struct cf_mod *mod);

        /* CAN frame checksum calculation after CAN frame modifications */
        struct {
                struct cgw_csum_xor xor;
                struct cgw_csum_crc8 crc8;
        } csum;
        struct {
                void (*xor)(struct canfd_frame *cf,
                            struct cgw_csum_xor *xor);
                void (*crc8)(struct canfd_frame *cf,
                             struct cgw_csum_crc8 *crc8);
        } csumfunc;
        u32 uid;
};

/* So far we just support CAN -> CAN routing and frame modifications.
 *
 * The internal can_can_gw structure contains data and attributes for
 * a CAN -> CAN gateway job.
 */
struct can_can_gw {
        struct can_filter filter;
        int src_idx;
        int dst_idx;
};

/* list entry for CAN gateways jobs */
struct cgw_job {
        struct hlist_node list;
        struct rcu_head rcu;
        u32 handled_frames;
        u32 dropped_frames;
        u32 deleted_frames;
        struct cf_mod mod;
        union {
                /* CAN frame data source */
                struct net_device *dev;
        } src;
        union {
                /* CAN frame data destination */
                struct net_device *dev;
        } dst;
        union {
                struct can_can_gw ccgw;
                /* tbc */
        };
        u8 gwtype;
        u8 limit_hops;
        u16 flags;
};

/* modification functions that are invoked in the hot path in can_can_gw_rcv */

#define MODFUNC(func, op) static void func(struct canfd_frame *cf, \
                                           struct cf_mod *mod) { op ; }

MODFUNC(mod_and_id, cf->can_id &= mod->modframe.and.can_id)
MODFUNC(mod_and_len, cf->len &= mod->modframe.and.len)
MODFUNC(mod_and_flags, cf->flags &= mod->modframe.and.flags)
MODFUNC(mod_and_data, *(u64 *)cf->data &= *(u64 *)mod->modframe.and.data)
MODFUNC(mod_or_id, cf->can_id |= mod->modframe.or.can_id)
MODFUNC(mod_or_len, cf->len |= mod->modframe.or.len)
MODFUNC(mod_or_flags, cf->flags |= mod->modframe.or.flags)
MODFUNC(mod_or_data, *(u64 *)cf->data |= *(u64 *)mod->modframe.or.data)
MODFUNC(mod_xor_id, cf->can_id ^= mod->modframe.xor.can_id)
MODFUNC(mod_xor_len, cf->len ^= mod->modframe.xor.len)
MODFUNC(mod_xor_flags, cf->flags ^= mod->modframe.xor.flags)
MODFUNC(mod_xor_data, *(u64 *)cf->data ^= *(u64 *)mod->modframe.xor.data)
MODFUNC(mod_set_id, cf->can_id = mod->modframe.set.can_id)
MODFUNC(mod_set_len, cf->len = mod->modframe.set.len)
MODFUNC(mod_set_flags, cf->flags = mod->modframe.set.flags)
MODFUNC(mod_set_data, *(u64 *)cf->data = *(u64 *)mod->modframe.set.data)

static void mod_and_fddata(struct canfd_frame *cf, struct cf_mod *mod)
{
        int i;

        for (i = 0; i < CANFD_MAX_DLEN; i += 8)
                *(u64 *)(cf->data + i) &= *(u64 *)(mod->modframe.and.data + i);
}

static void mod_or_fddata(struct canfd_frame *cf, struct cf_mod *mod)
{
        int i;

        for (i = 0; i < CANFD_MAX_DLEN; i += 8)
                *(u64 *)(cf->data + i) |= *(u64 *)(mod->modframe.or.data + i);
}

static void mod_xor_fddata(struct canfd_frame *cf, struct cf_mod *mod)
{
        int i;

        for (i = 0; i < CANFD_MAX_DLEN; i += 8)
                *(u64 *)(cf->data + i) ^= *(u64 *)(mod->modframe.xor.data + i);
}

static void mod_set_fddata(struct canfd_frame *cf, struct cf_mod *mod)
{
        memcpy(cf->data, mod->modframe.set.data, CANFD_MAX_DLEN);
}

/* retrieve valid CC DLC value and store it into 'len' */
static void mod_retrieve_ccdlc(struct canfd_frame *cf)
{
        struct can_frame *ccf = (struct can_frame *)cf;

        /* len8_dlc is only valid if len == CAN_MAX_DLEN */
        if (ccf->len != CAN_MAX_DLEN)
                return;

        /* do we have a valid len8_dlc value from 9 .. 15 ? */
        if (ccf->len8_dlc > CAN_MAX_DLEN && ccf->len8_dlc <= CAN_MAX_RAW_DLC)
                ccf->len = ccf->len8_dlc;
}

/* convert valid CC DLC value in 'len' into struct can_frame elements */
static void mod_store_ccdlc(struct canfd_frame *cf)
{
        struct can_frame *ccf = (struct can_frame *)cf;

        /* clear potential leftovers */
        ccf->len8_dlc = 0;

        /* plain data length 0 .. 8 - that was easy */
        if (ccf->len <= CAN_MAX_DLEN)
                return;

        /* potentially broken values are caught in can_can_gw_rcv() */
        if (ccf->len > CAN_MAX_RAW_DLC)
                return;

        /* we have a valid dlc value from 9 .. 15 in ccf->len */
        ccf->len8_dlc = ccf->len;
        ccf->len = CAN_MAX_DLEN;
}

static void mod_and_ccdlc(struct canfd_frame *cf, struct cf_mod *mod)
{
        mod_retrieve_ccdlc(cf);
        mod_and_len(cf, mod);
        mod_store_ccdlc(cf);
}

static void mod_or_ccdlc(struct canfd_frame *cf, struct cf_mod *mod)
{
        mod_retrieve_ccdlc(cf);
        mod_or_len(cf, mod);
        mod_store_ccdlc(cf);
}

static void mod_xor_ccdlc(struct canfd_frame *cf, struct cf_mod *mod)
{
        mod_retrieve_ccdlc(cf);
        mod_xor_len(cf, mod);
        mod_store_ccdlc(cf);
}

static void mod_set_ccdlc(struct canfd_frame *cf, struct cf_mod *mod)
{
        mod_set_len(cf, mod);
        mod_store_ccdlc(cf);
}

static void canframecpy(struct canfd_frame *dst, struct can_frame *src)
{
        /* Copy the struct members separately to ensure that no uninitialized
         * data are copied in the 3 bytes hole of the struct. This is needed
         * to make easy compares of the data in the struct cf_mod.
         */

        dst->can_id = src->can_id;
        dst->len = src->len;
        *(u64 *)dst->data = *(u64 *)src->data;
}

static void canfdframecpy(struct canfd_frame *dst, struct canfd_frame *src)
{
        /* Copy the struct members separately to ensure that no uninitialized
         * data are copied in the 2 bytes hole of the struct. This is needed
         * to make easy compares of the data in the struct cf_mod.
         */

        dst->can_id = src->can_id;
        dst->flags = src->flags;
        dst->len = src->len;
        memcpy(dst->data, src->data, CANFD_MAX_DLEN);
}

static int cgw_chk_csum_parms(s8 fr, s8 to, s8 re, struct rtcanmsg *r)
{
        s8 dlen = CAN_MAX_DLEN;

        if (r->flags & CGW_FLAGS_CAN_FD)
                dlen = CANFD_MAX_DLEN;

        /* absolute dlc values 0 .. 7 => 0 .. 7, e.g. data [0]
         * relative to received dlc -1 .. -8 :
         * e.g. for received dlc = 8
         * -1 => index = 7 (data[7])
         * -3 => index = 5 (data[5])
         * -8 => index = 0 (data[0])
         */

        if (fr >= -dlen && fr < dlen &&
            to >= -dlen && to < dlen &&
            re >= -dlen && re < dlen)
                return 0;
        else
                return -EINVAL;
}

static inline int calc_idx(int idx, int rx_len)
{
        if (idx < 0)
                return rx_len + idx;
        else
                return idx;
}

static void cgw_csum_xor_rel(struct canfd_frame *cf, struct cgw_csum_xor *xor)
{
        int from = calc_idx(xor->from_idx, cf->len);
        int to = calc_idx(xor->to_idx, cf->len);
        int res = calc_idx(xor->result_idx, cf->len);
        u8 val = xor->init_xor_val;
        int i;

        if (from < 0 || to < 0 || res < 0)
                return;

        if (from <= to) {
                for (i = from; i <= to; i++)
                        val ^= cf->data[i];
        } else {
                for (i = from; i >= to; i--)
                        val ^= cf->data[i];
        }

        cf->data[res] = val;
}

static void cgw_csum_xor_pos(struct canfd_frame *cf, struct cgw_csum_xor *xor)
{
        u8 val = xor->init_xor_val;
        int i;

        for (i = xor->from_idx; i <= xor->to_idx; i++)
                val ^= cf->data[i];

        cf->data[xor->result_idx] = val;
}

static void cgw_csum_xor_neg(struct canfd_frame *cf, struct cgw_csum_xor *xor)
{
        u8 val = xor->init_xor_val;
        int i;

        for (i = xor->from_idx; i >= xor->to_idx; i--)
                val ^= cf->data[i];

        cf->data[xor->result_idx] = val;
}

static void cgw_csum_crc8_rel(struct canfd_frame *cf,
                              struct cgw_csum_crc8 *crc8)
{
        int from = calc_idx(crc8->from_idx, cf->len);
        int to = calc_idx(crc8->to_idx, cf->len);
        int res = calc_idx(crc8->result_idx, cf->len);
        u8 crc = crc8->init_crc_val;
        int i;

        if (from < 0 || to < 0 || res < 0)
                return;

        if (from <= to) {
                for (i = crc8->from_idx; i <= crc8->to_idx; i++)
                        crc = crc8->crctab[crc ^ cf->data[i]];
        } else {
                for (i = crc8->from_idx; i >= crc8->to_idx; i--)
                        crc = crc8->crctab[crc ^ cf->data[i]];
        }

        switch (crc8->profile) {
        case CGW_CRC8PRF_1U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[0]];
                break;

        case  CGW_CRC8PRF_16U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[cf->data[1] & 0xF]];
                break;

        case CGW_CRC8PRF_SFFID_XOR:
                crc = crc8->crctab[crc ^ (cf->can_id & 0xFF) ^
                                   (cf->can_id >> 8 & 0xFF)];
                break;
        }

        cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val;
}

static void cgw_csum_crc8_pos(struct canfd_frame *cf,
                              struct cgw_csum_crc8 *crc8)
{
        u8 crc = crc8->init_crc_val;
        int i;

        for (i = crc8->from_idx; i <= crc8->to_idx; i++)
                crc = crc8->crctab[crc ^ cf->data[i]];

        switch (crc8->profile) {
        case CGW_CRC8PRF_1U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[0]];
                break;

        case  CGW_CRC8PRF_16U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[cf->data[1] & 0xF]];
                break;

        case CGW_CRC8PRF_SFFID_XOR:
                crc = crc8->crctab[crc ^ (cf->can_id & 0xFF) ^
                                   (cf->can_id >> 8 & 0xFF)];
                break;
        }

        cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val;
}

static void cgw_csum_crc8_neg(struct canfd_frame *cf,
                              struct cgw_csum_crc8 *crc8)
{
        u8 crc = crc8->init_crc_val;
        int i;

        for (i = crc8->from_idx; i >= crc8->to_idx; i--)
                crc = crc8->crctab[crc ^ cf->data[i]];

        switch (crc8->profile) {
        case CGW_CRC8PRF_1U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[0]];
                break;

        case  CGW_CRC8PRF_16U8:
                crc = crc8->crctab[crc ^ crc8->profile_data[cf->data[1] & 0xF]];
                break;

        case CGW_CRC8PRF_SFFID_XOR:
                crc = crc8->crctab[crc ^ (cf->can_id & 0xFF) ^
                                   (cf->can_id >> 8 & 0xFF)];
                break;
        }

        cf->data[crc8->result_idx] = crc ^ crc8->final_xor_val;
}

/* the receive & process & send function */
static void can_can_gw_rcv(struct sk_buff *skb, void *data)
{
        struct cgw_job *gwj = (struct cgw_job *)data;
        struct canfd_frame *cf;
        struct sk_buff *nskb;
        int modidx = 0;

        /* process strictly Classic CAN or CAN FD frames */
        if (gwj->flags & CGW_FLAGS_CAN_FD) {
                if (!can_is_canfd_skb(skb))
                        return;
        } else {
                if (!can_is_can_skb(skb))
                        return;
        }

        /* Do not handle CAN frames routed more than 'max_hops' times.
         * In general we should never catch this delimiter which is intended
         * to cover a misconfiguration protection (e.g. circular CAN routes).
         *
         * The Controller Area Network controllers only accept CAN frames with
         * correct CRCs - which are not visible in the controller registers.
         * According to skbuff.h documentation the csum_start element for IP
         * checksums is undefined/unused when ip_summed == CHECKSUM_UNNECESSARY.
         * Only CAN skbs can be processed here which already have this property.
         */

#define cgw_hops(skb) ((skb)->csum_start)

        BUG_ON(skb->ip_summed != CHECKSUM_UNNECESSARY);

        if (cgw_hops(skb) >= max_hops) {
                /* indicate deleted frames due to misconfiguration */
                gwj->deleted_frames++;
                return;
        }

        if (!(gwj->dst.dev->flags & IFF_UP)) {
                gwj->dropped_frames++;
                return;
        }

        /* is sending the skb back to the incoming interface not allowed? */
        if (!(gwj->flags & CGW_FLAGS_CAN_IIF_TX_OK) &&
            can_skb_prv(skb)->ifindex == gwj->dst.dev->ifindex)
                return;

        /* clone the given skb, which has not been done in can_rcv()
         *
         * When there is at least one modification function activated,
         * we need to copy the skb as we want to modify skb->data.
         */
        if (gwj->mod.modfunc[0])
                nskb = skb_copy(skb, GFP_ATOMIC);
        else
                nskb = skb_clone(skb, GFP_ATOMIC);

        if (!nskb) {
                gwj->dropped_frames++;
                return;
        }

        /* put the incremented hop counter in the cloned skb */
        cgw_hops(nskb) = cgw_hops(skb) + 1;

        /* first processing of this CAN frame -> adjust to private hop limit */
        if (gwj->limit_hops && cgw_hops(nskb) == 1)
                cgw_hops(nskb) = max_hops - gwj->limit_hops + 1;

        nskb->dev = gwj->dst.dev;

        /* pointer to modifiable CAN frame */
        cf = (struct canfd_frame *)nskb->data;

        /* perform preprocessed modification functions if there are any */
        while (modidx < MAX_MODFUNCTIONS && gwj->mod.modfunc[modidx])
                (*gwj->mod.modfunc[modidx++])(cf, &gwj->mod);

        /* Has the CAN frame been modified? */
        if (modidx) {
                /* get available space for the processed CAN frame type */
                int max_len = nskb->len - offsetof(struct canfd_frame, data);

                /* dlc may have changed, make sure it fits to the CAN frame */
                if (cf->len > max_len) {
                        /* delete frame due to misconfiguration */
                        gwj->deleted_frames++;
                        kfree_skb(nskb);
                        return;
                }

                /* check for checksum updates */
                if (gwj->mod.csumfunc.crc8)
                        (*gwj->mod.csumfunc.crc8)(cf, &gwj->mod.csum.crc8);

                if (gwj->mod.csumfunc.xor)
                        (*gwj->mod.csumfunc.xor)(cf, &gwj->mod.csum.xor);
        }

        /* clear the skb timestamp if not configured the other way */
        if (!(gwj->flags & CGW_FLAGS_CAN_SRC_TSTAMP))
                nskb->tstamp = 0;

        /* send to netdevice */
        if (can_send(nskb, gwj->flags & CGW_FLAGS_CAN_ECHO))
                gwj->dropped_frames++;
        else
                gwj->handled_frames++;
}

static inline int cgw_register_filter(struct net *net, struct cgw_job *gwj)
{
        return can_rx_register(net, gwj->src.dev, gwj->ccgw.filter.can_id,
                               gwj->ccgw.filter.can_mask, can_can_gw_rcv,
                               gwj, "gw", NULL);
}

static inline void cgw_unregister_filter(struct net *net, struct cgw_job *gwj)
{
        can_rx_unregister(net, gwj->src.dev, gwj->ccgw.filter.can_id,
                          gwj->ccgw.filter.can_mask, can_can_gw_rcv, gwj);
}

static void cgw_job_free_rcu(struct rcu_head *rcu_head)
{
        struct cgw_job *gwj = container_of(rcu_head, struct cgw_job, rcu);

        kmem_cache_free(cgw_cache, gwj);
}

static int cgw_notifier(struct notifier_block *nb,
                        unsigned long msg, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(dev);

        if (dev->type != ARPHRD_CAN)
                return NOTIFY_DONE;

        if (msg == NETDEV_UNREGISTER) {
                struct cgw_job *gwj = NULL;
                struct hlist_node *nx;

                ASSERT_RTNL();

                hlist_for_each_entry_safe(gwj, nx, &net->can.cgw_list, list) {
                        if (gwj->src.dev == dev || gwj->dst.dev == dev) {
                                hlist_del(&gwj->list);
                                cgw_unregister_filter(net, gwj);
                                call_rcu(&gwj->rcu, cgw_job_free_rcu);
                        }
                }
        }

        return NOTIFY_DONE;
}

static int cgw_put_job(struct sk_buff *skb, struct cgw_job *gwj, int type,
                       u32 pid, u32 seq, int flags)
{
        struct rtcanmsg *rtcan;
        struct nlmsghdr *nlh;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*rtcan), flags);
        if (!nlh)
                return -EMSGSIZE;

        rtcan = nlmsg_data(nlh);
        rtcan->can_family = AF_CAN;
        rtcan->gwtype = gwj->gwtype;
        rtcan->flags = gwj->flags;

        /* add statistics if available */

        if (gwj->handled_frames) {
                if (nla_put_u32(skb, CGW_HANDLED, gwj->handled_frames) < 0)
                        goto cancel;
        }

        if (gwj->dropped_frames) {
                if (nla_put_u32(skb, CGW_DROPPED, gwj->dropped_frames) < 0)
                        goto cancel;
        }

        if (gwj->deleted_frames) {
                if (nla_put_u32(skb, CGW_DELETED, gwj->deleted_frames) < 0)
                        goto cancel;
        }

        /* check non default settings of attributes */

        if (gwj->limit_hops) {
                if (nla_put_u8(skb, CGW_LIM_HOPS, gwj->limit_hops) < 0)
                        goto cancel;
        }

        if (gwj->flags & CGW_FLAGS_CAN_FD) {
                struct cgw_fdframe_mod mb;

                if (gwj->mod.modtype.and) {
                        memcpy(&mb.cf, &gwj->mod.modframe.and, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.and;
                        if (nla_put(skb, CGW_FDMOD_AND, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.or) {
                        memcpy(&mb.cf, &gwj->mod.modframe.or, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.or;
                        if (nla_put(skb, CGW_FDMOD_OR, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.xor) {
                        memcpy(&mb.cf, &gwj->mod.modframe.xor, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.xor;
                        if (nla_put(skb, CGW_FDMOD_XOR, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.set) {
                        memcpy(&mb.cf, &gwj->mod.modframe.set, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.set;
                        if (nla_put(skb, CGW_FDMOD_SET, sizeof(mb), &mb) < 0)
                                goto cancel;
                }
        } else {
                struct cgw_frame_mod mb;

                if (gwj->mod.modtype.and) {
                        memcpy(&mb.cf, &gwj->mod.modframe.and, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.and;
                        if (nla_put(skb, CGW_MOD_AND, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.or) {
                        memcpy(&mb.cf, &gwj->mod.modframe.or, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.or;
                        if (nla_put(skb, CGW_MOD_OR, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.xor) {
                        memcpy(&mb.cf, &gwj->mod.modframe.xor, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.xor;
                        if (nla_put(skb, CGW_MOD_XOR, sizeof(mb), &mb) < 0)
                                goto cancel;
                }

                if (gwj->mod.modtype.set) {
                        memcpy(&mb.cf, &gwj->mod.modframe.set, sizeof(mb.cf));
                        mb.modtype = gwj->mod.modtype.set;
                        if (nla_put(skb, CGW_MOD_SET, sizeof(mb), &mb) < 0)
                                goto cancel;
                }
        }

        if (gwj->mod.uid) {
                if (nla_put_u32(skb, CGW_MOD_UID, gwj->mod.uid) < 0)
                        goto cancel;
        }

        if (gwj->mod.csumfunc.crc8) {
                if (nla_put(skb, CGW_CS_CRC8, CGW_CS_CRC8_LEN,
                            &gwj->mod.csum.crc8) < 0)
                        goto cancel;
        }

        if (gwj->mod.csumfunc.xor) {
                if (nla_put(skb, CGW_CS_XOR, CGW_CS_XOR_LEN,
                            &gwj->mod.csum.xor) < 0)
                        goto cancel;
        }

        if (gwj->gwtype == CGW_TYPE_CAN_CAN) {
                if (gwj->ccgw.filter.can_id || gwj->ccgw.filter.can_mask) {
                        if (nla_put(skb, CGW_FILTER, sizeof(struct can_filter),
                                    &gwj->ccgw.filter) < 0)
                                goto cancel;
                }

                if (nla_put_u32(skb, CGW_SRC_IF, gwj->ccgw.src_idx) < 0)
                        goto cancel;

                if (nla_put_u32(skb, CGW_DST_IF, gwj->ccgw.dst_idx) < 0)
                        goto cancel;
        }

        nlmsg_end(skb, nlh);
        return 0;

cancel:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

/* Dump information about all CAN gateway jobs, in response to RTM_GETROUTE */
static int cgw_dump_jobs(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        struct cgw_job *gwj = NULL;
        int idx = 0;
        int s_idx = cb->args[0];

        rcu_read_lock();
        hlist_for_each_entry_rcu(gwj, &net->can.cgw_list, list) {
                if (idx < s_idx)
                        goto cont;

                if (cgw_put_job(skb, gwj, RTM_NEWROUTE,
                                NETLINK_CB(cb->skb).portid,
                                cb->nlh->nlmsg_seq, NLM_F_MULTI) < 0)
                        break;
cont:
                idx++;
        }
        rcu_read_unlock();

        cb->args[0] = idx;

        return skb->len;
}

static const struct nla_policy cgw_policy[CGW_MAX + 1] = {
        [CGW_MOD_AND]        = { .len = sizeof(struct cgw_frame_mod) },
        [CGW_MOD_OR]        = { .len = sizeof(struct cgw_frame_mod) },
        [CGW_MOD_XOR]        = { .len = sizeof(struct cgw_frame_mod) },
        [CGW_MOD_SET]        = { .len = sizeof(struct cgw_frame_mod) },
        [CGW_CS_XOR]        = { .len = sizeof(struct cgw_csum_xor) },
        [CGW_CS_CRC8]        = { .len = sizeof(struct cgw_csum_crc8) },
        [CGW_SRC_IF]        = { .type = NLA_U32 },
        [CGW_DST_IF]        = { .type = NLA_U32 },
        [CGW_FILTER]        = { .len = sizeof(struct can_filter) },
        [CGW_LIM_HOPS]        = { .type = NLA_U8 },
        [CGW_MOD_UID]        = { .type = NLA_U32 },
        [CGW_FDMOD_AND]        = { .len = sizeof(struct cgw_fdframe_mod) },
        [CGW_FDMOD_OR]        = { .len = sizeof(struct cgw_fdframe_mod) },
        [CGW_FDMOD_XOR]        = { .len = sizeof(struct cgw_fdframe_mod) },
        [CGW_FDMOD_SET]        = { .len = sizeof(struct cgw_fdframe_mod) },
};

/* check for common and gwtype specific attributes */
static int cgw_parse_attr(struct nlmsghdr *nlh, struct cf_mod *mod,
                          u8 gwtype, void *gwtypeattr, u8 *limhops)
{
        struct nlattr *tb[CGW_MAX + 1];
        struct rtcanmsg *r = nlmsg_data(nlh);
        int modidx = 0;
        int err = 0;

        /* initialize modification & checksum data space */
        memset(mod, 0, sizeof(*mod));

        err = nlmsg_parse_deprecated(nlh, sizeof(struct rtcanmsg), tb,
                                     CGW_MAX, cgw_policy, NULL);
        if (err < 0)
                return err;

        if (tb[CGW_LIM_HOPS]) {
                *limhops = nla_get_u8(tb[CGW_LIM_HOPS]);

                if (*limhops < 1 || *limhops > max_hops)
                        return -EINVAL;
        }

        /* check for AND/OR/XOR/SET modifications */
        if (r->flags & CGW_FLAGS_CAN_FD) {
                struct cgw_fdframe_mod mb;

                if (tb[CGW_FDMOD_AND]) {
                        nla_memcpy(&mb, tb[CGW_FDMOD_AND], CGW_FDMODATTR_LEN);

                        canfdframecpy(&mod->modframe.and, &mb.cf);
                        mod->modtype.and = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_and_id;

                        if (mb.modtype & CGW_MOD_LEN)
                                mod->modfunc[modidx++] = mod_and_len;

                        if (mb.modtype & CGW_MOD_FLAGS)
                                mod->modfunc[modidx++] = mod_and_flags;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_and_fddata;
                }

                if (tb[CGW_FDMOD_OR]) {
                        nla_memcpy(&mb, tb[CGW_FDMOD_OR], CGW_FDMODATTR_LEN);

                        canfdframecpy(&mod->modframe.or, &mb.cf);
                        mod->modtype.or = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_or_id;

                        if (mb.modtype & CGW_MOD_LEN)
                                mod->modfunc[modidx++] = mod_or_len;

                        if (mb.modtype & CGW_MOD_FLAGS)
                                mod->modfunc[modidx++] = mod_or_flags;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_or_fddata;
                }

                if (tb[CGW_FDMOD_XOR]) {
                        nla_memcpy(&mb, tb[CGW_FDMOD_XOR], CGW_FDMODATTR_LEN);

                        canfdframecpy(&mod->modframe.xor, &mb.cf);
                        mod->modtype.xor = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_xor_id;

                        if (mb.modtype & CGW_MOD_LEN)
                                mod->modfunc[modidx++] = mod_xor_len;

                        if (mb.modtype & CGW_MOD_FLAGS)
                                mod->modfunc[modidx++] = mod_xor_flags;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_xor_fddata;
                }

                if (tb[CGW_FDMOD_SET]) {
                        nla_memcpy(&mb, tb[CGW_FDMOD_SET], CGW_FDMODATTR_LEN);

                        canfdframecpy(&mod->modframe.set, &mb.cf);
                        mod->modtype.set = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_set_id;

                        if (mb.modtype & CGW_MOD_LEN)
                                mod->modfunc[modidx++] = mod_set_len;

                        if (mb.modtype & CGW_MOD_FLAGS)
                                mod->modfunc[modidx++] = mod_set_flags;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_set_fddata;
                }
        } else {
                struct cgw_frame_mod mb;

                if (tb[CGW_MOD_AND]) {
                        nla_memcpy(&mb, tb[CGW_MOD_AND], CGW_MODATTR_LEN);

                        canframecpy(&mod->modframe.and, &mb.cf);
                        mod->modtype.and = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_and_id;

                        if (mb.modtype & CGW_MOD_DLC)
                                mod->modfunc[modidx++] = mod_and_ccdlc;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_and_data;
                }

                if (tb[CGW_MOD_OR]) {
                        nla_memcpy(&mb, tb[CGW_MOD_OR], CGW_MODATTR_LEN);

                        canframecpy(&mod->modframe.or, &mb.cf);
                        mod->modtype.or = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_or_id;

                        if (mb.modtype & CGW_MOD_DLC)
                                mod->modfunc[modidx++] = mod_or_ccdlc;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_or_data;
                }

                if (tb[CGW_MOD_XOR]) {
                        nla_memcpy(&mb, tb[CGW_MOD_XOR], CGW_MODATTR_LEN);

                        canframecpy(&mod->modframe.xor, &mb.cf);
                        mod->modtype.xor = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_xor_id;

                        if (mb.modtype & CGW_MOD_DLC)
                                mod->modfunc[modidx++] = mod_xor_ccdlc;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_xor_data;
                }

                if (tb[CGW_MOD_SET]) {
                        nla_memcpy(&mb, tb[CGW_MOD_SET], CGW_MODATTR_LEN);

                        canframecpy(&mod->modframe.set, &mb.cf);
                        mod->modtype.set = mb.modtype;

                        if (mb.modtype & CGW_MOD_ID)
                                mod->modfunc[modidx++] = mod_set_id;

                        if (mb.modtype & CGW_MOD_DLC)
                                mod->modfunc[modidx++] = mod_set_ccdlc;

                        if (mb.modtype & CGW_MOD_DATA)
                                mod->modfunc[modidx++] = mod_set_data;
                }
        }

        /* check for checksum operations after CAN frame modifications */
        if (modidx) {
                if (tb[CGW_CS_CRC8]) {
                        struct cgw_csum_crc8 *c = nla_data(tb[CGW_CS_CRC8]);

                        err = cgw_chk_csum_parms(c->from_idx, c->to_idx,
                                                 c->result_idx, r);
                        if (err)
                                return err;

                        nla_memcpy(&mod->csum.crc8, tb[CGW_CS_CRC8],
                                   CGW_CS_CRC8_LEN);

                        /* select dedicated processing function to reduce
                         * runtime operations in receive hot path.
                         */
                        if (c->from_idx < 0 || c->to_idx < 0 ||
                            c->result_idx < 0)
                                mod->csumfunc.crc8 = cgw_csum_crc8_rel;
                        else if (c->from_idx <= c->to_idx)
                                mod->csumfunc.crc8 = cgw_csum_crc8_pos;
                        else
                                mod->csumfunc.crc8 = cgw_csum_crc8_neg;
                }

                if (tb[CGW_CS_XOR]) {
                        struct cgw_csum_xor *c = nla_data(tb[CGW_CS_XOR]);

                        err = cgw_chk_csum_parms(c->from_idx, c->to_idx,
                                                 c->result_idx, r);
                        if (err)
                                return err;

                        nla_memcpy(&mod->csum.xor, tb[CGW_CS_XOR],
                                   CGW_CS_XOR_LEN);

                        /* select dedicated processing function to reduce
                         * runtime operations in receive hot path.
                         */
                        if (c->from_idx < 0 || c->to_idx < 0 ||
                            c->result_idx < 0)
                                mod->csumfunc.xor = cgw_csum_xor_rel;
                        else if (c->from_idx <= c->to_idx)
                                mod->csumfunc.xor = cgw_csum_xor_pos;
                        else
                                mod->csumfunc.xor = cgw_csum_xor_neg;
                }

                if (tb[CGW_MOD_UID])
                        nla_memcpy(&mod->uid, tb[CGW_MOD_UID], sizeof(u32));
        }

        if (gwtype == CGW_TYPE_CAN_CAN) {
                /* check CGW_TYPE_CAN_CAN specific attributes */
                struct can_can_gw *ccgw = (struct can_can_gw *)gwtypeattr;

                memset(ccgw, 0, sizeof(*ccgw));

                /* check for can_filter in attributes */
                if (tb[CGW_FILTER])
                        nla_memcpy(&ccgw->filter, tb[CGW_FILTER],
                                   sizeof(struct can_filter));

                err = -ENODEV;

                /* specifying two interfaces is mandatory */
                if (!tb[CGW_SRC_IF] || !tb[CGW_DST_IF])
                        return err;

                ccgw->src_idx = nla_get_u32(tb[CGW_SRC_IF]);
                ccgw->dst_idx = nla_get_u32(tb[CGW_DST_IF]);

                /* both indices set to 0 for flushing all routing entries */
                if (!ccgw->src_idx && !ccgw->dst_idx)
                        return 0;

                /* only one index set to 0 is an error */
                if (!ccgw->src_idx || !ccgw->dst_idx)
                        return err;
        }

        /* add the checks for other gwtypes here */

        return 0;
}

static int cgw_create_job(struct sk_buff *skb,  struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct rtcanmsg *r;
        struct cgw_job *gwj;
        struct cf_mod mod;
        struct can_can_gw ccgw;
        u8 limhops = 0;
        int err = 0;

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (nlmsg_len(nlh) < sizeof(*r))
                return -EINVAL;

        r = nlmsg_data(nlh);
        if (r->can_family != AF_CAN)
                return -EPFNOSUPPORT;

        /* so far we only support CAN -> CAN routings */
        if (r->gwtype != CGW_TYPE_CAN_CAN)
                return -EINVAL;

        err = cgw_parse_attr(nlh, &mod, CGW_TYPE_CAN_CAN, &ccgw, &limhops);
        if (err < 0)
                return err;

        if (mod.uid) {
                ASSERT_RTNL();

                /* check for updating an existing job with identical uid */
                hlist_for_each_entry(gwj, &net->can.cgw_list, list) {
                        if (gwj->mod.uid != mod.uid)
                                continue;

                        /* interfaces & filters must be identical */
                        if (memcmp(&gwj->ccgw, &ccgw, sizeof(ccgw)))
                                return -EINVAL;

                        /* update modifications with disabled softirq & quit */
                        local_bh_disable();
                        memcpy(&gwj->mod, &mod, sizeof(mod));
                        local_bh_enable();
                        return 0;
                }
        }

        /* ifindex == 0 is not allowed for job creation */
        if (!ccgw.src_idx || !ccgw.dst_idx)
                return -ENODEV;

        gwj = kmem_cache_alloc(cgw_cache, GFP_KERNEL);
        if (!gwj)
                return -ENOMEM;

        gwj->handled_frames = 0;
        gwj->dropped_frames = 0;
        gwj->deleted_frames = 0;
        gwj->flags = r->flags;
        gwj->gwtype = r->gwtype;
        gwj->limit_hops = limhops;

        /* insert already parsed information */
        memcpy(&gwj->mod, &mod, sizeof(mod));
        memcpy(&gwj->ccgw, &ccgw, sizeof(ccgw));

        err = -ENODEV;

        gwj->src.dev = __dev_get_by_index(net, gwj->ccgw.src_idx);

        if (!gwj->src.dev)
                goto out;

        if (gwj->src.dev->type != ARPHRD_CAN)
                goto out;

        gwj->dst.dev = __dev_get_by_index(net, gwj->ccgw.dst_idx);

        if (!gwj->dst.dev)
                goto out;

        if (gwj->dst.dev->type != ARPHRD_CAN)
                goto out;

        /* is sending the skb back to the incoming interface intended? */
        if (gwj->src.dev == gwj->dst.dev &&
            !(gwj->flags & CGW_FLAGS_CAN_IIF_TX_OK)) {
                err = -EINVAL;
                goto out;
        }

        ASSERT_RTNL();

        err = cgw_register_filter(net, gwj);
        if (!err)
                hlist_add_head_rcu(&gwj->list, &net->can.cgw_list);
out:
        if (err)
                kmem_cache_free(cgw_cache, gwj);

        return err;
}

static void cgw_remove_all_jobs(struct net *net)
{
        struct cgw_job *gwj = NULL;
        struct hlist_node *nx;

        ASSERT_RTNL();

        hlist_for_each_entry_safe(gwj, nx, &net->can.cgw_list, list) {
                hlist_del(&gwj->list);
                cgw_unregister_filter(net, gwj);
                call_rcu(&gwj->rcu, cgw_job_free_rcu);
        }
}

static int cgw_remove_job(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct cgw_job *gwj = NULL;
        struct hlist_node *nx;
        struct rtcanmsg *r;
        struct cf_mod mod;
        struct can_can_gw ccgw;
        u8 limhops = 0;
        int err = 0;

        if (!netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (nlmsg_len(nlh) < sizeof(*r))
                return -EINVAL;

        r = nlmsg_data(nlh);
        if (r->can_family != AF_CAN)
                return -EPFNOSUPPORT;

        /* so far we only support CAN -> CAN routings */
        if (r->gwtype != CGW_TYPE_CAN_CAN)
                return -EINVAL;

        err = cgw_parse_attr(nlh, &mod, CGW_TYPE_CAN_CAN, &ccgw, &limhops);
        if (err < 0)
                return err;

        /* two interface indices both set to 0 => remove all entries */
        if (!ccgw.src_idx && !ccgw.dst_idx) {
                cgw_remove_all_jobs(net);
                return 0;
        }

        err = -EINVAL;

        ASSERT_RTNL();

        /* remove only the first matching entry */
        hlist_for_each_entry_safe(gwj, nx, &net->can.cgw_list, list) {
                if (gwj->flags != r->flags)
                        continue;

                if (gwj->limit_hops != limhops)
                        continue;

                /* we have a match when uid is enabled and identical */
                if (gwj->mod.uid || mod.uid) {
                        if (gwj->mod.uid != mod.uid)
                                continue;
                } else {
                        /* no uid => check for identical modifications */
                        if (memcmp(&gwj->mod, &mod, sizeof(mod)))
                                continue;
                }

                /* if (r->gwtype == CGW_TYPE_CAN_CAN) - is made sure here */
                if (memcmp(&gwj->ccgw, &ccgw, sizeof(ccgw)))
                        continue;

                hlist_del(&gwj->list);
                cgw_unregister_filter(net, gwj);
                call_rcu(&gwj->rcu, cgw_job_free_rcu);
                err = 0;
                break;
        }

        return err;
}

static int __net_init cangw_pernet_init(struct net *net)
{
        INIT_HLIST_HEAD(&net->can.cgw_list);
        return 0;
}

static void __net_exit cangw_pernet_exit_batch(struct list_head *net_list)
{
        struct net *net;

        rtnl_lock();
        list_for_each_entry(net, net_list, exit_list)
                cgw_remove_all_jobs(net);
        rtnl_unlock();
}

static struct pernet_operations cangw_pernet_ops = {
        .init = cangw_pernet_init,
        .exit_batch = cangw_pernet_exit_batch,
};

static const struct rtnl_msg_handler cgw_rtnl_msg_handlers[] __initconst_or_module = {
        {.owner = THIS_MODULE, .protocol = PF_CAN, .msgtype = RTM_NEWROUTE,
         .doit = cgw_create_job},
        {.owner = THIS_MODULE, .protocol = PF_CAN, .msgtype = RTM_DELROUTE,
         .doit = cgw_remove_job},
        {.owner = THIS_MODULE, .protocol = PF_CAN, .msgtype = RTM_GETROUTE,
         .dumpit = cgw_dump_jobs},
};

static __init int cgw_module_init(void)
{
        int ret;

        /* sanitize given module parameter */
        max_hops = clamp_t(unsigned int, max_hops, CGW_MIN_HOPS, CGW_MAX_HOPS);

        pr_info("can: netlink gateway - max_hops=%d\n",        max_hops);

        ret = register_pernet_subsys(&cangw_pernet_ops);
        if (ret)
                return ret;

        ret = -ENOMEM;
        cgw_cache = kmem_cache_create("can_gw", sizeof(struct cgw_job),
                                      0, 0, NULL);
        if (!cgw_cache)
                goto out_cache_create;

        /* set notifier */
        notifier.notifier_call = cgw_notifier;
        ret = register_netdevice_notifier(&notifier);
        if (ret)
                goto out_register_notifier;

        ret = rtnl_register_many(cgw_rtnl_msg_handlers);
        if (ret)
                goto out_rtnl_register;

        return 0;

out_rtnl_register:
        unregister_netdevice_notifier(&notifier);
out_register_notifier:
        kmem_cache_destroy(cgw_cache);
out_cache_create:
        unregister_pernet_subsys(&cangw_pernet_ops);

        return ret;
}

static __exit void cgw_module_exit(void)
{
        rtnl_unregister_all(PF_CAN);

        unregister_netdevice_notifier(&notifier);

        unregister_pernet_subsys(&cangw_pernet_ops);
        rcu_barrier(); /* Wait for completion of call_rcu()'s */

        kmem_cache_destroy(cgw_cache);
}

module_init(cgw_module_init);
module_exit(cgw_module_exit);












   10 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_INCLUDE_PATH ../../drivers/dma-buf
#define TRACE_SYSTEM sync_trace

#if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SYNC_H

#include "sync_debug.h"
#include <linux/tracepoint.h>

TRACE_EVENT(sync_timeline,
        TP_PROTO(struct sync_timeline *timeline),

        TP_ARGS(timeline),

        TP_STRUCT__entry(
                        __string(name, timeline->name)
                        __field(u32, value)
        ),

        TP_fast_assign(
                        __assign_str(name);
                        __entry->value = timeline->value;
        ),

        TP_printk("name=%s value=%d", __get_str(name), __entry->value)
);

#endif /* if !defined(_TRACE_SYNC_H) || defined(TRACE_HEADER_MULTI_READ) */

/* This part must be outside protection */
#include <trace/define_trace.h>

















































    5 























    5 





    5 











    5 
















    6 







    6 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Nano River Technologies viperboard driver
 *
 *  This is the core driver for the viperboard. There are cell drivers
 *  available for I2C, ADC and both GPIOs. SPI is not yet supported.
 *  The drivers do not support all features the board exposes. See user
 *  manual of the viperboard.
 *
 *  (C) 2012 by Lemonage GmbH
 *  Author: Lars Poeschel <poeschel@lemonage.de>
 *  All rights reserved.
 */

#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/mutex.h>

#include <linux/mfd/core.h>
#include <linux/mfd/viperboard.h>

#include <linux/usb.h>


static const struct usb_device_id vprbrd_table[] = {
        { USB_DEVICE(0x2058, 0x1005) },   /* Nano River Technologies */
        { }                               /* Terminating entry */
};

MODULE_DEVICE_TABLE(usb, vprbrd_table);

static const struct mfd_cell vprbrd_devs[] = {
        {
                .name = "viperboard-gpio",
        },
        {
                .name = "viperboard-i2c",
        },
        {
                .name = "viperboard-adc",
        },
};

static int vprbrd_probe(struct usb_interface *interface,
                              const struct usb_device_id *id)
{
        struct vprbrd *vb;

        u16 version = 0;
        int pipe, ret;

        /* allocate memory for our device state and initialize it */
        vb = kzalloc(sizeof(*vb), GFP_KERNEL);
        if (!vb)
                return -ENOMEM;

        mutex_init(&vb->lock);

        vb->usb_dev = usb_get_dev(interface_to_usbdev(interface));

        /* save our data pointer in this interface device */
        usb_set_intfdata(interface, vb);
        dev_set_drvdata(&vb->pdev.dev, vb);

        /* get version information, major first, minor then */
        pipe = usb_rcvctrlpipe(vb->usb_dev, 0);
        ret = usb_control_msg(vb->usb_dev, pipe, VPRBRD_USB_REQUEST_MAJOR,
                VPRBRD_USB_TYPE_IN, 0x0000, 0x0000, vb->buf, 1,
                VPRBRD_USB_TIMEOUT_MS);
        if (ret == 1)
                version = vb->buf[0];

        ret = usb_control_msg(vb->usb_dev, pipe, VPRBRD_USB_REQUEST_MINOR,
                VPRBRD_USB_TYPE_IN, 0x0000, 0x0000, vb->buf, 1,
                VPRBRD_USB_TIMEOUT_MS);
        if (ret == 1) {
                version <<= 8;
                version = version | vb->buf[0];
        }

        dev_info(&interface->dev,
                 "version %x.%02x found at bus %03d address %03d\n",
                 version >> 8, version & 0xff,
                 vb->usb_dev->bus->busnum, vb->usb_dev->devnum);

        ret = mfd_add_hotplug_devices(&interface->dev, vprbrd_devs,
                                      ARRAY_SIZE(vprbrd_devs));
        if (ret != 0) {
                dev_err(&interface->dev, "Failed to add mfd devices to core.");
                goto error;
        }

        return 0;

error:
        if (vb) {
                usb_put_dev(vb->usb_dev);
                kfree(vb);
        }

        return ret;
}

static void vprbrd_disconnect(struct usb_interface *interface)
{
        struct vprbrd *vb = usb_get_intfdata(interface);

        mfd_remove_devices(&interface->dev);
        usb_set_intfdata(interface, NULL);
        usb_put_dev(vb->usb_dev);
        kfree(vb);

        dev_dbg(&interface->dev, "disconnected\n");
}

static struct usb_driver vprbrd_driver = {
        .name                = "viperboard",
        .probe                = vprbrd_probe,
        .disconnect        = vprbrd_disconnect,
        .id_table        = vprbrd_table,
};

module_usb_driver(vprbrd_driver);

MODULE_DESCRIPTION("Nano River Technologies viperboard mfd core driver");
MODULE_AUTHOR("Lars Poeschel <poeschel@lemonage.de>");
MODULE_LICENSE("GPL");




































































































































































































   10 
   17 

    8 
  101 





   99 


   99 



















































































































































    6 












   12 


    1 

































   17 





































   13 





   14 
































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _DCCP_H
#define _DCCP_H
/*
 *  net/dccp/dccp.h
 *
 *  An implementation of the DCCP protocol
 *  Copyright (c) 2005 Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 *  Copyright (c) 2005-6 Ian McDonald <ian.mcdonald@jandi.co.nz>
 */

#include <linux/dccp.h>
#include <linux/ktime.h>
#include <net/snmp.h>
#include <net/sock.h>
#include <net/tcp.h>
#include "ackvec.h"

/*
 *         DCCP - specific warning and debugging macros.
 */
#define DCCP_WARN(fmt, ...)                                                \
        net_warn_ratelimited("%s: " fmt, __func__, ##__VA_ARGS__)
#define DCCP_CRIT(fmt, a...) printk(KERN_CRIT fmt " at %s:%d/%s()\n", ##a, \
                                         __FILE__, __LINE__, __func__)
#define DCCP_BUG(a...)       do { DCCP_CRIT("BUG: " a); dump_stack(); } while(0)
#define DCCP_BUG_ON(cond)    do { if (unlikely((cond) != 0))                   \
                                     DCCP_BUG("\"%s\" holds (exception!)", \
                                              __stringify(cond));          \
                             } while (0)

#define DCCP_PRINTK(enable, fmt, args...)        do { if (enable)             \
                                                        printk(fmt, ##args); \
                                                } while(0)
#define DCCP_PR_DEBUG(enable, fmt, a...)        DCCP_PRINTK(enable, KERN_DEBUG \
                                                  "%s: " fmt, __func__, ##a)

#ifdef CONFIG_IP_DCCP_DEBUG
extern bool dccp_debug;
#define dccp_pr_debug(format, a...)          DCCP_PR_DEBUG(dccp_debug, format, ##a)
#define dccp_pr_debug_cat(format, a...)   DCCP_PRINTK(dccp_debug, format, ##a)
#define dccp_debug(fmt, a...)                  dccp_pr_debug_cat(KERN_DEBUG fmt, ##a)
#else
#define dccp_pr_debug(format, a...)          do {} while (0)
#define dccp_pr_debug_cat(format, a...)          do {} while (0)
#define dccp_debug(format, a...)          do {} while (0)
#endif

extern struct inet_hashinfo dccp_hashinfo;

DECLARE_PER_CPU(unsigned int, dccp_orphan_count);

void dccp_time_wait(struct sock *sk, int state, int timeo);

/*
 *  Set safe upper bounds for header and option length. Since Data Offset is 8
 *  bits (RFC 4340, sec. 5.1), the total header length can never be more than
 *  4 * 255 = 1020 bytes. The largest possible header length is 28 bytes (X=1):
 *    - DCCP-Response with ACK Subheader and 4 bytes of Service code      OR
 *    - DCCP-Reset    with ACK Subheader and 4 bytes of Reset Code fields
 *  Hence a safe upper bound for the maximum option length is 1020-28 = 992
 */
#define MAX_DCCP_SPECIFIC_HEADER (255 * sizeof(uint32_t))
#define DCCP_MAX_PACKET_HDR 28
#define DCCP_MAX_OPT_LEN (MAX_DCCP_SPECIFIC_HEADER - DCCP_MAX_PACKET_HDR)
#define MAX_DCCP_HEADER (MAX_DCCP_SPECIFIC_HEADER + MAX_HEADER)

/* Upper bound for initial feature-negotiation overhead (padded to 32 bits) */
#define DCCP_FEATNEG_OVERHEAD         (32 * sizeof(uint32_t))

#define DCCP_TIMEWAIT_LEN (60 * HZ) /* how long to wait to destroy TIME-WAIT
                                     * state, about 60 seconds */

/* RFC 1122, 4.2.3.1 initial RTO value */
#define DCCP_TIMEOUT_INIT ((unsigned int)(3 * HZ))

/*
 * The maximum back-off value for retransmissions. This is needed for
 *  - retransmitting client-Requests (sec. 8.1.1),
 *  - retransmitting Close/CloseReq when closing (sec. 8.3),
 *  - feature-negotiation retransmission (sec. 6.6.3),
 *  - Acks in client-PARTOPEN state (sec. 8.1.5).
 */
#define DCCP_RTO_MAX ((unsigned int)(64 * HZ))

/*
 * RTT sampling: sanity bounds and fallback RTT value from RFC 4340, section 3.4
 */
#define DCCP_SANE_RTT_MIN        100
#define DCCP_FALLBACK_RTT        (USEC_PER_SEC / 5)
#define DCCP_SANE_RTT_MAX        (3 * USEC_PER_SEC)

/* sysctl variables for DCCP */
extern int  sysctl_dccp_request_retries;
extern int  sysctl_dccp_retries1;
extern int  sysctl_dccp_retries2;
extern int  sysctl_dccp_tx_qlen;
extern int  sysctl_dccp_sync_ratelimit;

/*
 *        48-bit sequence number arithmetic (signed and unsigned)
 */
#define INT48_MIN          0x800000000000LL                /* 2^47            */
#define UINT48_MAX          0xFFFFFFFFFFFFLL                /* 2^48 - 1 */
#define COMPLEMENT48(x)         (0x1000000000000LL - (x))        /* 2^48 - x */
#define TO_SIGNED48(x)         (((x) < INT48_MIN)? (x) : -COMPLEMENT48( (x)))
#define TO_UNSIGNED48(x) (((x) >= 0)?             (x) :  COMPLEMENT48(-(x)))
#define ADD48(a, b)         (((a) + (b)) & UINT48_MAX)
#define SUB48(a, b)         ADD48((a), COMPLEMENT48(b))

static inline void dccp_inc_seqno(u64 *seqno)
{
        *seqno = ADD48(*seqno, 1);
}

/* signed mod-2^48 distance: pos. if seqno1 < seqno2, neg. if seqno1 > seqno2 */
static inline s64 dccp_delta_seqno(const u64 seqno1, const u64 seqno2)
{
        u64 delta = SUB48(seqno2, seqno1);

        return TO_SIGNED48(delta);
}

/* is seq1 < seq2 ? */
static inline int before48(const u64 seq1, const u64 seq2)
{
        return (s64)((seq2 << 16) - (seq1 << 16)) > 0;
}

/* is seq1 > seq2 ? */
#define after48(seq1, seq2)        before48(seq2, seq1)

/* is seq2 <= seq1 <= seq3 ? */
static inline int between48(const u64 seq1, const u64 seq2, const u64 seq3)
{
        return (seq3 << 16) - (seq2 << 16) >= (seq1 << 16) - (seq2 << 16);
}

/**
 * dccp_loss_count - Approximate the number of lost data packets in a burst loss
 * @s1:  last known sequence number before the loss ('hole')
 * @s2:  first sequence number seen after the 'hole'
 * @ndp: NDP count on packet with sequence number @s2
 */
static inline u64 dccp_loss_count(const u64 s1, const u64 s2, const u64 ndp)
{
        s64 delta = dccp_delta_seqno(s1, s2);

        WARN_ON(delta < 0);
        delta -= ndp + 1;

        return delta > 0 ? delta : 0;
}

/**
 * dccp_loss_free - Evaluate condition for data loss from RFC 4340, 7.7.1
 */
static inline bool dccp_loss_free(const u64 s1, const u64 s2, const u64 ndp)
{
        return dccp_loss_count(s1, s2, ndp) == 0;
}

enum {
        DCCP_MIB_NUM = 0,
        DCCP_MIB_ACTIVEOPENS,                        /* ActiveOpens */
        DCCP_MIB_ESTABRESETS,                        /* EstabResets */
        DCCP_MIB_CURRESTAB,                        /* CurrEstab */
        DCCP_MIB_OUTSEGS,                        /* OutSegs */
        DCCP_MIB_OUTRSTS,
        DCCP_MIB_ABORTONTIMEOUT,
        DCCP_MIB_TIMEOUTS,
        DCCP_MIB_ABORTFAILED,
        DCCP_MIB_PASSIVEOPENS,
        DCCP_MIB_ATTEMPTFAILS,
        DCCP_MIB_OUTDATAGRAMS,
        DCCP_MIB_INERRS,
        DCCP_MIB_OPTMANDATORYERROR,
        DCCP_MIB_INVALIDOPT,
        __DCCP_MIB_MAX
};

#define DCCP_MIB_MAX        __DCCP_MIB_MAX
struct dccp_mib {
        unsigned long        mibs[DCCP_MIB_MAX];
};

DECLARE_SNMP_STAT(struct dccp_mib, dccp_statistics);
#define DCCP_INC_STATS(field)        SNMP_INC_STATS(dccp_statistics, field)
#define __DCCP_INC_STATS(field)        __SNMP_INC_STATS(dccp_statistics, field)
#define DCCP_DEC_STATS(field)        SNMP_DEC_STATS(dccp_statistics, field)

/*
 *         Checksumming routines
 */
static inline unsigned int dccp_csum_coverage(const struct sk_buff *skb)
{
        const struct dccp_hdr* dh = dccp_hdr(skb);

        if (dh->dccph_cscov == 0)
                return skb->len;
        return (dh->dccph_doff + dh->dccph_cscov - 1) * sizeof(u32);
}

static inline void dccp_csum_outgoing(struct sk_buff *skb)
{
        unsigned int cov = dccp_csum_coverage(skb);

        if (cov >= skb->len)
                dccp_hdr(skb)->dccph_cscov = 0;

        skb->csum = skb_checksum(skb, 0, (cov > skb->len)? skb->len : cov, 0);
}

void dccp_v4_send_check(struct sock *sk, struct sk_buff *skb);

int dccp_retransmit_skb(struct sock *sk);

void dccp_send_ack(struct sock *sk);
void dccp_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
                         struct request_sock *rsk);

void dccp_send_sync(struct sock *sk, const u64 seq,
                    const enum dccp_pkt_type pkt_type);

/*
 * TX Packet Dequeueing Interface
 */
void dccp_qpolicy_push(struct sock *sk, struct sk_buff *skb);
bool dccp_qpolicy_full(struct sock *sk);
void dccp_qpolicy_drop(struct sock *sk, struct sk_buff *skb);
struct sk_buff *dccp_qpolicy_top(struct sock *sk);
struct sk_buff *dccp_qpolicy_pop(struct sock *sk);
bool dccp_qpolicy_param_ok(struct sock *sk, __be32 param);

/*
 * TX Packet Output and TX Timers
 */
void dccp_write_xmit(struct sock *sk);
void dccp_write_space(struct sock *sk);
void dccp_flush_write_queue(struct sock *sk, long *time_budget);

void dccp_init_xmit_timers(struct sock *sk);
static inline void dccp_clear_xmit_timers(struct sock *sk)
{
        inet_csk_clear_xmit_timers(sk);
}

unsigned int dccp_sync_mss(struct sock *sk, u32 pmtu);

const char *dccp_packet_name(const int type);

void dccp_set_state(struct sock *sk, const int state);
void dccp_done(struct sock *sk);

int dccp_reqsk_init(struct request_sock *rq, struct dccp_sock const *dp,
                    struct sk_buff const *skb);

int dccp_v4_conn_request(struct sock *sk, struct sk_buff *skb);

struct sock *dccp_create_openreq_child(const struct sock *sk,
                                       const struct request_sock *req,
                                       const struct sk_buff *skb);

int dccp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);

struct sock *dccp_v4_request_recv_sock(const struct sock *sk, struct sk_buff *skb,
                                       struct request_sock *req,
                                       struct dst_entry *dst,
                                       struct request_sock *req_unhash,
                                       bool *own_req);
struct sock *dccp_check_req(struct sock *sk, struct sk_buff *skb,
                            struct request_sock *req);

int dccp_child_process(struct sock *parent, struct sock *child,
                       struct sk_buff *skb);
int dccp_rcv_state_process(struct sock *sk, struct sk_buff *skb,
                           struct dccp_hdr *dh, unsigned int len);
int dccp_rcv_established(struct sock *sk, struct sk_buff *skb,
                         const struct dccp_hdr *dh, const unsigned int len);

void dccp_destruct_common(struct sock *sk);
int dccp_init_sock(struct sock *sk, const __u8 ctl_sock_initialized);
void dccp_destroy_sock(struct sock *sk);

void dccp_close(struct sock *sk, long timeout);
struct sk_buff *dccp_make_response(const struct sock *sk, struct dst_entry *dst,
                                   struct request_sock *req);

int dccp_connect(struct sock *sk);
int dccp_disconnect(struct sock *sk, int flags);
int dccp_getsockopt(struct sock *sk, int level, int optname,
                    char __user *optval, int __user *optlen);
int dccp_setsockopt(struct sock *sk, int level, int optname,
                    sockptr_t optval, unsigned int optlen);
int dccp_ioctl(struct sock *sk, int cmd, int *karg);
int dccp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int dccp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
                 int *addr_len);
void dccp_shutdown(struct sock *sk, int how);
int inet_dccp_listen(struct socket *sock, int backlog);
__poll_t dccp_poll(struct file *file, struct socket *sock,
                       poll_table *wait);
int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
void dccp_req_err(struct sock *sk, u64 seq);

struct sk_buff *dccp_ctl_make_reset(struct sock *sk, struct sk_buff *skb);
int dccp_send_reset(struct sock *sk, enum dccp_reset_codes code);
void dccp_send_close(struct sock *sk, const int active);
int dccp_invalid_packet(struct sk_buff *skb);
u32 dccp_sample_rtt(struct sock *sk, long delta);

static inline bool dccp_bad_service_code(const struct sock *sk,
                                        const __be32 service)
{
        const struct dccp_sock *dp = dccp_sk(sk);

        if (dp->dccps_service == service)
                return false;
        return !dccp_list_has_service(dp->dccps_service_list, service);
}

/**
 * dccp_skb_cb  -  DCCP per-packet control information
 * @dccpd_type: one of %dccp_pkt_type (or unknown)
 * @dccpd_ccval: CCVal field (5.1), see e.g. RFC 4342, 8.1
 * @dccpd_reset_code: one of %dccp_reset_codes
 * @dccpd_reset_data: Data1..3 fields (depend on @dccpd_reset_code)
 * @dccpd_opt_len: total length of all options (5.8) in the packet
 * @dccpd_seq: sequence number
 * @dccpd_ack_seq: acknowledgment number subheader field value
 *
 * This is used for transmission as well as for reception.
 */
struct dccp_skb_cb {
        union {
                struct inet_skb_parm        h4;
#if IS_ENABLED(CONFIG_IPV6)
                struct inet6_skb_parm        h6;
#endif
        } header;
        __u8  dccpd_type:4;
        __u8  dccpd_ccval:4;
        __u8  dccpd_reset_code,
              dccpd_reset_data[3];
        __u16 dccpd_opt_len;
        __u64 dccpd_seq;
        __u64 dccpd_ack_seq;
};

#define DCCP_SKB_CB(__skb) ((struct dccp_skb_cb *)&((__skb)->cb[0]))

/* RFC 4340, sec. 7.7 */
static inline int dccp_non_data_packet(const struct sk_buff *skb)
{
        const __u8 type = DCCP_SKB_CB(skb)->dccpd_type;

        return type == DCCP_PKT_ACK         ||
               type == DCCP_PKT_CLOSE         ||
               type == DCCP_PKT_CLOSEREQ ||
               type == DCCP_PKT_RESET         ||
               type == DCCP_PKT_SYNC         ||
               type == DCCP_PKT_SYNCACK;
}

/* RFC 4340, sec. 7.7 */
static inline int dccp_data_packet(const struct sk_buff *skb)
{
        const __u8 type = DCCP_SKB_CB(skb)->dccpd_type;

        return type == DCCP_PKT_DATA         ||
               type == DCCP_PKT_DATAACK  ||
               type == DCCP_PKT_REQUEST  ||
               type == DCCP_PKT_RESPONSE;
}

static inline int dccp_packet_without_ack(const struct sk_buff *skb)
{
        const __u8 type = DCCP_SKB_CB(skb)->dccpd_type;

        return type == DCCP_PKT_DATA || type == DCCP_PKT_REQUEST;
}

#define DCCP_PKT_WITHOUT_ACK_SEQ (UINT48_MAX << 2)

static inline void dccp_hdr_set_seq(struct dccp_hdr *dh, const u64 gss)
{
        struct dccp_hdr_ext *dhx = (struct dccp_hdr_ext *)((void *)dh +
                                                           sizeof(*dh));
        dh->dccph_seq2 = 0;
        dh->dccph_seq = htons((gss >> 32) & 0xfffff);
        dhx->dccph_seq_low = htonl(gss & 0xffffffff);
}

static inline void dccp_hdr_set_ack(struct dccp_hdr_ack_bits *dhack,
                                    const u64 gsr)
{
        dhack->dccph_reserved1 = 0;
        dhack->dccph_ack_nr_high = htons(gsr >> 32);
        dhack->dccph_ack_nr_low  = htonl(gsr & 0xffffffff);
}

static inline void dccp_update_gsr(struct sock *sk, u64 seq)
{
        struct dccp_sock *dp = dccp_sk(sk);

        if (after48(seq, dp->dccps_gsr))
                dp->dccps_gsr = seq;
        /* Sequence validity window depends on remote Sequence Window (7.5.1) */
        dp->dccps_swl = SUB48(ADD48(dp->dccps_gsr, 1), dp->dccps_r_seq_win / 4);
        /*
         * Adjust SWL so that it is not below ISR. In contrast to RFC 4340,
         * 7.5.1 we perform this check beyond the initial handshake: W/W' are
         * always > 32, so for the first W/W' packets in the lifetime of a
         * connection we always have to adjust SWL.
         * A second reason why we are doing this is that the window depends on
         * the feature-remote value of Sequence Window: nothing stops the peer
         * from updating this value while we are busy adjusting SWL for the
         * first W packets (we would have to count from scratch again then).
         * Therefore it is safer to always make sure that the Sequence Window
         * is not artificially extended by a peer who grows SWL downwards by
         * continually updating the feature-remote Sequence-Window.
         * If sequence numbers wrap it is bad luck. But that will take a while
         * (48 bit), and this measure prevents Sequence-number attacks.
         */
        if (before48(dp->dccps_swl, dp->dccps_isr))
                dp->dccps_swl = dp->dccps_isr;
        dp->dccps_swh = ADD48(dp->dccps_gsr, (3 * dp->dccps_r_seq_win) / 4);
}

static inline void dccp_update_gss(struct sock *sk, u64 seq)
{
        struct dccp_sock *dp = dccp_sk(sk);

        dp->dccps_gss = seq;
        /* Ack validity window depends on local Sequence Window value (7.5.1) */
        dp->dccps_awl = SUB48(ADD48(dp->dccps_gss, 1), dp->dccps_l_seq_win);
        /* Adjust AWL so that it is not below ISS - see comment above for SWL */
        if (before48(dp->dccps_awl, dp->dccps_iss))
                dp->dccps_awl = dp->dccps_iss;
        dp->dccps_awh = dp->dccps_gss;
}

static inline int dccp_ackvec_pending(const struct sock *sk)
{
        return dccp_sk(sk)->dccps_hc_rx_ackvec != NULL &&
               !dccp_ackvec_is_empty(dccp_sk(sk)->dccps_hc_rx_ackvec);
}

static inline int dccp_ack_pending(const struct sock *sk)
{
        return dccp_ackvec_pending(sk) || inet_csk_ack_scheduled(sk);
}

int dccp_feat_signal_nn_change(struct sock *sk, u8 feat, u64 nn_val);
int dccp_feat_finalise_settings(struct dccp_sock *dp);
int dccp_feat_server_ccid_dependencies(struct dccp_request_sock *dreq);
int dccp_feat_insert_opts(struct dccp_sock*, struct dccp_request_sock*,
                          struct sk_buff *skb);
int dccp_feat_activate_values(struct sock *sk, struct list_head *fn);
void dccp_feat_list_purge(struct list_head *fn_list);

int dccp_insert_options(struct sock *sk, struct sk_buff *skb);
int dccp_insert_options_rsk(struct dccp_request_sock *, struct sk_buff *);
u32 dccp_timestamp(void);
void dccp_timestamping_init(void);
int dccp_insert_option(struct sk_buff *skb, unsigned char option,
                       const void *value, unsigned char len);

#ifdef CONFIG_SYSCTL
int dccp_sysctl_init(void);
void dccp_sysctl_exit(void);
#else
static inline int dccp_sysctl_init(void)
{
        return 0;
}

static inline void dccp_sysctl_exit(void)
{
}
#endif

#endif /* _DCCP_H */

















































































































































































































































































   95 








































    5 













    5 





    5 

    5 





    5 


    5 


    1 
    4 














































   91 
   91 











    2 























































    2 

















   95 

















   95 










   68 


   28 





   38 








   95 





   95 







   95 

   95 








   95 

    4 




   91 





















































































    3 





    3 



    2 









   90 








   87 
    5 
   90 




























    1 

















   12 







  100 
























    3 



    2 





























    4 











    1 
















   90 








    1 







    5 










   12 









   12 













   12 


   12 















  298 




  297 




































  108 



  108 
  108 


  108 




  108 



  376 




  377 
  378 
   45 








  378 





  378 























    1 




    5 





































  378 




















  378 













  446 






   35 


  422 
  420 

    1 



  422 
    3 
  419 



  421 




  107 
  291 
   22 
   25 
  300 








































  245 
















  241 
    6 

  244 




  245 






  229 
   17 









  236 




   10 
    6 

    5 






  384 
  377 
  245 
  245 



































   31 

   27 





    9 




   10 
    4 



   10 









    2 













    9 








   31 
   31 

    1 
    8 

   31 


    8 
   29 

   31 

























    6 



   13 
   11 











   24 






   23 


   23 



















  109 






    5 




    4 






   10 

   88 
    2 














































   14 





















   10 



    5 














  221 


  221 


  107 


   65 
   52 








  252 






  239 
   14 








    4 



   30 

  223 

  251 




















   35 




  220 



















  125 






    8 


    4 

    2 


  116 
    7 


    2 
  106 






  136 






   11 



    3 

    2 


  120 
    4 


    2 
   65 
   52 

  116 


















    7 
    5 



    1 






    1 






















  113 

















  113 



  113 








  113 

  113 

















































    3 






    3 










   63 






   57 















    1 
    7 


    7 
    7 

    7 






    7 











    7 

   34 












   34 

























  113 




   10 



  113 














    2 


    5 




















  127 

   14 












  113 

   22 
    1 







   91 





  111 
   16 













   25 


  363 









    4 

   18 



    1 










  106 

  113 

   21 

   25 

    3 

    4 

  134 

    2 






  383 




   39 




    1 









    1 



  378 



  378 









  373 


















  398 







    1 




   18 







   38 
  378 











  344 
   37 


  379 























































   15 








    6 

    2 



    3 




    1 

    1 
























   45 
   45 



   14 
    1 
   30 




















   21 








   21 



   18 











   29 












   28 


   28 




    2 



    5 
   25 















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
/*
 *        An async IO implementation for Linux
 *        Written by Benjamin LaHaise <bcrl@kvack.org>
 *
 *        Implements an efficient asynchronous io interface.
 *
 *        Copyright 2000, 2001, 2002 Red Hat, Inc.  All Rights Reserved.
 *        Copyright 2018 Christoph Hellwig.
 *
 *        See ../COPYING for licensing terms.
 */
#define pr_fmt(fmt) "%s: " fmt, __func__

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/time.h>
#include <linux/aio_abi.h>
#include <linux/export.h>
#include <linux/syscalls.h>
#include <linux/backing-dev.h>
#include <linux/refcount.h>
#include <linux/uio.h>

#include <linux/sched/signal.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/aio.h>
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/eventfd.h>
#include <linux/blkdev.h>
#include <linux/compat.h>
#include <linux/migrate.h>
#include <linux/ramfs.h>
#include <linux/percpu-refcount.h>
#include <linux/mount.h>
#include <linux/pseudo_fs.h>

#include <linux/uaccess.h>
#include <linux/nospec.h>

#include "internal.h"

#define KIOCB_KEY                0

#define AIO_RING_MAGIC                        0xa10a10a1
#define AIO_RING_COMPAT_FEATURES        1
#define AIO_RING_INCOMPAT_FEATURES        0
struct aio_ring {
        unsigned        id;        /* kernel internal index number */
        unsigned        nr;        /* number of io_events */
        unsigned        head;        /* Written to by userland or under ring_lock
                                 * mutex by aio_read_events_ring(). */
        unsigned        tail;

        unsigned        magic;
        unsigned        compat_features;
        unsigned        incompat_features;
        unsigned        header_length;        /* size of aio_ring */


        struct io_event                io_events[];
}; /* 128 bytes + ring size */

/*
 * Plugging is meant to work with larger batches of IOs. If we don't
 * have more than the below, then don't bother setting up a plug.
 */
#define AIO_PLUG_THRESHOLD        2

#define AIO_RING_PAGES        8

struct kioctx_table {
        struct rcu_head                rcu;
        unsigned                nr;
        struct kioctx __rcu        *table[] __counted_by(nr);
};

struct kioctx_cpu {
        unsigned                reqs_available;
};

struct ctx_rq_wait {
        struct completion comp;
        atomic_t count;
};

struct kioctx {
        struct percpu_ref        users;
        atomic_t                dead;

        struct percpu_ref        reqs;

        unsigned long                user_id;

        struct kioctx_cpu __percpu *cpu;

        /*
         * For percpu reqs_available, number of slots we move to/from global
         * counter at a time:
         */
        unsigned                req_batch;
        /*
         * This is what userspace passed to io_setup(), it's not used for
         * anything but counting against the global max_reqs quota.
         *
         * The real limit is nr_events - 1, which will be larger (see
         * aio_setup_ring())
         */
        unsigned                max_reqs;

        /* Size of ringbuffer, in units of struct io_event */
        unsigned                nr_events;

        unsigned long                mmap_base;
        unsigned long                mmap_size;

        struct folio                **ring_folios;
        long                        nr_pages;

        struct rcu_work                free_rwork;        /* see free_ioctx() */

        /*
         * signals when all in-flight requests are done
         */
        struct ctx_rq_wait        *rq_wait;

        struct {
                /*
                 * This counts the number of available slots in the ringbuffer,
                 * so we avoid overflowing it: it's decremented (if positive)
                 * when allocating a kiocb and incremented when the resulting
                 * io_event is pulled off the ringbuffer.
                 *
                 * We batch accesses to it with a percpu version.
                 */
                atomic_t        reqs_available;
        } ____cacheline_aligned_in_smp;

        struct {
                spinlock_t        ctx_lock;
                struct list_head active_reqs;        /* used for cancellation */
        } ____cacheline_aligned_in_smp;

        struct {
                struct mutex        ring_lock;
                wait_queue_head_t wait;
        } ____cacheline_aligned_in_smp;

        struct {
                unsigned        tail;
                unsigned        completed_events;
                spinlock_t        completion_lock;
        } ____cacheline_aligned_in_smp;

        struct folio                *internal_folios[AIO_RING_PAGES];
        struct file                *aio_ring_file;

        unsigned                id;
};

/*
 * First field must be the file pointer in all the
 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
 */
struct fsync_iocb {
        struct file                *file;
        struct work_struct        work;
        bool                        datasync;
        struct cred                *creds;
};

struct poll_iocb {
        struct file                *file;
        struct wait_queue_head        *head;
        __poll_t                events;
        bool                        cancelled;
        bool                        work_scheduled;
        bool                        work_need_resched;
        struct wait_queue_entry        wait;
        struct work_struct        work;
};

/*
 * NOTE! Each of the iocb union members has the file pointer
 * as the first entry in their struct definition. So you can
 * access the file pointer through any of the sub-structs,
 * or directly as just 'ki_filp' in this struct.
 */
struct aio_kiocb {
        union {
                struct file                *ki_filp;
                struct kiocb                rw;
                struct fsync_iocb        fsync;
                struct poll_iocb        poll;
        };

        struct kioctx                *ki_ctx;
        kiocb_cancel_fn                *ki_cancel;

        struct io_event                ki_res;

        struct list_head        ki_list;        /* the aio core uses this
                                                 * for cancellation */
        refcount_t                ki_refcnt;

        /*
         * If the aio_resfd field of the userspace iocb is not zero,
         * this is the underlying eventfd context to deliver events to.
         */
        struct eventfd_ctx        *ki_eventfd;
};

/*------ sysctl variables----*/
static DEFINE_SPINLOCK(aio_nr_lock);
static unsigned long aio_nr;                /* current system wide number of aio requests */
static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */
/*----end sysctl variables---*/
#ifdef CONFIG_SYSCTL
static const struct ctl_table aio_sysctls[] = {
        {
                .procname        = "aio-nr",
                .data                = &aio_nr,
                .maxlen                = sizeof(aio_nr),
                .mode                = 0444,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "aio-max-nr",
                .data                = &aio_max_nr,
                .maxlen                = sizeof(aio_max_nr),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
};

static void __init aio_sysctl_init(void)
{
        register_sysctl_init("fs", aio_sysctls);
}
#else
#define aio_sysctl_init() do { } while (0)
#endif

static struct kmem_cache        *kiocb_cachep;
static struct kmem_cache        *kioctx_cachep;

static struct vfsmount *aio_mnt;

static const struct file_operations aio_ring_fops;
static const struct address_space_operations aio_ctx_aops;

static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
{
        struct file *file;
        struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb);
        if (IS_ERR(inode))
                return ERR_CAST(inode);

        inode->i_mapping->a_ops = &aio_ctx_aops;
        inode->i_mapping->i_private_data = ctx;
        inode->i_size = PAGE_SIZE * nr_pages;

        file = alloc_file_pseudo(inode, aio_mnt, "[aio]",
                                O_RDWR, &aio_ring_fops);
        if (IS_ERR(file))
                iput(inode);
        return file;
}

static int aio_init_fs_context(struct fs_context *fc)
{
        if (!init_pseudo(fc, AIO_RING_MAGIC))
                return -ENOMEM;
        fc->s_iflags |= SB_I_NOEXEC;
        return 0;
}

/* aio_setup
 *        Creates the slab caches used by the aio routines, panic on
 *        failure as this is done early during the boot sequence.
 */
static int __init aio_setup(void)
{
        static struct file_system_type aio_fs = {
                .name                = "aio",
                .init_fs_context = aio_init_fs_context,
                .kill_sb        = kill_anon_super,
        };
        aio_mnt = kern_mount(&aio_fs);
        if (IS_ERR(aio_mnt))
                panic("Failed to create aio fs mount.");

        kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
        kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
        aio_sysctl_init();
        return 0;
}
__initcall(aio_setup);

static void put_aio_ring_file(struct kioctx *ctx)
{
        struct file *aio_ring_file = ctx->aio_ring_file;
        struct address_space *i_mapping;

        if (aio_ring_file) {
                truncate_setsize(file_inode(aio_ring_file), 0);

                /* Prevent further access to the kioctx from migratepages */
                i_mapping = aio_ring_file->f_mapping;
                spin_lock(&i_mapping->i_private_lock);
                i_mapping->i_private_data = NULL;
                ctx->aio_ring_file = NULL;
                spin_unlock(&i_mapping->i_private_lock);

                fput(aio_ring_file);
        }
}

static void aio_free_ring(struct kioctx *ctx)
{
        int i;

        /* Disconnect the kiotx from the ring file.  This prevents future
         * accesses to the kioctx from page migration.
         */
        put_aio_ring_file(ctx);

        for (i = 0; i < ctx->nr_pages; i++) {
                struct folio *folio = ctx->ring_folios[i];

                if (!folio)
                        continue;

                pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i,
                         folio_ref_count(folio));
                ctx->ring_folios[i] = NULL;
                folio_put(folio);
        }

        if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) {
                kfree(ctx->ring_folios);
                ctx->ring_folios = NULL;
        }
}

static int aio_ring_mremap(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct mm_struct *mm = vma->vm_mm;
        struct kioctx_table *table;
        int i, res = -EINVAL;

        spin_lock(&mm->ioctx_lock);
        rcu_read_lock();
        table = rcu_dereference(mm->ioctx_table);
        if (!table)
                goto out_unlock;

        for (i = 0; i < table->nr; i++) {
                struct kioctx *ctx;

                ctx = rcu_dereference(table->table[i]);
                if (ctx && ctx->aio_ring_file == file) {
                        if (!atomic_read(&ctx->dead)) {
                                ctx->user_id = ctx->mmap_base = vma->vm_start;
                                res = 0;
                        }
                        break;
                }
        }

out_unlock:
        rcu_read_unlock();
        spin_unlock(&mm->ioctx_lock);
        return res;
}

static const struct vm_operations_struct aio_ring_vm_ops = {
        .mremap                = aio_ring_mremap,
#if IS_ENABLED(CONFIG_MMU)
        .fault                = filemap_fault,
        .map_pages        = filemap_map_pages,
        .page_mkwrite        = filemap_page_mkwrite,
#endif
};

static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma)
{
        vm_flags_set(vma, VM_DONTEXPAND);
        vma->vm_ops = &aio_ring_vm_ops;
        return 0;
}

static const struct file_operations aio_ring_fops = {
        .mmap = aio_ring_mmap,
};

#if IS_ENABLED(CONFIG_MIGRATION)
static int aio_migrate_folio(struct address_space *mapping, struct folio *dst,
                        struct folio *src, enum migrate_mode mode)
{
        struct kioctx *ctx;
        unsigned long flags;
        pgoff_t idx;
        int rc = 0;

        /* mapping->i_private_lock here protects against the kioctx teardown.  */
        spin_lock(&mapping->i_private_lock);
        ctx = mapping->i_private_data;
        if (!ctx) {
                rc = -EINVAL;
                goto out;
        }

        /* The ring_lock mutex.  The prevents aio_read_events() from writing
         * to the ring's head, and prevents page migration from mucking in
         * a partially initialized kiotx.
         */
        if (!mutex_trylock(&ctx->ring_lock)) {
                rc = -EAGAIN;
                goto out;
        }

        idx = src->index;
        if (idx < (pgoff_t)ctx->nr_pages) {
                /* Make sure the old folio hasn't already been changed */
                if (ctx->ring_folios[idx] != src)
                        rc = -EAGAIN;
        } else
                rc = -EINVAL;

        if (rc != 0)
                goto out_unlock;

        /* Writeback must be complete */
        BUG_ON(folio_test_writeback(src));
        folio_get(dst);

        rc = folio_migrate_mapping(mapping, dst, src, 1);
        if (rc != MIGRATEPAGE_SUCCESS) {
                folio_put(dst);
                goto out_unlock;
        }

        /* Take completion_lock to prevent other writes to the ring buffer
         * while the old folio is copied to the new.  This prevents new
         * events from being lost.
         */
        spin_lock_irqsave(&ctx->completion_lock, flags);
        folio_copy(dst, src);
        folio_migrate_flags(dst, src);
        BUG_ON(ctx->ring_folios[idx] != src);
        ctx->ring_folios[idx] = dst;
        spin_unlock_irqrestore(&ctx->completion_lock, flags);

        /* The old folio is no longer accessible. */
        folio_put(src);

out_unlock:
        mutex_unlock(&ctx->ring_lock);
out:
        spin_unlock(&mapping->i_private_lock);
        return rc;
}
#else
#define aio_migrate_folio NULL
#endif

static const struct address_space_operations aio_ctx_aops = {
        .dirty_folio        = noop_dirty_folio,
        .migrate_folio        = aio_migrate_folio,
};

static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
{
        struct aio_ring *ring;
        struct mm_struct *mm = current->mm;
        unsigned long size, unused;
        int nr_pages;
        int i;
        struct file *file;

        /* Compensate for the ring buffer's head/tail overlap entry */
        nr_events += 2;        /* 1 is required, 2 for good luck */

        size = sizeof(struct aio_ring);
        size += sizeof(struct io_event) * nr_events;

        nr_pages = PFN_UP(size);
        if (nr_pages < 0)
                return -EINVAL;

        file = aio_private_file(ctx, nr_pages);
        if (IS_ERR(file)) {
                ctx->aio_ring_file = NULL;
                return -ENOMEM;
        }

        ctx->aio_ring_file = file;
        nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring))
                        / sizeof(struct io_event);

        ctx->ring_folios = ctx->internal_folios;
        if (nr_pages > AIO_RING_PAGES) {
                ctx->ring_folios = kcalloc(nr_pages, sizeof(struct folio *),
                                           GFP_KERNEL);
                if (!ctx->ring_folios) {
                        put_aio_ring_file(ctx);
                        return -ENOMEM;
                }
        }

        for (i = 0; i < nr_pages; i++) {
                struct folio *folio;

                folio = __filemap_get_folio(file->f_mapping, i,
                                            FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
                                            GFP_USER | __GFP_ZERO);
                if (IS_ERR(folio))
                        break;

                pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i,
                         folio_ref_count(folio));
                folio_end_read(folio, true);

                ctx->ring_folios[i] = folio;
        }
        ctx->nr_pages = i;

        if (unlikely(i != nr_pages)) {
                aio_free_ring(ctx);
                return -ENOMEM;
        }

        ctx->mmap_size = nr_pages * PAGE_SIZE;
        pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size);

        if (mmap_write_lock_killable(mm)) {
                ctx->mmap_size = 0;
                aio_free_ring(ctx);
                return -EINTR;
        }

        ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
                                 PROT_READ | PROT_WRITE,
                                 MAP_SHARED, 0, 0, &unused, NULL);
        mmap_write_unlock(mm);
        if (IS_ERR((void *)ctx->mmap_base)) {
                ctx->mmap_size = 0;
                aio_free_ring(ctx);
                return -ENOMEM;
        }

        pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base);

        ctx->user_id = ctx->mmap_base;
        ctx->nr_events = nr_events; /* trusted copy */

        ring = folio_address(ctx->ring_folios[0]);
        ring->nr = nr_events;        /* user copy */
        ring->id = ~0U;
        ring->head = ring->tail = 0;
        ring->magic = AIO_RING_MAGIC;
        ring->compat_features = AIO_RING_COMPAT_FEATURES;
        ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
        ring->header_length = sizeof(struct aio_ring);
        flush_dcache_folio(ctx->ring_folios[0]);

        return 0;
}

#define AIO_EVENTS_PER_PAGE        (PAGE_SIZE / sizeof(struct io_event))
#define AIO_EVENTS_FIRST_PAGE        ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
#define AIO_EVENTS_OFFSET        (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)

void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
{
        struct aio_kiocb *req;
        struct kioctx *ctx;
        unsigned long flags;

        /*
         * kiocb didn't come from aio or is neither a read nor a write, hence
         * ignore it.
         */
        if (!(iocb->ki_flags & IOCB_AIO_RW))
                return;

        req = container_of(iocb, struct aio_kiocb, rw);

        if (WARN_ON_ONCE(!list_empty(&req->ki_list)))
                return;

        ctx = req->ki_ctx;

        spin_lock_irqsave(&ctx->ctx_lock, flags);
        list_add_tail(&req->ki_list, &ctx->active_reqs);
        req->ki_cancel = cancel;
        spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
EXPORT_SYMBOL(kiocb_set_cancel_fn);

/*
 * free_ioctx() should be RCU delayed to synchronize against the RCU
 * protected lookup_ioctx() and also needs process context to call
 * aio_free_ring().  Use rcu_work.
 */
static void free_ioctx(struct work_struct *work)
{
        struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx,
                                          free_rwork);
        pr_debug("freeing %p\n", ctx);

        aio_free_ring(ctx);
        free_percpu(ctx->cpu);
        percpu_ref_exit(&ctx->reqs);
        percpu_ref_exit(&ctx->users);
        kmem_cache_free(kioctx_cachep, ctx);
}

static void free_ioctx_reqs(struct percpu_ref *ref)
{
        struct kioctx *ctx = container_of(ref, struct kioctx, reqs);

        /* At this point we know that there are no any in-flight requests */
        if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count))
                complete(&ctx->rq_wait->comp);

        /* Synchronize against RCU protected table->table[] dereferences */
        INIT_RCU_WORK(&ctx->free_rwork, free_ioctx);
        queue_rcu_work(system_wq, &ctx->free_rwork);
}

/*
 * When this function runs, the kioctx has been removed from the "hash table"
 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted -
 * now it's safe to cancel any that need to be.
 */
static void free_ioctx_users(struct percpu_ref *ref)
{
        struct kioctx *ctx = container_of(ref, struct kioctx, users);
        struct aio_kiocb *req;

        spin_lock_irq(&ctx->ctx_lock);

        while (!list_empty(&ctx->active_reqs)) {
                req = list_first_entry(&ctx->active_reqs,
                                       struct aio_kiocb, ki_list);
                req->ki_cancel(&req->rw);
                list_del_init(&req->ki_list);
        }

        spin_unlock_irq(&ctx->ctx_lock);

        percpu_ref_kill(&ctx->reqs);
        percpu_ref_put(&ctx->reqs);
}

static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm)
{
        unsigned i, new_nr;
        struct kioctx_table *table, *old;
        struct aio_ring *ring;

        spin_lock(&mm->ioctx_lock);
        table = rcu_dereference_raw(mm->ioctx_table);

        while (1) {
                if (table)
                        for (i = 0; i < table->nr; i++)
                                if (!rcu_access_pointer(table->table[i])) {
                                        ctx->id = i;
                                        rcu_assign_pointer(table->table[i], ctx);
                                        spin_unlock(&mm->ioctx_lock);

                                        /* While kioctx setup is in progress,
                                         * we are protected from page migration
                                         * changes ring_folios by ->ring_lock.
                                         */
                                        ring = folio_address(ctx->ring_folios[0]);
                                        ring->id = ctx->id;
                                        return 0;
                                }

                new_nr = (table ? table->nr : 1) * 4;
                spin_unlock(&mm->ioctx_lock);

                table = kzalloc(struct_size(table, table, new_nr), GFP_KERNEL);
                if (!table)
                        return -ENOMEM;

                table->nr = new_nr;

                spin_lock(&mm->ioctx_lock);
                old = rcu_dereference_raw(mm->ioctx_table);

                if (!old) {
                        rcu_assign_pointer(mm->ioctx_table, table);
                } else if (table->nr > old->nr) {
                        memcpy(table->table, old->table,
                               old->nr * sizeof(struct kioctx *));

                        rcu_assign_pointer(mm->ioctx_table, table);
                        kfree_rcu(old, rcu);
                } else {
                        kfree(table);
                        table = old;
                }
        }
}

static void aio_nr_sub(unsigned nr)
{
        spin_lock(&aio_nr_lock);
        if (WARN_ON(aio_nr - nr > aio_nr))
                aio_nr = 0;
        else
                aio_nr -= nr;
        spin_unlock(&aio_nr_lock);
}

/* ioctx_alloc
 *        Allocates and initializes an ioctx.  Returns an ERR_PTR if it failed.
 */
static struct kioctx *ioctx_alloc(unsigned nr_events)
{
        struct mm_struct *mm = current->mm;
        struct kioctx *ctx;
        int err = -ENOMEM;

        /*
         * Store the original nr_events -- what userspace passed to io_setup(),
         * for counting against the global limit -- before it changes.
         */
        unsigned int max_reqs = nr_events;

        /*
         * We keep track of the number of available ringbuffer slots, to prevent
         * overflow (reqs_available), and we also use percpu counters for this.
         *
         * So since up to half the slots might be on other cpu's percpu counters
         * and unavailable, double nr_events so userspace sees what they
         * expected: additionally, we move req_batch slots to/from percpu
         * counters at a time, so make sure that isn't 0:
         */
        nr_events = max(nr_events, num_possible_cpus() * 4);
        nr_events *= 2;

        /* Prevent overflows */
        if (nr_events > (0x10000000U / sizeof(struct io_event))) {
                pr_debug("ENOMEM: nr_events too high\n");
                return ERR_PTR(-EINVAL);
        }

        if (!nr_events || (unsigned long)max_reqs > aio_max_nr)
                return ERR_PTR(-EAGAIN);

        ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
        if (!ctx)
                return ERR_PTR(-ENOMEM);

        ctx->max_reqs = max_reqs;

        spin_lock_init(&ctx->ctx_lock);
        spin_lock_init(&ctx->completion_lock);
        mutex_init(&ctx->ring_lock);
        /* Protect against page migration throughout kiotx setup by keeping
         * the ring_lock mutex held until setup is complete. */
        mutex_lock(&ctx->ring_lock);
        init_waitqueue_head(&ctx->wait);

        INIT_LIST_HEAD(&ctx->active_reqs);

        if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL))
                goto err;

        if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL))
                goto err;

        ctx->cpu = alloc_percpu(struct kioctx_cpu);
        if (!ctx->cpu)
                goto err;

        err = aio_setup_ring(ctx, nr_events);
        if (err < 0)
                goto err;

        atomic_set(&ctx->reqs_available, ctx->nr_events - 1);
        ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4);
        if (ctx->req_batch < 1)
                ctx->req_batch = 1;

        /* limit the number of system wide aios */
        spin_lock(&aio_nr_lock);
        if (aio_nr + ctx->max_reqs > aio_max_nr ||
            aio_nr + ctx->max_reqs < aio_nr) {
                spin_unlock(&aio_nr_lock);
                err = -EAGAIN;
                goto err_ctx;
        }
        aio_nr += ctx->max_reqs;
        spin_unlock(&aio_nr_lock);

        percpu_ref_get(&ctx->users);        /* io_setup() will drop this ref */
        percpu_ref_get(&ctx->reqs);        /* free_ioctx_users() will drop this */

        err = ioctx_add_table(ctx, mm);
        if (err)
                goto err_cleanup;

        /* Release the ring_lock mutex now that all setup is complete. */
        mutex_unlock(&ctx->ring_lock);

        pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
                 ctx, ctx->user_id, mm, ctx->nr_events);
        return ctx;

err_cleanup:
        aio_nr_sub(ctx->max_reqs);
err_ctx:
        atomic_set(&ctx->dead, 1);
        if (ctx->mmap_size)
                vm_munmap(ctx->mmap_base, ctx->mmap_size);
        aio_free_ring(ctx);
err:
        mutex_unlock(&ctx->ring_lock);
        free_percpu(ctx->cpu);
        percpu_ref_exit(&ctx->reqs);
        percpu_ref_exit(&ctx->users);
        kmem_cache_free(kioctx_cachep, ctx);
        pr_debug("error allocating ioctx %d\n", err);
        return ERR_PTR(err);
}

/* kill_ioctx
 *        Cancels all outstanding aio requests on an aio context.  Used
 *        when the processes owning a context have all exited to encourage
 *        the rapid destruction of the kioctx.
 */
static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
                      struct ctx_rq_wait *wait)
{
        struct kioctx_table *table;

        spin_lock(&mm->ioctx_lock);
        if (atomic_xchg(&ctx->dead, 1)) {
                spin_unlock(&mm->ioctx_lock);
                return -EINVAL;
        }

        table = rcu_dereference_raw(mm->ioctx_table);
        WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id]));
        RCU_INIT_POINTER(table->table[ctx->id], NULL);
        spin_unlock(&mm->ioctx_lock);

        /* free_ioctx_reqs() will do the necessary RCU synchronization */
        wake_up_all(&ctx->wait);

        /*
         * It'd be more correct to do this in free_ioctx(), after all
         * the outstanding kiocbs have finished - but by then io_destroy
         * has already returned, so io_setup() could potentially return
         * -EAGAIN with no ioctxs actually in use (as far as userspace
         *  could tell).
         */
        aio_nr_sub(ctx->max_reqs);

        if (ctx->mmap_size)
                vm_munmap(ctx->mmap_base, ctx->mmap_size);

        ctx->rq_wait = wait;
        percpu_ref_kill(&ctx->users);
        return 0;
}

/*
 * exit_aio: called when the last user of mm goes away.  At this point, there is
 * no way for any new requests to be submited or any of the io_* syscalls to be
 * called on the context.
 *
 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
 * them.
 */
void exit_aio(struct mm_struct *mm)
{
        struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table);
        struct ctx_rq_wait wait;
        int i, skipped;

        if (!table)
                return;

        atomic_set(&wait.count, table->nr);
        init_completion(&wait.comp);

        skipped = 0;
        for (i = 0; i < table->nr; ++i) {
                struct kioctx *ctx =
                        rcu_dereference_protected(table->table[i], true);

                if (!ctx) {
                        skipped++;
                        continue;
                }

                /*
                 * We don't need to bother with munmap() here - exit_mmap(mm)
                 * is coming and it'll unmap everything. And we simply can't,
                 * this is not necessarily our ->mm.
                 * Since kill_ioctx() uses non-zero ->mmap_size as indicator
                 * that it needs to unmap the area, just set it to 0.
                 */
                ctx->mmap_size = 0;
                kill_ioctx(mm, ctx, &wait);
        }

        if (!atomic_sub_and_test(skipped, &wait.count)) {
                /* Wait until all IO for the context are done. */
                wait_for_completion(&wait.comp);
        }

        RCU_INIT_POINTER(mm->ioctx_table, NULL);
        kfree(table);
}

static void put_reqs_available(struct kioctx *ctx, unsigned nr)
{
        struct kioctx_cpu *kcpu;
        unsigned long flags;

        local_irq_save(flags);
        kcpu = this_cpu_ptr(ctx->cpu);
        kcpu->reqs_available += nr;

        while (kcpu->reqs_available >= ctx->req_batch * 2) {
                kcpu->reqs_available -= ctx->req_batch;
                atomic_add(ctx->req_batch, &ctx->reqs_available);
        }

        local_irq_restore(flags);
}

static bool __get_reqs_available(struct kioctx *ctx)
{
        struct kioctx_cpu *kcpu;
        bool ret = false;
        unsigned long flags;

        local_irq_save(flags);
        kcpu = this_cpu_ptr(ctx->cpu);
        if (!kcpu->reqs_available) {
                int avail = atomic_read(&ctx->reqs_available);

                do {
                        if (avail < ctx->req_batch)
                                goto out;
                } while (!atomic_try_cmpxchg(&ctx->reqs_available,
                                             &avail, avail - ctx->req_batch));

                kcpu->reqs_available += ctx->req_batch;
        }

        ret = true;
        kcpu->reqs_available--;
out:
        local_irq_restore(flags);
        return ret;
}

/* refill_reqs_available
 *        Updates the reqs_available reference counts used for tracking the
 *        number of free slots in the completion ring.  This can be called
 *        from aio_complete() (to optimistically update reqs_available) or
 *        from aio_get_req() (the we're out of events case).  It must be
 *        called holding ctx->completion_lock.
 */
static void refill_reqs_available(struct kioctx *ctx, unsigned head,
                                  unsigned tail)
{
        unsigned events_in_ring, completed;

        /* Clamp head since userland can write to it. */
        head %= ctx->nr_events;
        if (head <= tail)
                events_in_ring = tail - head;
        else
                events_in_ring = ctx->nr_events - (head - tail);

        completed = ctx->completed_events;
        if (events_in_ring < completed)
                completed -= events_in_ring;
        else
                completed = 0;

        if (!completed)
                return;

        ctx->completed_events -= completed;
        put_reqs_available(ctx, completed);
}

/* user_refill_reqs_available
 *        Called to refill reqs_available when aio_get_req() encounters an
 *        out of space in the completion ring.
 */
static void user_refill_reqs_available(struct kioctx *ctx)
{
        spin_lock_irq(&ctx->completion_lock);
        if (ctx->completed_events) {
                struct aio_ring *ring;
                unsigned head;

                /* Access of ring->head may race with aio_read_events_ring()
                 * here, but that's okay since whether we read the old version
                 * or the new version, and either will be valid.  The important
                 * part is that head cannot pass tail since we prevent
                 * aio_complete() from updating tail by holding
                 * ctx->completion_lock.  Even if head is invalid, the check
                 * against ctx->completed_events below will make sure we do the
                 * safe/right thing.
                 */
                ring = folio_address(ctx->ring_folios[0]);
                head = ring->head;

                refill_reqs_available(ctx, head, ctx->tail);
        }

        spin_unlock_irq(&ctx->completion_lock);
}

static bool get_reqs_available(struct kioctx *ctx)
{
        if (__get_reqs_available(ctx))
                return true;
        user_refill_reqs_available(ctx);
        return __get_reqs_available(ctx);
}

/* aio_get_req
 *        Allocate a slot for an aio request.
 * Returns NULL if no requests are free.
 *
 * The refcount is initialized to 2 - one for the async op completion,
 * one for the synchronous code that does this.
 */
static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
{
        struct aio_kiocb *req;

        req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
        if (unlikely(!req))
                return NULL;

        if (unlikely(!get_reqs_available(ctx))) {
                kmem_cache_free(kiocb_cachep, req);
                return NULL;
        }

        percpu_ref_get(&ctx->reqs);
        req->ki_ctx = ctx;
        INIT_LIST_HEAD(&req->ki_list);
        refcount_set(&req->ki_refcnt, 2);
        req->ki_eventfd = NULL;
        return req;
}

static struct kioctx *lookup_ioctx(unsigned long ctx_id)
{
        struct aio_ring __user *ring  = (void __user *)ctx_id;
        struct mm_struct *mm = current->mm;
        struct kioctx *ctx, *ret = NULL;
        struct kioctx_table *table;
        unsigned id;

        if (get_user(id, &ring->id))
                return NULL;

        rcu_read_lock();
        table = rcu_dereference(mm->ioctx_table);

        if (!table || id >= table->nr)
                goto out;

        id = array_index_nospec(id, table->nr);
        ctx = rcu_dereference(table->table[id]);
        if (ctx && ctx->user_id == ctx_id) {
                if (percpu_ref_tryget_live(&ctx->users))
                        ret = ctx;
        }
out:
        rcu_read_unlock();
        return ret;
}

static inline void iocb_destroy(struct aio_kiocb *iocb)
{
        if (iocb->ki_eventfd)
                eventfd_ctx_put(iocb->ki_eventfd);
        if (iocb->ki_filp)
                fput(iocb->ki_filp);
        percpu_ref_put(&iocb->ki_ctx->reqs);
        kmem_cache_free(kiocb_cachep, iocb);
}

struct aio_waiter {
        struct wait_queue_entry        w;
        size_t                        min_nr;
};

/* aio_complete
 *        Called when the io request on the given iocb is complete.
 */
static void aio_complete(struct aio_kiocb *iocb)
{
        struct kioctx        *ctx = iocb->ki_ctx;
        struct aio_ring        *ring;
        struct io_event        *ev_page, *event;
        unsigned tail, pos, head, avail;
        unsigned long        flags;

        /*
         * Add a completion event to the ring buffer. Must be done holding
         * ctx->completion_lock to prevent other code from messing with the tail
         * pointer since we might be called from irq context.
         */
        spin_lock_irqsave(&ctx->completion_lock, flags);

        tail = ctx->tail;
        pos = tail + AIO_EVENTS_OFFSET;

        if (++tail >= ctx->nr_events)
                tail = 0;

        ev_page = folio_address(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]);
        event = ev_page + pos % AIO_EVENTS_PER_PAGE;

        *event = iocb->ki_res;

        flush_dcache_folio(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]);

        pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb,
                 (void __user *)(unsigned long)iocb->ki_res.obj,
                 iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2);

        /* after flagging the request as done, we
         * must never even look at it again
         */
        smp_wmb();        /* make event visible before updating tail */

        ctx->tail = tail;

        ring = folio_address(ctx->ring_folios[0]);
        head = ring->head;
        ring->tail = tail;
        flush_dcache_folio(ctx->ring_folios[0]);

        ctx->completed_events++;
        if (ctx->completed_events > 1)
                refill_reqs_available(ctx, head, tail);

        avail = tail > head
                ? tail - head
                : tail + ctx->nr_events - head;
        spin_unlock_irqrestore(&ctx->completion_lock, flags);

        pr_debug("added to ring %p at [%u]\n", iocb, tail);

        /*
         * Check if the user asked us to deliver the result through an
         * eventfd. The eventfd_signal() function is safe to be called
         * from IRQ context.
         */
        if (iocb->ki_eventfd)
                eventfd_signal(iocb->ki_eventfd);

        /*
         * We have to order our ring_info tail store above and test
         * of the wait list below outside the wait lock.  This is
         * like in wake_up_bit() where clearing a bit has to be
         * ordered with the unlocked test.
         */
        smp_mb();

        if (waitqueue_active(&ctx->wait)) {
                struct aio_waiter *curr, *next;
                unsigned long flags;

                spin_lock_irqsave(&ctx->wait.lock, flags);
                list_for_each_entry_safe(curr, next, &ctx->wait.head, w.entry)
                        if (avail >= curr->min_nr) {
                                wake_up_process(curr->w.private);
                                list_del_init_careful(&curr->w.entry);
                        }
                spin_unlock_irqrestore(&ctx->wait.lock, flags);
        }
}

static inline void iocb_put(struct aio_kiocb *iocb)
{
        if (refcount_dec_and_test(&iocb->ki_refcnt)) {
                aio_complete(iocb);
                iocb_destroy(iocb);
        }
}

/* aio_read_events_ring
 *        Pull an event off of the ioctx's event ring.  Returns the number of
 *        events fetched
 */
static long aio_read_events_ring(struct kioctx *ctx,
                                 struct io_event __user *event, long nr)
{
        struct aio_ring *ring;
        unsigned head, tail, pos;
        long ret = 0;
        int copy_ret;

        /*
         * The mutex can block and wake us up and that will cause
         * wait_event_interruptible_hrtimeout() to schedule without sleeping
         * and repeat. This should be rare enough that it doesn't cause
         * peformance issues. See the comment in read_events() for more detail.
         */
        sched_annotate_sleep();
        mutex_lock(&ctx->ring_lock);

        /* Access to ->ring_folios here is protected by ctx->ring_lock. */
        ring = folio_address(ctx->ring_folios[0]);
        head = ring->head;
        tail = ring->tail;

        /*
         * Ensure that once we've read the current tail pointer, that
         * we also see the events that were stored up to the tail.
         */
        smp_rmb();

        pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events);

        if (head == tail)
                goto out;

        head %= ctx->nr_events;
        tail %= ctx->nr_events;

        while (ret < nr) {
                long avail;
                struct io_event *ev;
                struct folio *folio;

                avail = (head <= tail ?  tail : ctx->nr_events) - head;
                if (head == tail)
                        break;

                pos = head + AIO_EVENTS_OFFSET;
                folio = ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE];
                pos %= AIO_EVENTS_PER_PAGE;

                avail = min(avail, nr - ret);
                avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos);

                ev = folio_address(folio);
                copy_ret = copy_to_user(event + ret, ev + pos,
                                        sizeof(*ev) * avail);

                if (unlikely(copy_ret)) {
                        ret = -EFAULT;
                        goto out;
                }

                ret += avail;
                head += avail;
                head %= ctx->nr_events;
        }

        ring = folio_address(ctx->ring_folios[0]);
        ring->head = head;
        flush_dcache_folio(ctx->ring_folios[0]);

        pr_debug("%li  h%u t%u\n", ret, head, tail);
out:
        mutex_unlock(&ctx->ring_lock);

        return ret;
}

static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr,
                            struct io_event __user *event, long *i)
{
        long ret = aio_read_events_ring(ctx, event + *i, nr - *i);

        if (ret > 0)
                *i += ret;

        if (unlikely(atomic_read(&ctx->dead)))
                ret = -EINVAL;

        if (!*i)
                *i = ret;

        return ret < 0 || *i >= min_nr;
}

static long read_events(struct kioctx *ctx, long min_nr, long nr,
                        struct io_event __user *event,
                        ktime_t until)
{
        struct hrtimer_sleeper        t;
        struct aio_waiter        w;
        long ret = 0, ret2 = 0;

        /*
         * Note that aio_read_events() is being called as the conditional - i.e.
         * we're calling it after prepare_to_wait() has set task state to
         * TASK_INTERRUPTIBLE.
         *
         * But aio_read_events() can block, and if it blocks it's going to flip
         * the task state back to TASK_RUNNING.
         *
         * This should be ok, provided it doesn't flip the state back to
         * TASK_RUNNING and return 0 too much - that causes us to spin. That
         * will only happen if the mutex_lock() call blocks, and we then find
         * the ringbuffer empty. So in practice we should be ok, but it's
         * something to be aware of when touching this code.
         */
        aio_read_events(ctx, min_nr, nr, event, &ret);
        if (until == 0 || ret < 0 || ret >= min_nr)
                return ret;

        hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
        if (until != KTIME_MAX) {
                hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns);
                hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL);
        }

        init_wait(&w.w);

        while (1) {
                unsigned long nr_got = ret;

                w.min_nr = min_nr - ret;

                ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE);
                if (!ret2 && !t.task)
                        ret2 = -ETIME;

                if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2)
                        break;

                if (nr_got == ret)
                        schedule();
        }

        finish_wait(&ctx->wait, &w.w);
        hrtimer_cancel(&t.timer);
        destroy_hrtimer_on_stack(&t.timer);

        return ret;
}

/* sys_io_setup:
 *        Create an aio_context capable of receiving at least nr_events.
 *        ctxp must not point to an aio_context that already exists, and
 *        must be initialized to 0 prior to the call.  On successful
 *        creation of the aio_context, *ctxp is filled in with the resulting 
 *        handle.  May fail with -EINVAL if *ctxp is not initialized,
 *        if the specified nr_events exceeds internal limits.  May fail 
 *        with -EAGAIN if the specified nr_events exceeds the user's limit 
 *        of available events.  May fail with -ENOMEM if insufficient kernel
 *        resources are available.  May fail with -EFAULT if an invalid
 *        pointer is passed for ctxp.  Will fail with -ENOSYS if not
 *        implemented.
 */
SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
{
        struct kioctx *ioctx = NULL;
        unsigned long ctx;
        long ret;

        ret = get_user(ctx, ctxp);
        if (unlikely(ret))
                goto out;

        ret = -EINVAL;
        if (unlikely(ctx || nr_events == 0)) {
                pr_debug("EINVAL: ctx %lu nr_events %u\n",
                         ctx, nr_events);
                goto out;
        }

        ioctx = ioctx_alloc(nr_events);
        ret = PTR_ERR(ioctx);
        if (!IS_ERR(ioctx)) {
                ret = put_user(ioctx->user_id, ctxp);
                if (ret)
                        kill_ioctx(current->mm, ioctx, NULL);
                percpu_ref_put(&ioctx->users);
        }

out:
        return ret;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p)
{
        struct kioctx *ioctx = NULL;
        unsigned long ctx;
        long ret;

        ret = get_user(ctx, ctx32p);
        if (unlikely(ret))
                goto out;

        ret = -EINVAL;
        if (unlikely(ctx || nr_events == 0)) {
                pr_debug("EINVAL: ctx %lu nr_events %u\n",
                         ctx, nr_events);
                goto out;
        }

        ioctx = ioctx_alloc(nr_events);
        ret = PTR_ERR(ioctx);
        if (!IS_ERR(ioctx)) {
                /* truncating is ok because it's a user address */
                ret = put_user((u32)ioctx->user_id, ctx32p);
                if (ret)
                        kill_ioctx(current->mm, ioctx, NULL);
                percpu_ref_put(&ioctx->users);
        }

out:
        return ret;
}
#endif

/* sys_io_destroy:
 *        Destroy the aio_context specified.  May cancel any outstanding 
 *        AIOs and block on completion.  Will fail with -ENOSYS if not
 *        implemented.  May fail with -EINVAL if the context pointed to
 *        is invalid.
 */
SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
{
        struct kioctx *ioctx = lookup_ioctx(ctx);
        if (likely(NULL != ioctx)) {
                struct ctx_rq_wait wait;
                int ret;

                init_completion(&wait.comp);
                atomic_set(&wait.count, 1);

                /* Pass requests_done to kill_ioctx() where it can be set
                 * in a thread-safe way. If we try to set it here then we have
                 * a race condition if two io_destroy() called simultaneously.
                 */
                ret = kill_ioctx(current->mm, ioctx, &wait);
                percpu_ref_put(&ioctx->users);

                /* Wait until all IO for the context are done. Otherwise kernel
                 * keep using user-space buffers even if user thinks the context
                 * is destroyed.
                 */
                if (!ret)
                        wait_for_completion(&wait.comp);

                return ret;
        }
        pr_debug("EINVAL: invalid context id\n");
        return -EINVAL;
}

static void aio_remove_iocb(struct aio_kiocb *iocb)
{
        struct kioctx *ctx = iocb->ki_ctx;
        unsigned long flags;

        spin_lock_irqsave(&ctx->ctx_lock, flags);
        list_del(&iocb->ki_list);
        spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}

static void aio_complete_rw(struct kiocb *kiocb, long res)
{
        struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);

        if (!list_empty_careful(&iocb->ki_list))
                aio_remove_iocb(iocb);

        if (kiocb->ki_flags & IOCB_WRITE) {
                struct inode *inode = file_inode(kiocb->ki_filp);

                if (S_ISREG(inode->i_mode))
                        kiocb_end_write(kiocb);
        }

        iocb->ki_res.res = res;
        iocb->ki_res.res2 = 0;
        iocb_put(iocb);
}

static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb, int rw_type)
{
        int ret;

        req->ki_complete = aio_complete_rw;
        req->private = NULL;
        req->ki_pos = iocb->aio_offset;
        req->ki_flags = req->ki_filp->f_iocb_flags | IOCB_AIO_RW;
        if (iocb->aio_flags & IOCB_FLAG_RESFD)
                req->ki_flags |= IOCB_EVENTFD;
        if (iocb->aio_flags & IOCB_FLAG_IOPRIO) {
                /*
                 * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then
                 * aio_reqprio is interpreted as an I/O scheduling
                 * class and priority.
                 */
                ret = ioprio_check_cap(iocb->aio_reqprio);
                if (ret) {
                        pr_debug("aio ioprio check cap error: %d\n", ret);
                        return ret;
                }

                req->ki_ioprio = iocb->aio_reqprio;
        } else
                req->ki_ioprio = get_current_ioprio();

        ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags, rw_type);
        if (unlikely(ret))
                return ret;

        req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */
        return 0;
}

static ssize_t aio_setup_rw(int rw, const struct iocb *iocb,
                struct iovec **iovec, bool vectored, bool compat,
                struct iov_iter *iter)
{
        void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf;
        size_t len = iocb->aio_nbytes;

        if (!vectored) {
                ssize_t ret = import_ubuf(rw, buf, len, iter);
                *iovec = NULL;
                return ret;
        }

        return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat);
}

static inline void aio_rw_done(struct kiocb *req, ssize_t ret)
{
        switch (ret) {
        case -EIOCBQUEUED:
                break;
        case -ERESTARTSYS:
        case -ERESTARTNOINTR:
        case -ERESTARTNOHAND:
        case -ERESTART_RESTARTBLOCK:
                /*
                 * There's no easy way to restart the syscall since other AIO's
                 * may be already running. Just fail this IO with EINTR.
                 */
                ret = -EINTR;
                fallthrough;
        default:
                req->ki_complete(req, ret);
        }
}

static int aio_read(struct kiocb *req, const struct iocb *iocb,
                        bool vectored, bool compat)
{
        struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
        struct iov_iter iter;
        struct file *file;
        int ret;

        ret = aio_prep_rw(req, iocb, READ);
        if (ret)
                return ret;
        file = req->ki_filp;
        if (unlikely(!(file->f_mode & FMODE_READ)))
                return -EBADF;
        if (unlikely(!file->f_op->read_iter))
                return -EINVAL;

        ret = aio_setup_rw(ITER_DEST, iocb, &iovec, vectored, compat, &iter);
        if (ret < 0)
                return ret;
        ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
        if (!ret)
                aio_rw_done(req, file->f_op->read_iter(req, &iter));
        kfree(iovec);
        return ret;
}

static int aio_write(struct kiocb *req, const struct iocb *iocb,
                         bool vectored, bool compat)
{
        struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
        struct iov_iter iter;
        struct file *file;
        int ret;

        ret = aio_prep_rw(req, iocb, WRITE);
        if (ret)
                return ret;
        file = req->ki_filp;

        if (unlikely(!(file->f_mode & FMODE_WRITE)))
                return -EBADF;
        if (unlikely(!file->f_op->write_iter))
                return -EINVAL;

        ret = aio_setup_rw(ITER_SOURCE, iocb, &iovec, vectored, compat, &iter);
        if (ret < 0)
                return ret;
        ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
        if (!ret) {
                if (S_ISREG(file_inode(file)->i_mode))
                        kiocb_start_write(req);
                req->ki_flags |= IOCB_WRITE;
                aio_rw_done(req, file->f_op->write_iter(req, &iter));
        }
        kfree(iovec);
        return ret;
}

static void aio_fsync_work(struct work_struct *work)
{
        struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work);
        const struct cred *old_cred = override_creds(iocb->fsync.creds);

        iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync);
        revert_creds(old_cred);
        put_cred(iocb->fsync.creds);
        iocb_put(iocb);
}

static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb,
                     bool datasync)
{
        if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes ||
                        iocb->aio_rw_flags))
                return -EINVAL;

        if (unlikely(!req->file->f_op->fsync))
                return -EINVAL;

        req->creds = prepare_creds();
        if (!req->creds)
                return -ENOMEM;

        req->datasync = datasync;
        INIT_WORK(&req->work, aio_fsync_work);
        schedule_work(&req->work);
        return 0;
}

static void aio_poll_put_work(struct work_struct *work)
{
        struct poll_iocb *req = container_of(work, struct poll_iocb, work);
        struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);

        iocb_put(iocb);
}

/*
 * Safely lock the waitqueue which the request is on, synchronizing with the
 * case where the ->poll() provider decides to free its waitqueue early.
 *
 * Returns true on success, meaning that req->head->lock was locked, req->wait
 * is on req->head, and an RCU read lock was taken.  Returns false if the
 * request was already removed from its waitqueue (which might no longer exist).
 */
static bool poll_iocb_lock_wq(struct poll_iocb *req)
{
        wait_queue_head_t *head;

        /*
         * While we hold the waitqueue lock and the waitqueue is nonempty,
         * wake_up_pollfree() will wait for us.  However, taking the waitqueue
         * lock in the first place can race with the waitqueue being freed.
         *
         * We solve this as eventpoll does: by taking advantage of the fact that
         * all users of wake_up_pollfree() will RCU-delay the actual free.  If
         * we enter rcu_read_lock() and see that the pointer to the queue is
         * non-NULL, we can then lock it without the memory being freed out from
         * under us, then check whether the request is still on the queue.
         *
         * Keep holding rcu_read_lock() as long as we hold the queue lock, in
         * case the caller deletes the entry from the queue, leaving it empty.
         * In that case, only RCU prevents the queue memory from being freed.
         */
        rcu_read_lock();
        head = smp_load_acquire(&req->head);
        if (head) {
                spin_lock(&head->lock);
                if (!list_empty(&req->wait.entry))
                        return true;
                spin_unlock(&head->lock);
        }
        rcu_read_unlock();
        return false;
}

static void poll_iocb_unlock_wq(struct poll_iocb *req)
{
        spin_unlock(&req->head->lock);
        rcu_read_unlock();
}

static void aio_poll_complete_work(struct work_struct *work)
{
        struct poll_iocb *req = container_of(work, struct poll_iocb, work);
        struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
        struct poll_table_struct pt = { ._key = req->events };
        struct kioctx *ctx = iocb->ki_ctx;
        __poll_t mask = 0;

        if (!READ_ONCE(req->cancelled))
                mask = vfs_poll(req->file, &pt) & req->events;

        /*
         * Note that ->ki_cancel callers also delete iocb from active_reqs after
         * calling ->ki_cancel.  We need the ctx_lock roundtrip here to
         * synchronize with them.  In the cancellation case the list_del_init
         * itself is not actually needed, but harmless so we keep it in to
         * avoid further branches in the fast path.
         */
        spin_lock_irq(&ctx->ctx_lock);
        if (poll_iocb_lock_wq(req)) {
                if (!mask && !READ_ONCE(req->cancelled)) {
                        /*
                         * The request isn't actually ready to be completed yet.
                         * Reschedule completion if another wakeup came in.
                         */
                        if (req->work_need_resched) {
                                schedule_work(&req->work);
                                req->work_need_resched = false;
                        } else {
                                req->work_scheduled = false;
                        }
                        poll_iocb_unlock_wq(req);
                        spin_unlock_irq(&ctx->ctx_lock);
                        return;
                }
                list_del_init(&req->wait.entry);
                poll_iocb_unlock_wq(req);
        } /* else, POLLFREE has freed the waitqueue, so we must complete */
        list_del_init(&iocb->ki_list);
        iocb->ki_res.res = mangle_poll(mask);
        spin_unlock_irq(&ctx->ctx_lock);

        iocb_put(iocb);
}

/* assumes we are called with irqs disabled */
static int aio_poll_cancel(struct kiocb *iocb)
{
        struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw);
        struct poll_iocb *req = &aiocb->poll;

        if (poll_iocb_lock_wq(req)) {
                WRITE_ONCE(req->cancelled, true);
                if (!req->work_scheduled) {
                        schedule_work(&aiocb->poll.work);
                        req->work_scheduled = true;
                }
                poll_iocb_unlock_wq(req);
        } /* else, the request was force-cancelled by POLLFREE already */

        return 0;
}

static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
                void *key)
{
        struct poll_iocb *req = container_of(wait, struct poll_iocb, wait);
        struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll);
        __poll_t mask = key_to_poll(key);
        unsigned long flags;

        /* for instances that support it check for an event match first: */
        if (mask && !(mask & req->events))
                return 0;

        /*
         * Complete the request inline if possible.  This requires that three
         * conditions be met:
         *   1. An event mask must have been passed.  If a plain wakeup was done
         *        instead, then mask == 0 and we have to call vfs_poll() to get
         *        the events, so inline completion isn't possible.
         *   2. The completion work must not have already been scheduled.
         *   3. ctx_lock must not be busy.  We have to use trylock because we
         *        already hold the waitqueue lock, so this inverts the normal
         *        locking order.  Use irqsave/irqrestore because not all
         *        filesystems (e.g. fuse) call this function with IRQs disabled,
         *        yet IRQs have to be disabled before ctx_lock is obtained.
         */
        if (mask && !req->work_scheduled &&
            spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) {
                struct kioctx *ctx = iocb->ki_ctx;

                list_del_init(&req->wait.entry);
                list_del(&iocb->ki_list);
                iocb->ki_res.res = mangle_poll(mask);
                if (iocb->ki_eventfd && !eventfd_signal_allowed()) {
                        iocb = NULL;
                        INIT_WORK(&req->work, aio_poll_put_work);
                        schedule_work(&req->work);
                }
                spin_unlock_irqrestore(&ctx->ctx_lock, flags);
                if (iocb)
                        iocb_put(iocb);
        } else {
                /*
                 * Schedule the completion work if needed.  If it was already
                 * scheduled, record that another wakeup came in.
                 *
                 * Don't remove the request from the waitqueue here, as it might
                 * not actually be complete yet (we won't know until vfs_poll()
                 * is called), and we must not miss any wakeups.  POLLFREE is an
                 * exception to this; see below.
                 */
                if (req->work_scheduled) {
                        req->work_need_resched = true;
                } else {
                        schedule_work(&req->work);
                        req->work_scheduled = true;
                }

                /*
                 * If the waitqueue is being freed early but we can't complete
                 * the request inline, we have to tear down the request as best
                 * we can.  That means immediately removing the request from its
                 * waitqueue and preventing all further accesses to the
                 * waitqueue via the request.  We also need to schedule the
                 * completion work (done above).  Also mark the request as
                 * cancelled, to potentially skip an unneeded call to ->poll().
                 */
                if (mask & POLLFREE) {
                        WRITE_ONCE(req->cancelled, true);
                        list_del_init(&req->wait.entry);

                        /*
                         * Careful: this *must* be the last step, since as soon
                         * as req->head is NULL'ed out, the request can be
                         * completed and freed, since aio_poll_complete_work()
                         * will no longer need to take the waitqueue lock.
                         */
                        smp_store_release(&req->head, NULL);
                }
        }
        return 1;
}

struct aio_poll_table {
        struct poll_table_struct        pt;
        struct aio_kiocb                *iocb;
        bool                                queued;
        int                                error;
};

static void
aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,
                struct poll_table_struct *p)
{
        struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt);

        /* multiple wait queues per file are not supported */
        if (unlikely(pt->queued)) {
                pt->error = -EINVAL;
                return;
        }

        pt->queued = true;
        pt->error = 0;
        pt->iocb->poll.head = head;
        add_wait_queue(head, &pt->iocb->poll.wait);
}

static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb)
{
        struct kioctx *ctx = aiocb->ki_ctx;
        struct poll_iocb *req = &aiocb->poll;
        struct aio_poll_table apt;
        bool cancel = false;
        __poll_t mask;

        /* reject any unknown events outside the normal event mask. */
        if ((u16)iocb->aio_buf != iocb->aio_buf)
                return -EINVAL;
        /* reject fields that are not defined for poll */
        if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags)
                return -EINVAL;

        INIT_WORK(&req->work, aio_poll_complete_work);
        req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;

        req->head = NULL;
        req->cancelled = false;
        req->work_scheduled = false;
        req->work_need_resched = false;

        apt.pt._qproc = aio_poll_queue_proc;
        apt.pt._key = req->events;
        apt.iocb = aiocb;
        apt.queued = false;
        apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */

        /* initialized the list so that we can do list_empty checks */
        INIT_LIST_HEAD(&req->wait.entry);
        init_waitqueue_func_entry(&req->wait, aio_poll_wake);

        mask = vfs_poll(req->file, &apt.pt) & req->events;
        spin_lock_irq(&ctx->ctx_lock);
        if (likely(apt.queued)) {
                bool on_queue = poll_iocb_lock_wq(req);

                if (!on_queue || req->work_scheduled) {
                        /*
                         * aio_poll_wake() already either scheduled the async
                         * completion work, or completed the request inline.
                         */
                        if (apt.error) /* unsupported case: multiple queues */
                                cancel = true;
                        apt.error = 0;
                        mask = 0;
                }
                if (mask || apt.error) {
                        /* Steal to complete synchronously. */
                        list_del_init(&req->wait.entry);
                } else if (cancel) {
                        /* Cancel if possible (may be too late though). */
                        WRITE_ONCE(req->cancelled, true);
                } else if (on_queue) {
                        /*
                         * Actually waiting for an event, so add the request to
                         * active_reqs so that it can be cancelled if needed.
                         */
                        list_add_tail(&aiocb->ki_list, &ctx->active_reqs);
                        aiocb->ki_cancel = aio_poll_cancel;
                }
                if (on_queue)
                        poll_iocb_unlock_wq(req);
        }
        if (mask) { /* no async, we'd stolen it */
                aiocb->ki_res.res = mangle_poll(mask);
                apt.error = 0;
        }
        spin_unlock_irq(&ctx->ctx_lock);
        if (mask)
                iocb_put(aiocb);
        return apt.error;
}

static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb,
                           struct iocb __user *user_iocb, struct aio_kiocb *req,
                           bool compat)
{
        req->ki_filp = fget(iocb->aio_fildes);
        if (unlikely(!req->ki_filp))
                return -EBADF;

        if (iocb->aio_flags & IOCB_FLAG_RESFD) {
                struct eventfd_ctx *eventfd;
                /*
                 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
                 * instance of the file* now. The file descriptor must be
                 * an eventfd() fd, and will be signaled for each completed
                 * event using the eventfd_signal() function.
                 */
                eventfd = eventfd_ctx_fdget(iocb->aio_resfd);
                if (IS_ERR(eventfd))
                        return PTR_ERR(eventfd);

                req->ki_eventfd = eventfd;
        }

        if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) {
                pr_debug("EFAULT: aio_key\n");
                return -EFAULT;
        }

        req->ki_res.obj = (u64)(unsigned long)user_iocb;
        req->ki_res.data = iocb->aio_data;
        req->ki_res.res = 0;
        req->ki_res.res2 = 0;

        switch (iocb->aio_lio_opcode) {
        case IOCB_CMD_PREAD:
                return aio_read(&req->rw, iocb, false, compat);
        case IOCB_CMD_PWRITE:
                return aio_write(&req->rw, iocb, false, compat);
        case IOCB_CMD_PREADV:
                return aio_read(&req->rw, iocb, true, compat);
        case IOCB_CMD_PWRITEV:
                return aio_write(&req->rw, iocb, true, compat);
        case IOCB_CMD_FSYNC:
                return aio_fsync(&req->fsync, iocb, false);
        case IOCB_CMD_FDSYNC:
                return aio_fsync(&req->fsync, iocb, true);
        case IOCB_CMD_POLL:
                return aio_poll(req, iocb);
        default:
                pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode);
                return -EINVAL;
        }
}

static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
                         bool compat)
{
        struct aio_kiocb *req;
        struct iocb iocb;
        int err;

        if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb))))
                return -EFAULT;

        /* enforce forwards compatibility on users */
        if (unlikely(iocb.aio_reserved2)) {
                pr_debug("EINVAL: reserve field set\n");
                return -EINVAL;
        }

        /* prevent overflows */
        if (unlikely(
            (iocb.aio_buf != (unsigned long)iocb.aio_buf) ||
            (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) ||
            ((ssize_t)iocb.aio_nbytes < 0)
           )) {
                pr_debug("EINVAL: overflow check\n");
                return -EINVAL;
        }

        req = aio_get_req(ctx);
        if (unlikely(!req))
                return -EAGAIN;

        err = __io_submit_one(ctx, &iocb, user_iocb, req, compat);

        /* Done with the synchronous reference */
        iocb_put(req);

        /*
         * If err is 0, we'd either done aio_complete() ourselves or have
         * arranged for that to be done asynchronously.  Anything non-zero
         * means that we need to destroy req ourselves.
         */
        if (unlikely(err)) {
                iocb_destroy(req);
                put_reqs_available(ctx, 1);
        }
        return err;
}

/* sys_io_submit:
 *        Queue the nr iocbs pointed to by iocbpp for processing.  Returns
 *        the number of iocbs queued.  May return -EINVAL if the aio_context
 *        specified by ctx_id is invalid, if nr is < 0, if the iocb at
 *        *iocbpp[0] is not properly initialized, if the operation specified
 *        is invalid for the file descriptor in the iocb.  May fail with
 *        -EFAULT if any of the data structures point to invalid data.  May
 *        fail with -EBADF if the file descriptor specified in the first
 *        iocb is invalid.  May fail with -EAGAIN if insufficient resources
 *        are available to queue any iocbs.  Will return 0 if nr is 0.  Will
 *        fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
                struct iocb __user * __user *, iocbpp)
{
        struct kioctx *ctx;
        long ret = 0;
        int i = 0;
        struct blk_plug plug;

        if (unlikely(nr < 0))
                return -EINVAL;

        ctx = lookup_ioctx(ctx_id);
        if (unlikely(!ctx)) {
                pr_debug("EINVAL: invalid context id\n");
                return -EINVAL;
        }

        if (nr > ctx->nr_events)
                nr = ctx->nr_events;

        if (nr > AIO_PLUG_THRESHOLD)
                blk_start_plug(&plug);
        for (i = 0; i < nr; i++) {
                struct iocb __user *user_iocb;

                if (unlikely(get_user(user_iocb, iocbpp + i))) {
                        ret = -EFAULT;
                        break;
                }

                ret = io_submit_one(ctx, user_iocb, false);
                if (ret)
                        break;
        }
        if (nr > AIO_PLUG_THRESHOLD)
                blk_finish_plug(&plug);

        percpu_ref_put(&ctx->users);
        return i ? i : ret;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,
                       int, nr, compat_uptr_t __user *, iocbpp)
{
        struct kioctx *ctx;
        long ret = 0;
        int i = 0;
        struct blk_plug plug;

        if (unlikely(nr < 0))
                return -EINVAL;

        ctx = lookup_ioctx(ctx_id);
        if (unlikely(!ctx)) {
                pr_debug("EINVAL: invalid context id\n");
                return -EINVAL;
        }

        if (nr > ctx->nr_events)
                nr = ctx->nr_events;

        if (nr > AIO_PLUG_THRESHOLD)
                blk_start_plug(&plug);
        for (i = 0; i < nr; i++) {
                compat_uptr_t user_iocb;

                if (unlikely(get_user(user_iocb, iocbpp + i))) {
                        ret = -EFAULT;
                        break;
                }

                ret = io_submit_one(ctx, compat_ptr(user_iocb), true);
                if (ret)
                        break;
        }
        if (nr > AIO_PLUG_THRESHOLD)
                blk_finish_plug(&plug);

        percpu_ref_put(&ctx->users);
        return i ? i : ret;
}
#endif

/* sys_io_cancel:
 *        Attempts to cancel an iocb previously passed to io_submit.  If
 *        the operation is successfully cancelled, the resulting event is
 *        copied into the memory pointed to by result without being placed
 *        into the completion queue and 0 is returned.  May fail with
 *        -EFAULT if any of the data structures pointed to are invalid.
 *        May fail with -EINVAL if aio_context specified by ctx_id is
 *        invalid.  May fail with -EAGAIN if the iocb specified was not
 *        cancelled.  Will fail with -ENOSYS if not implemented.
 */
SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
                struct io_event __user *, result)
{
        struct kioctx *ctx;
        struct aio_kiocb *kiocb;
        int ret = -EINVAL;
        u32 key;
        u64 obj = (u64)(unsigned long)iocb;

        if (unlikely(get_user(key, &iocb->aio_key)))
                return -EFAULT;
        if (unlikely(key != KIOCB_KEY))
                return -EINVAL;

        ctx = lookup_ioctx(ctx_id);
        if (unlikely(!ctx))
                return -EINVAL;

        spin_lock_irq(&ctx->ctx_lock);
        list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) {
                if (kiocb->ki_res.obj == obj) {
                        ret = kiocb->ki_cancel(&kiocb->rw);
                        list_del_init(&kiocb->ki_list);
                        break;
                }
        }
        spin_unlock_irq(&ctx->ctx_lock);

        if (!ret) {
                /*
                 * The result argument is no longer used - the io_event is
                 * always delivered via the ring buffer. -EINPROGRESS indicates
                 * cancellation is progress:
                 */
                ret = -EINPROGRESS;
        }

        percpu_ref_put(&ctx->users);

        return ret;
}

static long do_io_getevents(aio_context_t ctx_id,
                long min_nr,
                long nr,
                struct io_event __user *events,
                struct timespec64 *ts)
{
        ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX;
        struct kioctx *ioctx = lookup_ioctx(ctx_id);
        long ret = -EINVAL;

        if (likely(ioctx)) {
                if (likely(min_nr <= nr && min_nr >= 0))
                        ret = read_events(ioctx, min_nr, nr, events, until);
                percpu_ref_put(&ioctx->users);
        }

        return ret;
}

/* io_getevents:
 *        Attempts to read at least min_nr events and up to nr events from
 *        the completion queue for the aio_context specified by ctx_id. If
 *        it succeeds, the number of read events is returned. May fail with
 *        -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is
 *        out of range, if timeout is out of range.  May fail with -EFAULT
 *        if any of the memory specified is invalid.  May return 0 or
 *        < min_nr if the timeout specified by timeout has elapsed
 *        before sufficient events are available, where timeout == NULL
 *        specifies an infinite timeout. Note that the timeout pointed to by
 *        timeout is relative.  Will fail with -ENOSYS if not implemented.
 */
#ifdef CONFIG_64BIT

SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
                long, min_nr,
                long, nr,
                struct io_event __user *, events,
                struct __kernel_timespec __user *, timeout)
{
        struct timespec64        ts;
        int                        ret;

        if (timeout && unlikely(get_timespec64(&ts, timeout)))
                return -EFAULT;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);
        if (!ret && signal_pending(current))
                ret = -EINTR;
        return ret;
}

#endif

struct __aio_sigset {
        const sigset_t __user        *sigmask;
        size_t                sigsetsize;
};

SYSCALL_DEFINE6(io_pgetevents,
                aio_context_t, ctx_id,
                long, min_nr,
                long, nr,
                struct io_event __user *, events,
                struct __kernel_timespec __user *, timeout,
                const struct __aio_sigset __user *, usig)
{
        struct __aio_sigset        ksig = { NULL, };
        struct timespec64        ts;
        bool interrupted;
        int ret;

        if (timeout && unlikely(get_timespec64(&ts, timeout)))
                return -EFAULT;

        if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
                return -EFAULT;

        ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize);
        if (ret)
                return ret;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);

        interrupted = signal_pending(current);
        restore_saved_sigmask_unless(interrupted);
        if (interrupted && !ret)
                ret = -ERESTARTNOHAND;

        return ret;
}

#if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT)

SYSCALL_DEFINE6(io_pgetevents_time32,
                aio_context_t, ctx_id,
                long, min_nr,
                long, nr,
                struct io_event __user *, events,
                struct old_timespec32 __user *, timeout,
                const struct __aio_sigset __user *, usig)
{
        struct __aio_sigset        ksig = { NULL, };
        struct timespec64        ts;
        bool interrupted;
        int ret;

        if (timeout && unlikely(get_old_timespec32(&ts, timeout)))
                return -EFAULT;

        if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
                return -EFAULT;


        ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize);
        if (ret)
                return ret;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL);

        interrupted = signal_pending(current);
        restore_saved_sigmask_unless(interrupted);
        if (interrupted && !ret)
                ret = -ERESTARTNOHAND;

        return ret;
}

#endif

#if defined(CONFIG_COMPAT_32BIT_TIME)

SYSCALL_DEFINE5(io_getevents_time32, __u32, ctx_id,
                __s32, min_nr,
                __s32, nr,
                struct io_event __user *, events,
                struct old_timespec32 __user *, timeout)
{
        struct timespec64 t;
        int ret;

        if (timeout && get_old_timespec32(&t, timeout))
                return -EFAULT;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);
        if (!ret && signal_pending(current))
                ret = -EINTR;
        return ret;
}

#endif

#ifdef CONFIG_COMPAT

struct __compat_aio_sigset {
        compat_uptr_t                sigmask;
        compat_size_t                sigsetsize;
};

#if defined(CONFIG_COMPAT_32BIT_TIME)

COMPAT_SYSCALL_DEFINE6(io_pgetevents,
                compat_aio_context_t, ctx_id,
                compat_long_t, min_nr,
                compat_long_t, nr,
                struct io_event __user *, events,
                struct old_timespec32 __user *, timeout,
                const struct __compat_aio_sigset __user *, usig)
{
        struct __compat_aio_sigset ksig = { 0, };
        struct timespec64 t;
        bool interrupted;
        int ret;

        if (timeout && get_old_timespec32(&t, timeout))
                return -EFAULT;

        if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
                return -EFAULT;

        ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize);
        if (ret)
                return ret;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);

        interrupted = signal_pending(current);
        restore_saved_sigmask_unless(interrupted);
        if (interrupted && !ret)
                ret = -ERESTARTNOHAND;

        return ret;
}

#endif

COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64,
                compat_aio_context_t, ctx_id,
                compat_long_t, min_nr,
                compat_long_t, nr,
                struct io_event __user *, events,
                struct __kernel_timespec __user *, timeout,
                const struct __compat_aio_sigset __user *, usig)
{
        struct __compat_aio_sigset ksig = { 0, };
        struct timespec64 t;
        bool interrupted;
        int ret;

        if (timeout && get_timespec64(&t, timeout))
                return -EFAULT;

        if (usig && copy_from_user(&ksig, usig, sizeof(ksig)))
                return -EFAULT;

        ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize);
        if (ret)
                return ret;

        ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL);

        interrupted = signal_pending(current);
        restore_saved_sigmask_unless(interrupted);
        if (interrupted && !ret)
                ret = -ERESTARTNOHAND;

        return ret;
}
#endif

















































































































 10180 





































































































































































































































































































































































































































































































 10172 







































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
// SPDX-License-Identifier: GPL-2.0
/*
 * Detect hard and soft lockups on a system
 *
 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
 *
 * Note: Most of this code is borrowed heavily from the original softlockup
 * detector, so thanks to Ingo for the initial implementation.
 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
 * to those contributors as well.
 */

#define pr_fmt(fmt) "watchdog: " fmt

#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/kernel_stat.h>
#include <linux/kvm_para.h>
#include <linux/math64.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/stop_machine.h>
#include <linux/sysctl.h>
#include <linux/tick.h>

#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>

#include <asm/irq_regs.h>

static DEFINE_MUTEX(watchdog_mutex);

#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)
# define WATCHDOG_HARDLOCKUP_DEFAULT        1
#else
# define WATCHDOG_HARDLOCKUP_DEFAULT        0
#endif

#define NUM_SAMPLE_PERIODS        5

unsigned long __read_mostly watchdog_enabled;
int __read_mostly watchdog_user_enabled = 1;
static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;
static int __read_mostly watchdog_softlockup_user_enabled = 1;
int __read_mostly watchdog_thresh = 10;
static int __read_mostly watchdog_hardlockup_available;

struct cpumask watchdog_cpumask __read_mostly;
unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);

#ifdef CONFIG_HARDLOCKUP_DETECTOR

# ifdef CONFIG_SMP
int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
# endif /* CONFIG_SMP */

/*
 * Should we panic when a soft-lockup or hard-lockup occurs:
 */
unsigned int __read_mostly hardlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
/*
 * We may not want to enable hard lockup detection by default in all cases,
 * for example when running the kernel as a guest on a hypervisor. In these
 * cases this function can be called to disable hard lockup detection. This
 * function should only be executed once by the boot processor before the
 * kernel command line parameters are parsed, because otherwise it is not
 * possible to override this in hardlockup_panic_setup().
 */
void __init hardlockup_detector_disable(void)
{
        watchdog_hardlockup_user_enabled = 0;
}

static int __init hardlockup_panic_setup(char *str)
{
next:
        if (!strncmp(str, "panic", 5))
                hardlockup_panic = 1;
        else if (!strncmp(str, "nopanic", 7))
                hardlockup_panic = 0;
        else if (!strncmp(str, "0", 1))
                watchdog_hardlockup_user_enabled = 0;
        else if (!strncmp(str, "1", 1))
                watchdog_hardlockup_user_enabled = 1;
        else if (!strncmp(str, "r", 1))
                hardlockup_config_perf_event(str + 1);
        while (*(str++)) {
                if (*str == ',') {
                        str++;
                        goto next;
                }
        }
        return 1;
}
__setup("nmi_watchdog=", hardlockup_panic_setup);

#endif /* CONFIG_HARDLOCKUP_DETECTOR */

#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)

static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
static unsigned long hard_lockup_nmi_warn;

notrace void arch_touch_nmi_watchdog(void)
{
        /*
         * Using __raw here because some code paths have
         * preemption enabled.  If preemption is enabled
         * then interrupts should be enabled too, in which
         * case we shouldn't have to worry about the watchdog
         * going off.
         */
        raw_cpu_write(watchdog_hardlockup_touched, true);
}
EXPORT_SYMBOL(arch_touch_nmi_watchdog);

void watchdog_hardlockup_touch_cpu(unsigned int cpu)
{
        per_cpu(watchdog_hardlockup_touched, cpu) = true;
}

static bool is_hardlockup(unsigned int cpu)
{
        int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));

        if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
                return true;

        /*
         * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
         * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
         * written/read by a single CPU.
         */
        per_cpu(hrtimer_interrupts_saved, cpu) = hrint;

        return false;
}

static void watchdog_hardlockup_kick(void)
{
        int new_interrupts;

        new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));
        watchdog_buddy_check_hardlockup(new_interrupts);
}

void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
{
        if (per_cpu(watchdog_hardlockup_touched, cpu)) {
                per_cpu(watchdog_hardlockup_touched, cpu) = false;
                return;
        }

        /*
         * Check for a hardlockup by making sure the CPU's timer
         * interrupt is incrementing. The timer interrupt should have
         * fired multiple times before we overflow'd. If it hasn't
         * then this is a good indication the cpu is stuck
         */
        if (is_hardlockup(cpu)) {
                unsigned int this_cpu = smp_processor_id();
                unsigned long flags;

                /* Only print hardlockups once. */
                if (per_cpu(watchdog_hardlockup_warned, cpu))
                        return;

                /*
                 * Prevent multiple hard-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (sysctl_hardlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
                                return;
                }

                /*
                 * NOTE: we call printk_cpu_sync_get_irqsave() after printing
                 * the lockup message. While it would be nice to serialize
                 * that printout, we really want to make sure that if some
                 * other CPU somehow locked up while holding the lock associated
                 * with printk_cpu_sync_get_irqsave() that we can still at least
                 * get the message about the lockup out.
                 */
                pr_emerg("CPU%u: Watchdog detected hard LOCKUP on cpu %u\n", this_cpu, cpu);
                printk_cpu_sync_get_irqsave(flags);

                print_modules();
                print_irqtrace_events(current);
                if (cpu == this_cpu) {
                        if (regs)
                                show_regs(regs);
                        else
                                dump_stack();
                        printk_cpu_sync_put_irqrestore(flags);
                } else {
                        printk_cpu_sync_put_irqrestore(flags);
                        trigger_single_cpu_backtrace(cpu);
                }

                if (sysctl_hardlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(cpu);
                        if (!hardlockup_panic)
                                clear_bit_unlock(0, &hard_lockup_nmi_warn);
                }

                if (hardlockup_panic)
                        nmi_panic(regs, "Hard LOCKUP");

                per_cpu(watchdog_hardlockup_warned, cpu) = true;
        } else {
                per_cpu(watchdog_hardlockup_warned, cpu) = false;
        }
}

#else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

static inline void watchdog_hardlockup_kick(void) { }

#endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

/*
 * These functions can be overridden based on the configured hardlockdup detector.
 *
 * watchdog_hardlockup_enable/disable can be implemented to start and stop when
 * softlockup watchdog start and stop. The detector must select the
 * SOFTLOCKUP_DETECTOR Kconfig.
 */
void __weak watchdog_hardlockup_enable(unsigned int cpu) { }

void __weak watchdog_hardlockup_disable(unsigned int cpu) { }

/*
 * Watchdog-detector specific API.
 *
 * Return 0 when hardlockup watchdog is available, negative value otherwise.
 * Note that the negative value means that a delayed probe might
 * succeed later.
 */
int __weak __init watchdog_hardlockup_probe(void)
{
        return -ENODEV;
}

/**
 * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration
 *
 * The reconfiguration steps are:
 * watchdog_hardlockup_stop();
 * update_variables();
 * watchdog_hardlockup_start();
 */
void __weak watchdog_hardlockup_stop(void) { }

/**
 * watchdog_hardlockup_start - Start the watchdog after reconfiguration
 *
 * Counterpart to watchdog_hardlockup_stop().
 *
 * The following variables have been updated in update_variables() and
 * contain the currently valid configuration:
 * - watchdog_enabled
 * - watchdog_thresh
 * - watchdog_cpumask
 */
void __weak watchdog_hardlockup_start(void) { }

/**
 * lockup_detector_update_enable - Update the sysctl enable bit
 *
 * Caller needs to make sure that the hard watchdogs are off, so this
 * can't race with watchdog_hardlockup_disable().
 */
static void lockup_detector_update_enable(void)
{
        watchdog_enabled = 0;
        if (!watchdog_user_enabled)
                return;
        if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
        if (watchdog_softlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR

/*
 * Delay the soflockup report when running a known slow code.
 * It does _not_ affect the timestamp of the last successdul reschedule.
 */
#define SOFTLOCKUP_DELAY_REPORT        ULONG_MAX

#ifdef CONFIG_SMP
int __read_mostly sysctl_softlockup_all_cpu_backtrace;
#endif

static struct cpumask watchdog_allowed_mask __read_mostly;

/* Global variables, exported for sysctl */
unsigned int __read_mostly softlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);

static bool softlockup_initialized __read_mostly;
static u64 __read_mostly sample_period;

/* Timestamp taken after the last successful reschedule. */
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
/* Timestamp of the last softlockup report. */
static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static unsigned long soft_lockup_nmi_warn;

static int __init softlockup_panic_setup(char *str)
{
        softlockup_panic = simple_strtoul(str, NULL, 0);
        return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);

static int __init nowatchdog_setup(char *str)
{
        watchdog_user_enabled = 0;
        return 1;
}
__setup("nowatchdog", nowatchdog_setup);

static int __init nosoftlockup_setup(char *str)
{
        watchdog_softlockup_user_enabled = 0;
        return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);

static int __init watchdog_thresh_setup(char *str)
{
        get_option(&str, &watchdog_thresh);
        return 1;
}
__setup("watchdog_thresh=", watchdog_thresh_setup);

static void __lockup_detector_cleanup(void);

#ifdef CONFIG_SOFTLOCKUP_DETECTOR_INTR_STORM
enum stats_per_group {
        STATS_SYSTEM,
        STATS_SOFTIRQ,
        STATS_HARDIRQ,
        STATS_IDLE,
        NUM_STATS_PER_GROUP,
};

static const enum cpu_usage_stat tracked_stats[NUM_STATS_PER_GROUP] = {
        CPUTIME_SYSTEM,
        CPUTIME_SOFTIRQ,
        CPUTIME_IRQ,
        CPUTIME_IDLE,
};

static DEFINE_PER_CPU(u16, cpustat_old[NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_util[NUM_SAMPLE_PERIODS][NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_tail);

/*
 * We don't need nanosecond resolution. A granularity of 16ms is
 * sufficient for our precision, allowing us to use u16 to store
 * cpustats, which will roll over roughly every ~1000 seconds.
 * 2^24 ~= 16 * 10^6
 */
static u16 get_16bit_precision(u64 data_ns)
{
        return data_ns >> 24LL; /* 2^24ns ~= 16.8ms */
}

static void update_cpustat(void)
{
        int i;
        u8 util;
        u16 old_stat, new_stat;
        struct kernel_cpustat kcpustat;
        u64 *cpustat = kcpustat.cpustat;
        u8 tail = __this_cpu_read(cpustat_tail);
        u16 sample_period_16 = get_16bit_precision(sample_period);

        kcpustat_cpu_fetch(&kcpustat, smp_processor_id());

        for (i = 0; i < NUM_STATS_PER_GROUP; i++) {
                old_stat = __this_cpu_read(cpustat_old[i]);
                new_stat = get_16bit_precision(cpustat[tracked_stats[i]]);
                util = DIV_ROUND_UP(100 * (new_stat - old_stat), sample_period_16);
                __this_cpu_write(cpustat_util[tail][i], util);
                __this_cpu_write(cpustat_old[i], new_stat);
        }

        __this_cpu_write(cpustat_tail, (tail + 1) % NUM_SAMPLE_PERIODS);
}

static void print_cpustat(void)
{
        int i, group;
        u8 tail = __this_cpu_read(cpustat_tail);
        u64 sample_period_second = sample_period;

        do_div(sample_period_second, NSEC_PER_SEC);

        /*
         * Outputting the "watchdog" prefix on every line is redundant and not
         * concise, and the original alarm information is sufficient for
         * positioning in logs, hence here printk() is used instead of pr_crit().
         */
        printk(KERN_CRIT "CPU#%d Utilization every %llus during lockup:\n",
               smp_processor_id(), sample_period_second);

        for (i = 0; i < NUM_SAMPLE_PERIODS; i++) {
                group = (tail + i) % NUM_SAMPLE_PERIODS;
                printk(KERN_CRIT "\t#%d: %3u%% system,\t%3u%% softirq,\t"
                        "%3u%% hardirq,\t%3u%% idle\n", i + 1,
                        __this_cpu_read(cpustat_util[group][STATS_SYSTEM]),
                        __this_cpu_read(cpustat_util[group][STATS_SOFTIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_HARDIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_IDLE]));
        }
}

#define HARDIRQ_PERCENT_THRESH          50
#define NUM_HARDIRQ_REPORT              5
struct irq_counts {
        int irq;
        u32 counts;
};

static DEFINE_PER_CPU(bool, snapshot_taken);

/* Tabulate the most frequent interrupts. */
static void tabulate_irq_count(struct irq_counts *irq_counts, int irq, u32 counts, int rank)
{
        int i;
        struct irq_counts new_count = {irq, counts};

        for (i = 0; i < rank; i++) {
                if (counts > irq_counts[i].counts)
                        swap(new_count, irq_counts[i]);
        }
}

/*
 * If the hardirq time exceeds HARDIRQ_PERCENT_THRESH% of the sample_period,
 * then the cause of softlockup might be interrupt storm. In this case, it
 * would be useful to start interrupt counting.
 */
static bool need_counting_irqs(void)
{
        u8 util;
        int tail = __this_cpu_read(cpustat_tail);

        tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT;
        util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]);
        return util > HARDIRQ_PERCENT_THRESH;
}

static void start_counting_irqs(void)
{
        if (!__this_cpu_read(snapshot_taken)) {
                kstat_snapshot_irqs();
                __this_cpu_write(snapshot_taken, true);
        }
}

static void stop_counting_irqs(void)
{
        __this_cpu_write(snapshot_taken, false);
}

static void print_irq_counts(void)
{
        unsigned int i, count;
        struct irq_counts irq_counts_sorted[NUM_HARDIRQ_REPORT] = {
                {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}
        };

        if (__this_cpu_read(snapshot_taken)) {
                for_each_active_irq(i) {
                        count = kstat_get_irq_since_snapshot(i);
                        tabulate_irq_count(irq_counts_sorted, i, count, NUM_HARDIRQ_REPORT);
                }

                /*
                 * Outputting the "watchdog" prefix on every line is redundant and not
                 * concise, and the original alarm information is sufficient for
                 * positioning in logs, hence here printk() is used instead of pr_crit().
                 */
                printk(KERN_CRIT "CPU#%d Detect HardIRQ Time exceeds %d%%. Most frequent HardIRQs:\n",
                       smp_processor_id(), HARDIRQ_PERCENT_THRESH);

                for (i = 0; i < NUM_HARDIRQ_REPORT; i++) {
                        if (irq_counts_sorted[i].irq == -1)
                                break;

                        printk(KERN_CRIT "\t#%u: %-10u\tirq#%d\n",
                               i + 1, irq_counts_sorted[i].counts,
                               irq_counts_sorted[i].irq);
                }

                /*
                 * If the hardirq time is less than HARDIRQ_PERCENT_THRESH% in the last
                 * sample_period, then we suspect the interrupt storm might be subsiding.
                 */
                if (!need_counting_irqs())
                        stop_counting_irqs();
        }
}

static void report_cpu_status(void)
{
        print_cpustat();
        print_irq_counts();
}
#else
static inline void update_cpustat(void) { }
static inline void report_cpu_status(void) { }
static inline bool need_counting_irqs(void) { return false; }
static inline void start_counting_irqs(void) { }
static inline void stop_counting_irqs(void) { }
#endif

/*
 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
 * lockups can have false positives under extreme conditions. So we generally
 * want a higher threshold for soft lockups than for hard lockups. So we couple
 * the thresholds with a factor: we make the soft threshold twice the amount of
 * time the hard threshold is.
 */
static int get_softlockup_thresh(void)
{
        return watchdog_thresh * 2;
}

/*
 * Returns seconds, approximately.  We don't need nanosecond
 * resolution, and we don't need to waste time with a big divide when
 * 2^30ns == 1.074s.
 */
static unsigned long get_timestamp(void)
{
        return running_clock() >> 30LL;  /* 2^30 ~= 10^9 */
}

static void set_sample_period(void)
{
        /*
         * convert watchdog_thresh from seconds to ns
         * the divide by 5 is to give hrtimer several chances (two
         * or three with the current relation between the soft
         * and hard thresholds) to increment before the
         * hardlockup detector generates a warning
         */
        sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / NUM_SAMPLE_PERIODS);
        watchdog_update_hrtimer_threshold(sample_period);
}

static void update_report_ts(void)
{
        __this_cpu_write(watchdog_report_ts, get_timestamp());
}

/* Commands for resetting the watchdog */
static void update_touch_ts(void)
{
        __this_cpu_write(watchdog_touch_ts, get_timestamp());
        update_report_ts();
}

/**
 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
 *
 * Call when the scheduler may have stalled for legitimate reasons
 * preventing the watchdog task from executing - e.g. the scheduler
 * entering idle state.  This should only be used for scheduler events.
 * Use touch_softlockup_watchdog() for everything else.
 */
notrace void touch_softlockup_watchdog_sched(void)
{
        /*
         * Preemption can be enabled.  It doesn't matter which CPU's watchdog
         * report period gets restarted here, so use the raw_ operation.
         */
        raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

notrace void touch_softlockup_watchdog(void)
{
        touch_softlockup_watchdog_sched();
        wq_watchdog_touch(raw_smp_processor_id());
}
EXPORT_SYMBOL(touch_softlockup_watchdog);

void touch_all_softlockup_watchdogs(void)
{
        int cpu;

        /*
         * watchdog_mutex cannpt be taken here, as this might be called
         * from (soft)interrupt context, so the access to
         * watchdog_allowed_cpumask might race with a concurrent update.
         *
         * The watchdog time stamp can race against a concurrent real
         * update as well, the only side effect might be a cycle delay for
         * the softlockup check.
         */
        for_each_cpu(cpu, &watchdog_allowed_mask) {
                per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
                wq_watchdog_touch(cpu);
        }
}

void touch_softlockup_watchdog_sync(void)
{
        __this_cpu_write(softlockup_touch_sync, true);
        __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

static int is_softlockup(unsigned long touch_ts,
                         unsigned long period_ts,
                         unsigned long now)
{
        if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
                /*
                 * If period_ts has not been updated during a sample_period, then
                 * in the subsequent few sample_periods, period_ts might also not
                 * be updated, which could indicate a potential softlockup. In
                 * this case, if we suspect the cause of the potential softlockup
                 * might be interrupt storm, then we need to count the interrupts
                 * to find which interrupt is storming.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() / NUM_SAMPLE_PERIODS) &&
                    need_counting_irqs())
                        start_counting_irqs();

                /*
                 * A poorly behaving BPF scheduler can live-lock the system into
                 * soft lockups. Tell sched_ext to try ejecting the BPF
                 * scheduler when close to a soft lockup.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() * 3 / 4))
                        scx_softlockup(now - touch_ts);

                /* Warn about unreasonable delays. */
                if (time_after(now, period_ts + get_softlockup_thresh()))
                        return now - touch_ts;
        }
        return 0;
}

/* watchdog detector functions */
static DEFINE_PER_CPU(struct completion, softlockup_completion);
static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);

/*
 * The watchdog feed function - touches the timestamp.
 *
 * It only runs once every sample_period seconds (4 seconds by
 * default) to reset the softlockup timestamp. If this gets delayed
 * for more than 2*watchdog_thresh seconds then the debug-printout
 * triggers in watchdog_timer_fn().
 */
static int softlockup_fn(void *data)
{
        update_touch_ts();
        stop_counting_irqs();
        complete(this_cpu_ptr(&softlockup_completion));

        return 0;
}

/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
        unsigned long touch_ts, period_ts, now;
        struct pt_regs *regs = get_irq_regs();
        int duration;
        int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
        unsigned long flags;

        if (!watchdog_enabled)
                return HRTIMER_NORESTART;

        watchdog_hardlockup_kick();

        /* kick the softlockup detector */
        if (completion_done(this_cpu_ptr(&softlockup_completion))) {
                reinit_completion(this_cpu_ptr(&softlockup_completion));
                stop_one_cpu_nowait(smp_processor_id(),
                                softlockup_fn, NULL,
                                this_cpu_ptr(&softlockup_stop_work));
        }

        /* .. and repeat */
        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

        /*
         * Read the current timestamp first. It might become invalid anytime
         * when a virtual machine is stopped by the host or when the watchog
         * is touched from NMI.
         */
        now = get_timestamp();
        /*
         * If a virtual machine is stopped by the host it can look to
         * the watchdog like a soft lockup. This function touches the watchdog.
         */
        kvm_check_and_clear_guest_paused();
        /*
         * The stored timestamp is comparable with @now only when not touched.
         * It might get touched anytime from NMI. Make sure that is_softlockup()
         * uses the same (valid) value.
         */
        period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));

        update_cpustat();

        /* Reset the interval when touched by known problematic code. */
        if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
                if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
                        /*
                         * If the time stamp was touched atomically
                         * make sure the scheduler tick is up to date.
                         */
                        __this_cpu_write(softlockup_touch_sync, false);
                        sched_clock_tick();
                }

                update_report_ts();
                return HRTIMER_RESTART;
        }

        /* Check for a softlockup. */
        touch_ts = __this_cpu_read(watchdog_touch_ts);
        duration = is_softlockup(touch_ts, period_ts, now);
        if (unlikely(duration)) {
                /*
                 * Prevent multiple soft-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (softlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
                                return HRTIMER_RESTART;
                }

                /* Start period for the next softlockup warning. */
                update_report_ts();

                printk_cpu_sync_get_irqsave(flags);
                pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
                        smp_processor_id(), duration,
                        current->comm, task_pid_nr(current));
                report_cpu_status();
                print_modules();
                print_irqtrace_events(current);
                if (regs)
                        show_regs(regs);
                else
                        dump_stack();
                printk_cpu_sync_put_irqrestore(flags);

                if (softlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(smp_processor_id());
                        if (!softlockup_panic)
                                clear_bit_unlock(0, &soft_lockup_nmi_warn);
                }

                add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
                if (softlockup_panic)
                        panic("softlockup: hung tasks");
        }

        return HRTIMER_RESTART;
}

static void watchdog_enable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
        struct completion *done = this_cpu_ptr(&softlockup_completion);

        WARN_ON_ONCE(cpu != smp_processor_id());

        init_completion(done);
        complete(done);

        /*
         * Start the timer first to prevent the hardlockup watchdog triggering
         * before the timer has a chance to fire.
         */
        hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
        hrtimer->function = watchdog_timer_fn;
        hrtimer_start(hrtimer, ns_to_ktime(sample_period),
                      HRTIMER_MODE_REL_PINNED_HARD);

        /* Initialize timestamp */
        update_touch_ts();
        /* Enable the hardlockup detector */
        if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)
                watchdog_hardlockup_enable(cpu);
}

static void watchdog_disable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);

        WARN_ON_ONCE(cpu != smp_processor_id());

        /*
         * Disable the hardlockup detector first. That prevents that a large
         * delay between disabling the timer and disabling the hardlockup
         * detector causes a false positive.
         */
        watchdog_hardlockup_disable(cpu);
        hrtimer_cancel(hrtimer);
        wait_for_completion(this_cpu_ptr(&softlockup_completion));
}

static int softlockup_stop_fn(void *data)
{
        watchdog_disable(smp_processor_id());
        return 0;
}

static void softlockup_stop_all(void)
{
        int cpu;

        if (!softlockup_initialized)
                return;

        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);

        cpumask_clear(&watchdog_allowed_mask);
}

static int softlockup_start_fn(void *data)
{
        watchdog_enable(smp_processor_id());
        return 0;
}

static void softlockup_start_all(void)
{
        int cpu;

        cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
}

int lockup_detector_online_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_enable(cpu);
        return 0;
}

int lockup_detector_offline_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_disable(cpu);
        return 0;
}

static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();

        softlockup_stop_all();
        set_sample_period();
        lockup_detector_update_enable();
        if (watchdog_enabled && watchdog_thresh)
                softlockup_start_all();

        watchdog_hardlockup_start();
        cpus_read_unlock();
        /*
         * Must be called outside the cpus locked section to prevent
         * recursive locking in the perf code.
         */
        __lockup_detector_cleanup();
}

void lockup_detector_reconfigure(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        mutex_unlock(&watchdog_mutex);
}

/*
 * Create the watchdog infrastructure and configure the detector(s).
 */
static __init void lockup_detector_setup(void)
{
        /*
         * If sysctl is off and watchdog got disabled on the command line,
         * nothing to do here.
         */
        lockup_detector_update_enable();

        if (!IS_ENABLED(CONFIG_SYSCTL) &&
            !(watchdog_enabled && watchdog_thresh))
                return;

        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        softlockup_initialized = true;
        mutex_unlock(&watchdog_mutex);
}

#else /* CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();
        lockup_detector_update_enable();
        watchdog_hardlockup_start();
        cpus_read_unlock();
}
void lockup_detector_reconfigure(void)
{
        __lockup_detector_reconfigure();
}
static inline void lockup_detector_setup(void)
{
        __lockup_detector_reconfigure();
}
#endif /* !CONFIG_SOFTLOCKUP_DETECTOR */

static void __lockup_detector_cleanup(void)
{
        lockdep_assert_held(&watchdog_mutex);
        hardlockup_detector_perf_cleanup();
}

/**
 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
 *
 * Caller must not hold the cpu hotplug rwsem.
 */
void lockup_detector_cleanup(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_cleanup();
        mutex_unlock(&watchdog_mutex);
}

/**
 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
 *
 * Special interface for parisc. It prevents lockup detector warnings from
 * the default pm_poweroff() function which busy loops forever.
 */
void lockup_detector_soft_poweroff(void)
{
        watchdog_enabled = 0;
}

#ifdef CONFIG_SYSCTL

/* Propagate any changes to the watchdog infrastructure */
static void proc_watchdog_update(void)
{
        /* Remove impossible cpus to keep sysctl output clean. */
        cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
        __lockup_detector_reconfigure();
}

/*
 * common function for watchdog, nmi_watchdog and soft_watchdog parameter
 *
 * caller             | table->data points to            | 'which'
 * -------------------|----------------------------------|-------------------------------
 * proc_watchdog      | watchdog_user_enabled            | WATCHDOG_HARDLOCKUP_ENABLED |
 *                    |                                  | WATCHDOG_SOFTOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_nmi_watchdog  | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
 */
static int proc_watchdog_common(int which, const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old, *param = table->data;

        mutex_lock(&watchdog_mutex);

        old = *param;
        if (!write) {
                /*
                 * On read synchronize the userspace interface. This is a
                 * racy snapshot.
                 */
                *param = (watchdog_enabled & which) != 0;
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                *param = old;
        } else {
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                if (!err && old != READ_ONCE(*param))
                        proc_watchdog_update();
        }
        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * /proc/sys/kernel/watchdog
 */
static int proc_watchdog(const struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
                                    WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/nmi_watchdog
 */
static int proc_nmi_watchdog(const struct ctl_table *table, int write,
                             void *buffer, size_t *lenp, loff_t *ppos)
{
        if (!watchdog_hardlockup_available && write)
                return -ENOTSUPP;
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR
/*
 * /proc/sys/kernel/soft_watchdog
 */
static int proc_soft_watchdog(const struct ctl_table *table, int write,
                              void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}
#endif

/*
 * /proc/sys/kernel/watchdog_thresh
 */
static int proc_watchdog_thresh(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old;

        mutex_lock(&watchdog_mutex);

        old = READ_ONCE(watchdog_thresh);
        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

        if (!err && write && old != READ_ONCE(watchdog_thresh))
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * The cpumask is the mask of possible cpus that the watchdog can run
 * on, not the mask of cpus it is actually running on.  This allows the
 * user to specify a mask that will include cpus that have not yet
 * been brought online, if desired.
 */
static int proc_watchdog_cpumask(const struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        int err;

        mutex_lock(&watchdog_mutex);

        err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
        if (!err && write)
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

static const int sixty = 60;

static const struct ctl_table watchdog_sysctls[] = {
        {
                .procname       = "watchdog",
                .data                = &watchdog_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "watchdog_thresh",
                .data                = &watchdog_thresh,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_watchdog_thresh,
                .extra1                = SYSCTL_ZERO,
                .extra2                = (void *)&sixty,
        },
        {
                .procname        = "watchdog_cpumask",
                .data                = &watchdog_cpumask_bits,
                .maxlen                = NR_CPUS,
                .mode                = 0644,
                .proc_handler        = proc_watchdog_cpumask,
        },
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
        {
                .procname       = "soft_watchdog",
                .data                = &watchdog_softlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_soft_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "softlockup_panic",
                .data                = &softlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "softlockup_all_cpu_backtrace",
                .data                = &sysctl_softlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
#ifdef CONFIG_HARDLOCKUP_DETECTOR
        {
                .procname        = "hardlockup_panic",
                .data                = &hardlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "hardlockup_all_cpu_backtrace",
                .data                = &sysctl_hardlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
};

static struct ctl_table watchdog_hardlockup_sysctl[] = {
        {
                .procname       = "nmi_watchdog",
                .data                = &watchdog_hardlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0444,
                .proc_handler   = proc_nmi_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
};

static void __init watchdog_sysctl_init(void)
{
        register_sysctl_init("kernel", watchdog_sysctls);

        if (watchdog_hardlockup_available)
                watchdog_hardlockup_sysctl[0].mode = 0644;
        register_sysctl_init("kernel", watchdog_hardlockup_sysctl);
}

#else
#define watchdog_sysctl_init() do { } while (0)
#endif /* CONFIG_SYSCTL */

static void __init lockup_detector_delay_init(struct work_struct *work);
static bool allow_lockup_detector_init_retry __initdata;

static struct work_struct detector_work __initdata =
                __WORK_INITIALIZER(detector_work, lockup_detector_delay_init);

static void __init lockup_detector_delay_init(struct work_struct *work)
{
        int ret;

        ret = watchdog_hardlockup_probe();
        if (ret) {
                if (ret == -ENODEV)
                        pr_info("NMI not fully supported\n");
                else
                        pr_info("Delayed init of the lockup detector failed: %d\n", ret);
                pr_info("Hard watchdog permanently disabled\n");
                return;
        }

        allow_lockup_detector_init_retry = false;

        watchdog_hardlockup_available = true;
        lockup_detector_setup();
}

/*
 * lockup_detector_retry_init - retry init lockup detector if possible.
 *
 * Retry hardlockup detector init. It is useful when it requires some
 * functionality that has to be initialized later on a particular
 * platform.
 */
void __init lockup_detector_retry_init(void)
{
        /* Must be called before late init calls */
        if (!allow_lockup_detector_init_retry)
                return;

        schedule_work(&detector_work);
}

/*
 * Ensure that optional delayed hardlockup init is proceed before
 * the init code and memory is freed.
 */
static int __init lockup_detector_check(void)
{
        /* Prevent any later retry. */
        allow_lockup_detector_init_retry = false;

        /* Make sure no work is pending. */
        flush_work(&detector_work);

        watchdog_sysctl_init();

        return 0;

}
late_initcall_sync(lockup_detector_check);

void __init lockup_detector_init(void)
{
        if (tick_nohz_full_enabled())
                pr_info("Disabling watchdog on nohz_full cores by default\n");

        cpumask_copy(&watchdog_cpumask,
                     housekeeping_cpumask(HK_TYPE_TIMER));

        if (!watchdog_hardlockup_probe())
                watchdog_hardlockup_available = true;
        else
                allow_lockup_detector_init_retry = true;

        lockup_detector_setup();
}






















    3 






    3 
    3 


    3 









































































































































































































































    3 





















































































































































    7 






    7 


















































































































































    3 




























    3 



    3 
    3 


















































































    4 




































































    5 





    5 
    5 


    5 





    5 


    3 
    2 










    4 











    4 









    3 






    3 

























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */
#include <linux/statfs.h>
#include <linux/seq_file.h>
#include <linux/crc32c.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/exportfs.h>
#include <linux/backing-dev.h>
#include "xattr.h"

#define CREATE_TRACE_POINTS
#include <trace/events/erofs.h>

static struct kmem_cache *erofs_inode_cachep __read_mostly;

void _erofs_printk(struct super_block *sb, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;
        int level;

        va_start(args, fmt);

        level = printk_get_level(fmt);
        vaf.fmt = printk_skip_level(fmt);
        vaf.va = &args;
        if (sb)
                printk("%c%cerofs (device %s): %pV",
                                KERN_SOH_ASCII, level, sb->s_id, &vaf);
        else
                printk("%c%cerofs: %pV", KERN_SOH_ASCII, level, &vaf);
        va_end(args);
}

static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata)
{
        struct erofs_super_block *dsb = sbdata + EROFS_SUPER_OFFSET;
        u32 len = 1 << EROFS_SB(sb)->blkszbits, crc;

        if (len > EROFS_SUPER_OFFSET)
                len -= EROFS_SUPER_OFFSET;
        len -= offsetof(struct erofs_super_block, checksum) +
                        sizeof(dsb->checksum);

        /* skip .magic(pre-verified) and .checksum(0) fields */
        crc = crc32c(0x5045B54A, (&dsb->checksum) + 1, len);
        if (crc == le32_to_cpu(dsb->checksum))
                return 0;
        erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected",
                  crc, le32_to_cpu(dsb->checksum));
        return -EBADMSG;
}

static void erofs_inode_init_once(void *ptr)
{
        struct erofs_inode *vi = ptr;

        inode_init_once(&vi->vfs_inode);
}

static struct inode *erofs_alloc_inode(struct super_block *sb)
{
        struct erofs_inode *vi =
                alloc_inode_sb(sb, erofs_inode_cachep, GFP_KERNEL);

        if (!vi)
                return NULL;

        /* zero out everything except vfs_inode */
        memset(vi, 0, offsetof(struct erofs_inode, vfs_inode));
        return &vi->vfs_inode;
}

static void erofs_free_inode(struct inode *inode)
{
        struct erofs_inode *vi = EROFS_I(inode);

        if (inode->i_op == &erofs_fast_symlink_iops)
                kfree(inode->i_link);
        kfree(vi->xattr_shared_xattrs);
        kmem_cache_free(erofs_inode_cachep, vi);
}

/* read variable-sized metadata, offset will be aligned by 4-byte */
void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf,
                          erofs_off_t *offset, int *lengthp)
{
        u8 *buffer, *ptr;
        int len, i, cnt;

        *offset = round_up(*offset, 4);
        ptr = erofs_bread(buf, *offset, EROFS_KMAP);
        if (IS_ERR(ptr))
                return ptr;

        len = le16_to_cpu(*(__le16 *)ptr);
        if (!len)
                len = U16_MAX + 1;
        buffer = kmalloc(len, GFP_KERNEL);
        if (!buffer)
                return ERR_PTR(-ENOMEM);
        *offset += sizeof(__le16);
        *lengthp = len;

        for (i = 0; i < len; i += cnt) {
                cnt = min_t(int, sb->s_blocksize - erofs_blkoff(sb, *offset),
                            len - i);
                ptr = erofs_bread(buf, *offset, EROFS_KMAP);
                if (IS_ERR(ptr)) {
                        kfree(buffer);
                        return ptr;
                }
                memcpy(buffer + i, ptr, cnt);
                *offset += cnt;
        }
        return buffer;
}

#ifndef CONFIG_EROFS_FS_ZIP
static int z_erofs_parse_cfgs(struct super_block *sb,
                              struct erofs_super_block *dsb)
{
        if (!dsb->u1.available_compr_algs)
                return 0;

        erofs_err(sb, "compression disabled, unable to mount compressed EROFS");
        return -EOPNOTSUPP;
}
#endif

static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb,
                             struct erofs_device_info *dif, erofs_off_t *pos)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_fscache *fscache;
        struct erofs_deviceslot *dis;
        struct file *file;

        dis = erofs_read_metabuf(buf, sb, *pos, EROFS_KMAP);
        if (IS_ERR(dis))
                return PTR_ERR(dis);

        if (!sbi->devs->flatdev && !dif->path) {
                if (!dis->tag[0]) {
                        erofs_err(sb, "empty device tag @ pos %llu", *pos);
                        return -EINVAL;
                }
                dif->path = kmemdup_nul(dis->tag, sizeof(dis->tag), GFP_KERNEL);
                if (!dif->path)
                        return -ENOMEM;
        }

        if (erofs_is_fscache_mode(sb)) {
                fscache = erofs_fscache_register_cookie(sb, dif->path, 0);
                if (IS_ERR(fscache))
                        return PTR_ERR(fscache);
                dif->fscache = fscache;
        } else if (!sbi->devs->flatdev) {
                file = erofs_is_fileio_mode(sbi) ?
                                filp_open(dif->path, O_RDONLY | O_LARGEFILE, 0) :
                                bdev_file_open_by_path(dif->path,
                                                BLK_OPEN_READ, sb->s_type, NULL);
                if (IS_ERR(file))
                        return PTR_ERR(file);

                if (!erofs_is_fileio_mode(sbi)) {
                        dif->dax_dev = fs_dax_get_by_bdev(file_bdev(file),
                                        &dif->dax_part_off, NULL, NULL);
                } else if (!S_ISREG(file_inode(file)->i_mode)) {
                        fput(file);
                        return -EINVAL;
                }
                dif->file = file;
        }

        dif->blocks = le32_to_cpu(dis->blocks);
        dif->mapped_blkaddr = le32_to_cpu(dis->mapped_blkaddr);
        sbi->total_blocks += dif->blocks;
        *pos += EROFS_DEVT_SLOT_SIZE;
        return 0;
}

static int erofs_scan_devices(struct super_block *sb,
                              struct erofs_super_block *dsb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        unsigned int ondisk_extradevs;
        erofs_off_t pos;
        struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
        struct erofs_device_info *dif;
        int id, err = 0;

        sbi->total_blocks = sbi->dif0.blocks;
        if (!erofs_sb_has_device_table(sbi))
                ondisk_extradevs = 0;
        else
                ondisk_extradevs = le16_to_cpu(dsb->extra_devices);

        if (sbi->devs->extra_devices &&
            ondisk_extradevs != sbi->devs->extra_devices) {
                erofs_err(sb, "extra devices don't match (ondisk %u, given %u)",
                          ondisk_extradevs, sbi->devs->extra_devices);
                return -EINVAL;
        }
        if (!ondisk_extradevs)
                return 0;

        if (!sbi->devs->extra_devices && !erofs_is_fscache_mode(sb))
                sbi->devs->flatdev = true;

        sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
        pos = le16_to_cpu(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
        down_read(&sbi->devs->rwsem);
        if (sbi->devs->extra_devices) {
                idr_for_each_entry(&sbi->devs->tree, dif, id) {
                        err = erofs_init_device(&buf, sb, dif, &pos);
                        if (err)
                                break;
                }
        } else {
                for (id = 0; id < ondisk_extradevs; id++) {
                        dif = kzalloc(sizeof(*dif), GFP_KERNEL);
                        if (!dif) {
                                err = -ENOMEM;
                                break;
                        }

                        err = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL);
                        if (err < 0) {
                                kfree(dif);
                                break;
                        }
                        ++sbi->devs->extra_devices;

                        err = erofs_init_device(&buf, sb, dif, &pos);
                        if (err)
                                break;
                }
        }
        up_read(&sbi->devs->rwsem);
        erofs_put_metabuf(&buf);
        return err;
}

static int erofs_read_superblock(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
        struct erofs_super_block *dsb;
        void *data;
        int ret;

        data = erofs_read_metabuf(&buf, sb, 0, EROFS_KMAP);
        if (IS_ERR(data)) {
                erofs_err(sb, "cannot read erofs superblock");
                return PTR_ERR(data);
        }

        dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET);
        ret = -EINVAL;
        if (le32_to_cpu(dsb->magic) != EROFS_SUPER_MAGIC_V1) {
                erofs_err(sb, "cannot find valid erofs superblock");
                goto out;
        }

        sbi->blkszbits  = dsb->blkszbits;
        if (sbi->blkszbits < 9 || sbi->blkszbits > PAGE_SHIFT) {
                erofs_err(sb, "blkszbits %u isn't supported", sbi->blkszbits);
                goto out;
        }
        if (dsb->dirblkbits) {
                erofs_err(sb, "dirblkbits %u isn't supported", dsb->dirblkbits);
                goto out;
        }

        sbi->feature_compat = le32_to_cpu(dsb->feature_compat);
        if (erofs_sb_has_sb_chksum(sbi)) {
                ret = erofs_superblock_csum_verify(sb, data);
                if (ret)
                        goto out;
        }

        ret = -EINVAL;
        sbi->feature_incompat = le32_to_cpu(dsb->feature_incompat);
        if (sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT) {
                erofs_err(sb, "unidentified incompatible feature %x, please upgrade kernel",
                          sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT);
                goto out;
        }

        sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE;
        if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) {
                erofs_err(sb, "invalid sb_extslots %u (more than a fs block)",
                          sbi->sb_size);
                goto out;
        }
        sbi->dif0.blocks = le32_to_cpu(dsb->blocks);
        sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
#ifdef CONFIG_EROFS_FS_XATTR
        sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
        sbi->xattr_prefix_start = le32_to_cpu(dsb->xattr_prefix_start);
        sbi->xattr_prefix_count = dsb->xattr_prefix_count;
        sbi->xattr_filter_reserved = dsb->xattr_filter_reserved;
#endif
        sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact));
        sbi->root_nid = le16_to_cpu(dsb->root_nid);
        sbi->packed_nid = le64_to_cpu(dsb->packed_nid);
        sbi->inos = le64_to_cpu(dsb->inos);

        sbi->build_time = le64_to_cpu(dsb->build_time);
        sbi->build_time_nsec = le32_to_cpu(dsb->build_time_nsec);

        super_set_uuid(sb, (void *)dsb->uuid, sizeof(dsb->uuid));

        ret = strscpy(sbi->volume_name, dsb->volume_name,
                      sizeof(dsb->volume_name));
        if (ret < 0) {        /* -E2BIG */
                erofs_err(sb, "bad volume name without NIL terminator");
                ret = -EFSCORRUPTED;
                goto out;
        }

        /* parse on-disk compression configurations */
        ret = z_erofs_parse_cfgs(sb, dsb);
        if (ret < 0)
                goto out;

        /* handle multiple devices */
        ret = erofs_scan_devices(sb, dsb);

        if (erofs_is_fscache_mode(sb))
                erofs_info(sb, "[deprecated] fscache-based on-demand read feature in use. Use at your own risk!");
out:
        erofs_put_metabuf(&buf);
        return ret;
}

static void erofs_default_options(struct erofs_sb_info *sbi)
{
#ifdef CONFIG_EROFS_FS_ZIP
        sbi->opt.cache_strategy = EROFS_ZIP_CACHE_READAROUND;
        sbi->opt.max_sync_decompress_pages = 3;
        sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_AUTO;
#endif
#ifdef CONFIG_EROFS_FS_XATTR
        set_opt(&sbi->opt, XATTR_USER);
#endif
#ifdef CONFIG_EROFS_FS_POSIX_ACL
        set_opt(&sbi->opt, POSIX_ACL);
#endif
}

enum {
        Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum,
        Opt_device, Opt_fsid, Opt_domain_id, Opt_directio,
        Opt_err
};

static const struct constant_table erofs_param_cache_strategy[] = {
        {"disabled",        EROFS_ZIP_CACHE_DISABLED},
        {"readahead",        EROFS_ZIP_CACHE_READAHEAD},
        {"readaround",        EROFS_ZIP_CACHE_READAROUND},
        {}
};

static const struct constant_table erofs_dax_param_enums[] = {
        {"always",        EROFS_MOUNT_DAX_ALWAYS},
        {"never",        EROFS_MOUNT_DAX_NEVER},
        {}
};

static const struct fs_parameter_spec erofs_fs_parameters[] = {
        fsparam_flag_no("user_xattr",        Opt_user_xattr),
        fsparam_flag_no("acl",                Opt_acl),
        fsparam_enum("cache_strategy",        Opt_cache_strategy,
                     erofs_param_cache_strategy),
        fsparam_flag("dax",             Opt_dax),
        fsparam_enum("dax",                Opt_dax_enum, erofs_dax_param_enums),
        fsparam_string("device",        Opt_device),
        fsparam_string("fsid",                Opt_fsid),
        fsparam_string("domain_id",        Opt_domain_id),
        fsparam_flag_no("directio",        Opt_directio),
        {}
};

static bool erofs_fc_set_dax_mode(struct fs_context *fc, unsigned int mode)
{
#ifdef CONFIG_FS_DAX
        struct erofs_sb_info *sbi = fc->s_fs_info;

        switch (mode) {
        case EROFS_MOUNT_DAX_ALWAYS:
                set_opt(&sbi->opt, DAX_ALWAYS);
                clear_opt(&sbi->opt, DAX_NEVER);
                return true;
        case EROFS_MOUNT_DAX_NEVER:
                set_opt(&sbi->opt, DAX_NEVER);
                clear_opt(&sbi->opt, DAX_ALWAYS);
                return true;
        default:
                DBG_BUGON(1);
                return false;
        }
#else
        errorfc(fc, "dax options not supported");
        return false;
#endif
}

static int erofs_fc_parse_param(struct fs_context *fc,
                                struct fs_parameter *param)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;
        struct fs_parse_result result;
        struct erofs_device_info *dif;
        int opt, ret;

        opt = fs_parse(fc, erofs_fs_parameters, param, &result);
        if (opt < 0)
                return opt;

        switch (opt) {
        case Opt_user_xattr:
#ifdef CONFIG_EROFS_FS_XATTR
                if (result.boolean)
                        set_opt(&sbi->opt, XATTR_USER);
                else
                        clear_opt(&sbi->opt, XATTR_USER);
#else
                errorfc(fc, "{,no}user_xattr options not supported");
#endif
                break;
        case Opt_acl:
#ifdef CONFIG_EROFS_FS_POSIX_ACL
                if (result.boolean)
                        set_opt(&sbi->opt, POSIX_ACL);
                else
                        clear_opt(&sbi->opt, POSIX_ACL);
#else
                errorfc(fc, "{,no}acl options not supported");
#endif
                break;
        case Opt_cache_strategy:
#ifdef CONFIG_EROFS_FS_ZIP
                sbi->opt.cache_strategy = result.uint_32;
#else
                errorfc(fc, "compression not supported, cache_strategy ignored");
#endif
                break;
        case Opt_dax:
                if (!erofs_fc_set_dax_mode(fc, EROFS_MOUNT_DAX_ALWAYS))
                        return -EINVAL;
                break;
        case Opt_dax_enum:
                if (!erofs_fc_set_dax_mode(fc, result.uint_32))
                        return -EINVAL;
                break;
        case Opt_device:
                dif = kzalloc(sizeof(*dif), GFP_KERNEL);
                if (!dif)
                        return -ENOMEM;
                dif->path = kstrdup(param->string, GFP_KERNEL);
                if (!dif->path) {
                        kfree(dif);
                        return -ENOMEM;
                }
                down_write(&sbi->devs->rwsem);
                ret = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL);
                up_write(&sbi->devs->rwsem);
                if (ret < 0) {
                        kfree(dif->path);
                        kfree(dif);
                        return ret;
                }
                ++sbi->devs->extra_devices;
                break;
#ifdef CONFIG_EROFS_FS_ONDEMAND
        case Opt_fsid:
                kfree(sbi->fsid);
                sbi->fsid = kstrdup(param->string, GFP_KERNEL);
                if (!sbi->fsid)
                        return -ENOMEM;
                break;
        case Opt_domain_id:
                kfree(sbi->domain_id);
                sbi->domain_id = kstrdup(param->string, GFP_KERNEL);
                if (!sbi->domain_id)
                        return -ENOMEM;
                break;
#else
        case Opt_fsid:
        case Opt_domain_id:
                errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name);
                break;
#endif
        case Opt_directio:
#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
                if (result.boolean)
                        set_opt(&sbi->opt, DIRECT_IO);
                else
                        clear_opt(&sbi->opt, DIRECT_IO);
#else
                errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name);
#endif
                break;
        }
        return 0;
}

static struct inode *erofs_nfs_get_inode(struct super_block *sb,
                                         u64 ino, u32 generation)
{
        return erofs_iget(sb, ino);
}

static struct dentry *erofs_fh_to_dentry(struct super_block *sb,
                struct fid *fid, int fh_len, int fh_type)
{
        return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
                                    erofs_nfs_get_inode);
}

static struct dentry *erofs_fh_to_parent(struct super_block *sb,
                struct fid *fid, int fh_len, int fh_type)
{
        return generic_fh_to_parent(sb, fid, fh_len, fh_type,
                                    erofs_nfs_get_inode);
}

static struct dentry *erofs_get_parent(struct dentry *child)
{
        erofs_nid_t nid;
        unsigned int d_type;
        int err;

        err = erofs_namei(d_inode(child), &dotdot_name, &nid, &d_type);
        if (err)
                return ERR_PTR(err);
        return d_obtain_alias(erofs_iget(child->d_sb, nid));
}

static const struct export_operations erofs_export_ops = {
        .encode_fh = generic_encode_ino32_fh,
        .fh_to_dentry = erofs_fh_to_dentry,
        .fh_to_parent = erofs_fh_to_parent,
        .get_parent = erofs_get_parent,
};

static void erofs_set_sysfs_name(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        if (sbi->domain_id)
                super_set_sysfs_name_generic(sb, "%s,%s", sbi->domain_id,
                                             sbi->fsid);
        else if (sbi->fsid)
                super_set_sysfs_name_generic(sb, "%s", sbi->fsid);
        else if (erofs_is_fileio_mode(sbi))
                super_set_sysfs_name_generic(sb, "%s",
                                             bdi_dev_name(sb->s_bdi));
        else
                super_set_sysfs_name_id(sb);
}

static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct inode *inode;
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        int err;

        sb->s_magic = EROFS_SUPER_MAGIC;
        sb->s_flags |= SB_RDONLY | SB_NOATIME;
        sb->s_maxbytes = MAX_LFS_FILESIZE;
        sb->s_op = &erofs_sops;

        sbi->blkszbits = PAGE_SHIFT;
        if (!sb->s_bdev) {
                sb->s_blocksize = PAGE_SIZE;
                sb->s_blocksize_bits = PAGE_SHIFT;

                if (erofs_is_fscache_mode(sb)) {
                        err = erofs_fscache_register_fs(sb);
                        if (err)
                                return err;
                }
                err = super_setup_bdi(sb);
                if (err)
                        return err;
        } else {
                if (!sb_set_blocksize(sb, PAGE_SIZE)) {
                        errorfc(fc, "failed to set initial blksize");
                        return -EINVAL;
                }

                sbi->dif0.dax_dev = fs_dax_get_by_bdev(sb->s_bdev,
                                &sbi->dif0.dax_part_off, NULL, NULL);
        }

        err = erofs_read_superblock(sb);
        if (err)
                return err;

        if (sb->s_blocksize_bits != sbi->blkszbits) {
                if (erofs_is_fscache_mode(sb)) {
                        errorfc(fc, "unsupported blksize for fscache mode");
                        return -EINVAL;
                }

                if (erofs_is_fileio_mode(sbi)) {
                        sb->s_blocksize = 1 << sbi->blkszbits;
                        sb->s_blocksize_bits = sbi->blkszbits;
                } else if (!sb_set_blocksize(sb, 1 << sbi->blkszbits)) {
                        errorfc(fc, "failed to set erofs blksize");
                        return -EINVAL;
                }
        }

        if (test_opt(&sbi->opt, DAX_ALWAYS)) {
                if (!sbi->dif0.dax_dev) {
                        errorfc(fc, "DAX unsupported by block device. Turning off DAX.");
                        clear_opt(&sbi->opt, DAX_ALWAYS);
                } else if (sbi->blkszbits != PAGE_SHIFT) {
                        errorfc(fc, "unsupported blocksize for DAX");
                        clear_opt(&sbi->opt, DAX_ALWAYS);
                }
        }

        sb->s_time_gran = 1;
        sb->s_xattr = erofs_xattr_handlers;
        sb->s_export_op = &erofs_export_ops;

        if (test_opt(&sbi->opt, POSIX_ACL))
                sb->s_flags |= SB_POSIXACL;
        else
                sb->s_flags &= ~SB_POSIXACL;

#ifdef CONFIG_EROFS_FS_ZIP
        xa_init(&sbi->managed_pslots);
#endif

        inode = erofs_iget(sb, sbi->root_nid);
        if (IS_ERR(inode))
                return PTR_ERR(inode);

        if (!S_ISDIR(inode->i_mode)) {
                erofs_err(sb, "rootino(nid %llu) is not a directory(i_mode %o)",
                          sbi->root_nid, inode->i_mode);
                iput(inode);
                return -EINVAL;
        }

        sb->s_root = d_make_root(inode);
        if (!sb->s_root)
                return -ENOMEM;

        erofs_shrinker_register(sb);
        if (erofs_sb_has_fragments(sbi) && sbi->packed_nid) {
                sbi->packed_inode = erofs_iget(sb, sbi->packed_nid);
                if (IS_ERR(sbi->packed_inode)) {
                        err = PTR_ERR(sbi->packed_inode);
                        sbi->packed_inode = NULL;
                        return err;
                }
        }
        err = erofs_init_managed_cache(sb);
        if (err)
                return err;

        err = erofs_xattr_prefixes_init(sb);
        if (err)
                return err;

        erofs_set_sysfs_name(sb);
        err = erofs_register_sysfs(sb);
        if (err)
                return err;

        erofs_info(sb, "mounted with root inode @ nid %llu.", sbi->root_nid);
        return 0;
}

static int erofs_fc_get_tree(struct fs_context *fc)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;
        int ret;

        if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid)
                return get_tree_nodev(fc, erofs_fc_fill_super);

        ret = get_tree_bdev_flags(fc, erofs_fc_fill_super,
                IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ?
                        GET_TREE_BDEV_QUIET_LOOKUP : 0);
#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
        if (ret == -ENOTBLK) {
                struct file *file;

                if (!fc->source)
                        return invalf(fc, "No source specified");
                file = filp_open(fc->source, O_RDONLY | O_LARGEFILE, 0);
                if (IS_ERR(file))
                        return PTR_ERR(file);
                sbi->dif0.file = file;

                if (S_ISREG(file_inode(sbi->dif0.file)->i_mode) &&
                    sbi->dif0.file->f_mapping->a_ops->read_folio)
                        return get_tree_nodev(fc, erofs_fc_fill_super);
        }
#endif
        return ret;
}

static int erofs_fc_reconfigure(struct fs_context *fc)
{
        struct super_block *sb = fc->root->d_sb;
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_sb_info *new_sbi = fc->s_fs_info;

        DBG_BUGON(!sb_rdonly(sb));

        if (new_sbi->fsid || new_sbi->domain_id)
                erofs_info(sb, "ignoring reconfiguration for fsid|domain_id.");

        if (test_opt(&new_sbi->opt, POSIX_ACL))
                fc->sb_flags |= SB_POSIXACL;
        else
                fc->sb_flags &= ~SB_POSIXACL;

        sbi->opt = new_sbi->opt;

        fc->sb_flags |= SB_RDONLY;
        return 0;
}

static int erofs_release_device_info(int id, void *ptr, void *data)
{
        struct erofs_device_info *dif = ptr;

        fs_put_dax(dif->dax_dev, NULL);
        if (dif->file)
                fput(dif->file);
        erofs_fscache_unregister_cookie(dif->fscache);
        dif->fscache = NULL;
        kfree(dif->path);
        kfree(dif);
        return 0;
}

static void erofs_free_dev_context(struct erofs_dev_context *devs)
{
        if (!devs)
                return;
        idr_for_each(&devs->tree, &erofs_release_device_info, NULL);
        idr_destroy(&devs->tree);
        kfree(devs);
}

static void erofs_sb_free(struct erofs_sb_info *sbi)
{
        erofs_free_dev_context(sbi->devs);
        kfree(sbi->fsid);
        kfree(sbi->domain_id);
        if (sbi->dif0.file)
                fput(sbi->dif0.file);
        kfree(sbi);
}

static void erofs_fc_free(struct fs_context *fc)
{
        struct erofs_sb_info *sbi = fc->s_fs_info;

        if (sbi) /* free here if an error occurs before transferring to sb */
                erofs_sb_free(sbi);
}

static const struct fs_context_operations erofs_context_ops = {
        .parse_param        = erofs_fc_parse_param,
        .get_tree       = erofs_fc_get_tree,
        .reconfigure    = erofs_fc_reconfigure,
        .free                = erofs_fc_free,
};

static int erofs_init_fs_context(struct fs_context *fc)
{
        struct erofs_sb_info *sbi;

        sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
        if (!sbi)
                return -ENOMEM;

        sbi->devs = kzalloc(sizeof(struct erofs_dev_context), GFP_KERNEL);
        if (!sbi->devs) {
                kfree(sbi);
                return -ENOMEM;
        }
        fc->s_fs_info = sbi;

        idr_init(&sbi->devs->tree);
        init_rwsem(&sbi->devs->rwsem);
        erofs_default_options(sbi);
        fc->ops = &erofs_context_ops;
        return 0;
}

static void erofs_kill_sb(struct super_block *sb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        if ((IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && sbi->fsid) ||
            sbi->dif0.file)
                kill_anon_super(sb);
        else
                kill_block_super(sb);
        fs_put_dax(sbi->dif0.dax_dev, NULL);
        erofs_fscache_unregister_fs(sb);
        erofs_sb_free(sbi);
        sb->s_fs_info = NULL;
}

static void erofs_put_super(struct super_block *sb)
{
        struct erofs_sb_info *const sbi = EROFS_SB(sb);

        DBG_BUGON(!sbi);

        erofs_unregister_sysfs(sb);
        erofs_shrinker_unregister(sb);
        erofs_xattr_prefixes_cleanup(sb);
#ifdef CONFIG_EROFS_FS_ZIP
        iput(sbi->managed_cache);
        sbi->managed_cache = NULL;
#endif
        iput(sbi->packed_inode);
        sbi->packed_inode = NULL;
        erofs_free_dev_context(sbi->devs);
        sbi->devs = NULL;
        erofs_fscache_unregister_fs(sb);
}

static struct file_system_type erofs_fs_type = {
        .owner          = THIS_MODULE,
        .name           = "erofs",
        .init_fs_context = erofs_init_fs_context,
        .kill_sb        = erofs_kill_sb,
        .fs_flags       = FS_REQUIRES_DEV | FS_ALLOW_IDMAP,
};
MODULE_ALIAS_FS("erofs");

static int __init erofs_module_init(void)
{
        int err;

        erofs_check_ondisk_layout_definitions();

        erofs_inode_cachep = kmem_cache_create("erofs_inode",
                        sizeof(struct erofs_inode), 0,
                        SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
                        erofs_inode_init_once);
        if (!erofs_inode_cachep)
                return -ENOMEM;

        err = erofs_init_shrinker();
        if (err)
                goto shrinker_err;

        err = z_erofs_init_subsystem();
        if (err)
                goto zip_err;

        err = erofs_init_sysfs();
        if (err)
                goto sysfs_err;

        err = register_filesystem(&erofs_fs_type);
        if (err)
                goto fs_err;

        return 0;

fs_err:
        erofs_exit_sysfs();
sysfs_err:
        z_erofs_exit_subsystem();
zip_err:
        erofs_exit_shrinker();
shrinker_err:
        kmem_cache_destroy(erofs_inode_cachep);
        return err;
}

static void __exit erofs_module_exit(void)
{
        unregister_filesystem(&erofs_fs_type);

        /* Ensure all RCU free inodes / pclusters are safe to be destroyed. */
        rcu_barrier();

        erofs_exit_sysfs();
        z_erofs_exit_subsystem();
        erofs_exit_shrinker();
        kmem_cache_destroy(erofs_inode_cachep);
}

static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
{
        struct super_block *sb = dentry->d_sb;
        struct erofs_sb_info *sbi = EROFS_SB(sb);

        buf->f_type = sb->s_magic;
        buf->f_bsize = sb->s_blocksize;
        buf->f_blocks = sbi->total_blocks;
        buf->f_bfree = buf->f_bavail = 0;
        buf->f_files = ULLONG_MAX;
        buf->f_ffree = ULLONG_MAX - sbi->inos;
        buf->f_namelen = EROFS_NAME_LEN;

        if (uuid_is_null(&sb->s_uuid))
                buf->f_fsid = u64_to_fsid(!sb->s_bdev ? 0 :
                                huge_encode_dev(sb->s_bdev->bd_dev));
        else
                buf->f_fsid = uuid_to_fsid(sb->s_uuid.b);
        return 0;
}

static int erofs_show_options(struct seq_file *seq, struct dentry *root)
{
        struct erofs_sb_info *sbi = EROFS_SB(root->d_sb);
        struct erofs_mount_opts *opt = &sbi->opt;

        if (IS_ENABLED(CONFIG_EROFS_FS_XATTR))
                seq_puts(seq, test_opt(opt, XATTR_USER) ?
                                ",user_xattr" : ",nouser_xattr");
        if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL))
                seq_puts(seq, test_opt(opt, POSIX_ACL) ? ",acl" : ",noacl");
        if (IS_ENABLED(CONFIG_EROFS_FS_ZIP))
                seq_printf(seq, ",cache_strategy=%s",
                          erofs_param_cache_strategy[opt->cache_strategy].name);
        if (test_opt(opt, DAX_ALWAYS))
                seq_puts(seq, ",dax=always");
        if (test_opt(opt, DAX_NEVER))
                seq_puts(seq, ",dax=never");
        if (erofs_is_fileio_mode(sbi) && test_opt(opt, DIRECT_IO))
                seq_puts(seq, ",directio");
#ifdef CONFIG_EROFS_FS_ONDEMAND
        if (sbi->fsid)
                seq_printf(seq, ",fsid=%s", sbi->fsid);
        if (sbi->domain_id)
                seq_printf(seq, ",domain_id=%s", sbi->domain_id);
#endif
        return 0;
}

const struct super_operations erofs_sops = {
        .put_super = erofs_put_super,
        .alloc_inode = erofs_alloc_inode,
        .free_inode = erofs_free_inode,
        .statfs = erofs_statfs,
        .show_options = erofs_show_options,
};

module_init(erofs_module_init);
module_exit(erofs_module_exit);

MODULE_DESCRIPTION("Enhanced ROM File System");
MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc.");
MODULE_LICENSE("GPL");


















































   13 






    2 












    9 
    2 




    8 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
// SPDX-License-Identifier: GPL-2.0-only
/*
 * (C) 2013 Astaro GmbH & Co KG
 */

#include <linux/module.h>
#include <linux/skbuff.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_ecache.h>
#include <net/netfilter/nf_conntrack_labels.h>
#include <linux/netfilter/x_tables.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Florian Westphal <fw@strlen.de>");
MODULE_DESCRIPTION("Xtables: add/match connection tracking labels");
MODULE_ALIAS("ipt_connlabel");
MODULE_ALIAS("ip6t_connlabel");

static bool
connlabel_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_connlabel_mtinfo *info = par->matchinfo;
        enum ip_conntrack_info ctinfo;
        struct nf_conn_labels *labels;
        struct nf_conn *ct;
        bool invert = info->options & XT_CONNLABEL_OP_INVERT;

        ct = nf_ct_get(skb, &ctinfo);
        if (ct == NULL)
                return invert;

        labels = nf_ct_labels_find(ct);
        if (!labels)
                return invert;

        if (test_bit(info->bit, labels->bits))
                return !invert;

        if (info->options & XT_CONNLABEL_OP_SET) {
                if (!test_and_set_bit(info->bit, labels->bits))
                        nf_conntrack_event_cache(IPCT_LABEL, ct);

                return !invert;
        }

        return invert;
}

static int connlabel_mt_check(const struct xt_mtchk_param *par)
{
        const int options = XT_CONNLABEL_OP_INVERT |
                            XT_CONNLABEL_OP_SET;
        struct xt_connlabel_mtinfo *info = par->matchinfo;
        int ret;

        if (info->options & ~options) {
                pr_info_ratelimited("Unknown options in mask %x\n",
                                    info->options);
                return -EINVAL;
        }

        ret = nf_ct_netns_get(par->net, par->family);
        if (ret < 0) {
                pr_info_ratelimited("cannot load conntrack support for proto=%u\n",
                                    par->family);
                return ret;
        }

        ret = nf_connlabels_get(par->net, info->bit);
        if (ret < 0)
                nf_ct_netns_put(par->net, par->family);
        return ret;
}

static void connlabel_mt_destroy(const struct xt_mtdtor_param *par)
{
        nf_connlabels_put(par->net);
        nf_ct_netns_put(par->net, par->family);
}

static struct xt_match connlabels_mt_reg __read_mostly = {
        .name           = "connlabel",
        .family         = NFPROTO_UNSPEC,
        .checkentry     = connlabel_mt_check,
        .match          = connlabel_mt,
        .matchsize      = sizeof(struct xt_connlabel_mtinfo),
        .destroy        = connlabel_mt_destroy,
        .me             = THIS_MODULE,
};

static int __init connlabel_mt_init(void)
{
        return xt_register_match(&connlabels_mt_reg);
}

static void __exit connlabel_mt_exit(void)
{
        xt_unregister_match(&connlabels_mt_reg);
}

module_init(connlabel_mt_init);
module_exit(connlabel_mt_exit);



















































































































   69 



   48 
























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *
 * Copyright (C) Jonathan Naylor G4KLX (g4klx@g4klx.demon.co.uk)
 */
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <net/ax25.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/uaccess.h>
#include <linux/fcntl.h>
#include <linux/mm.h>
#include <linux/interrupt.h>

/*
 * The default broadcast address of an interface is QST-0; the default address
 * is LINUX-1.  The null address is defined as a callsign of all spaces with
 * an SSID of zero.
 */

const ax25_address ax25_bcast =
        {{'Q' << 1, 'S' << 1, 'T' << 1, ' ' << 1, ' ' << 1, ' ' << 1, 0 << 1}};
const ax25_address ax25_defaddr =
        {{'L' << 1, 'I' << 1, 'N' << 1, 'U' << 1, 'X' << 1, ' ' << 1, 1 << 1}};
const ax25_address null_ax25_address =
        {{' ' << 1, ' ' << 1, ' ' << 1, ' ' << 1, ' ' << 1, ' ' << 1, 0 << 1}};

EXPORT_SYMBOL_GPL(ax25_bcast);
EXPORT_SYMBOL_GPL(ax25_defaddr);
EXPORT_SYMBOL(null_ax25_address);

/*
 *        ax25 -> ascii conversion
 */
char *ax2asc(char *buf, const ax25_address *a)
{
        char c, *s;
        int n;

        for (n = 0, s = buf; n < 6; n++) {
                c = (a->ax25_call[n] >> 1) & 0x7F;

                if (c != ' ') *s++ = c;
        }

        *s++ = '-';

        if ((n = ((a->ax25_call[6] >> 1) & 0x0F)) > 9) {
                *s++ = '1';
                n -= 10;
        }

        *s++ = n + '0';
        *s++ = '\0';

        if (*buf == '\0' || *buf == '-')
           return "*";

        return buf;

}

EXPORT_SYMBOL(ax2asc);

/*
 *        ascii -> ax25 conversion
 */
void asc2ax(ax25_address *addr, const char *callsign)
{
        const char *s;
        int n;

        for (s = callsign, n = 0; n < 6; n++) {
                if (*s != '\0' && *s != '-')
                        addr->ax25_call[n] = *s++;
                else
                        addr->ax25_call[n] = ' ';
                addr->ax25_call[n] <<= 1;
                addr->ax25_call[n] &= 0xFE;
        }

        if (*s++ == '\0') {
                addr->ax25_call[6] = 0x00;
                return;
        }

        addr->ax25_call[6] = *s++ - '0';

        if (*s != '\0') {
                addr->ax25_call[6] *= 10;
                addr->ax25_call[6] += *s++ - '0';
        }

        addr->ax25_call[6] <<= 1;
        addr->ax25_call[6] &= 0x1E;
}

EXPORT_SYMBOL(asc2ax);

/*
 *        Compare two ax.25 addresses
 */
int ax25cmp(const ax25_address *a, const ax25_address *b)
{
        int ct = 0;

        while (ct < 6) {
                if ((a->ax25_call[ct] & 0xFE) != (b->ax25_call[ct] & 0xFE))        /* Clean off repeater bits */
                        return 1;
                ct++;
        }

        if ((a->ax25_call[ct] & 0x1E) == (b->ax25_call[ct] & 0x1E))        /* SSID without control bit */
                return 0;

        return 2;                        /* Partial match */
}

EXPORT_SYMBOL(ax25cmp);

/*
 *        Compare two AX.25 digipeater paths.
 */
int ax25digicmp(const ax25_digi *digi1, const ax25_digi *digi2)
{
        int i;

        if (digi1->ndigi != digi2->ndigi)
                return 1;

        if (digi1->lastrepeat != digi2->lastrepeat)
                return 1;

        for (i = 0; i < digi1->ndigi; i++)
                if (ax25cmp(&digi1->calls[i], &digi2->calls[i]) != 0)
                        return 1;

        return 0;
}

/*
 *        Given an AX.25 address pull of to, from, digi list, command/response and the start of data
 *
 */
const unsigned char *ax25_addr_parse(const unsigned char *buf, int len,
        ax25_address *src, ax25_address *dest, ax25_digi *digi, int *flags,
        int *dama)
{
        int d = 0;

        if (len < 14) return NULL;

        if (flags != NULL) {
                *flags = 0;

                if (buf[6] & AX25_CBIT)
                        *flags = AX25_COMMAND;
                if (buf[13] & AX25_CBIT)
                        *flags = AX25_RESPONSE;
        }

        if (dama != NULL)
                *dama = ~buf[13] & AX25_DAMA_FLAG;

        /* Copy to, from */
        if (dest != NULL)
                memcpy(dest, buf + 0, AX25_ADDR_LEN);
        if (src != NULL)
                memcpy(src,  buf + 7, AX25_ADDR_LEN);

        buf += 2 * AX25_ADDR_LEN;
        len -= 2 * AX25_ADDR_LEN;

        digi->lastrepeat = -1;
        digi->ndigi      = 0;

        while (!(buf[-1] & AX25_EBIT)) {
                if (d >= AX25_MAX_DIGIS)
                        return NULL;
                if (len < AX25_ADDR_LEN)
                        return NULL;

                memcpy(&digi->calls[d], buf, AX25_ADDR_LEN);
                digi->ndigi = d + 1;

                if (buf[6] & AX25_HBIT) {
                        digi->repeated[d] = 1;
                        digi->lastrepeat  = d;
                } else {
                        digi->repeated[d] = 0;
                }

                buf += AX25_ADDR_LEN;
                len -= AX25_ADDR_LEN;
                d++;
        }

        return buf;
}

/*
 *        Assemble an AX.25 header from the bits
 */
int ax25_addr_build(unsigned char *buf, const ax25_address *src,
        const ax25_address *dest, const ax25_digi *d, int flag, int modulus)
{
        int len = 0;
        int ct  = 0;

        memcpy(buf, dest, AX25_ADDR_LEN);
        buf[6] &= ~(AX25_EBIT | AX25_CBIT);
        buf[6] |= AX25_SSSID_SPARE;

        if (flag == AX25_COMMAND) buf[6] |= AX25_CBIT;

        buf += AX25_ADDR_LEN;
        len += AX25_ADDR_LEN;

        memcpy(buf, src, AX25_ADDR_LEN);
        buf[6] &= ~(AX25_EBIT | AX25_CBIT);
        buf[6] &= ~AX25_SSSID_SPARE;

        if (modulus == AX25_MODULUS)
                buf[6] |= AX25_SSSID_SPARE;
        else
                buf[6] |= AX25_ESSID_SPARE;

        if (flag == AX25_RESPONSE) buf[6] |= AX25_CBIT;

        /*
         *        Fast path the normal digiless path
         */
        if (d == NULL || d->ndigi == 0) {
                buf[6] |= AX25_EBIT;
                return 2 * AX25_ADDR_LEN;
        }

        buf += AX25_ADDR_LEN;
        len += AX25_ADDR_LEN;

        while (ct < d->ndigi) {
                memcpy(buf, &d->calls[ct], AX25_ADDR_LEN);

                if (d->repeated[ct])
                        buf[6] |= AX25_HBIT;
                else
                        buf[6] &= ~AX25_HBIT;

                buf[6] &= ~AX25_EBIT;
                buf[6] |= AX25_SSSID_SPARE;

                buf += AX25_ADDR_LEN;
                len += AX25_ADDR_LEN;
                ct++;
        }

        buf[-1] |= AX25_EBIT;

        return len;
}

int ax25_addr_size(const ax25_digi *dp)
{
        if (dp == NULL)
                return 2 * AX25_ADDR_LEN;

        return AX25_ADDR_LEN * (2 + dp->ndigi);
}

/*
 *        Reverse Digipeat List. May not pass both parameters as same struct
 */
void ax25_digi_invert(const ax25_digi *in, ax25_digi *out)
{
        int ct;

        out->ndigi      = in->ndigi;
        out->lastrepeat = in->ndigi - in->lastrepeat - 2;

        /* Invert the digipeaters */
        for (ct = 0; ct < in->ndigi; ct++) {
                out->calls[ct] = in->calls[in->ndigi - ct - 1];

                if (ct <= out->lastrepeat) {
                        out->calls[ct].ax25_call[6] |= AX25_HBIT;
                        out->repeated[ct]            = 1;
                } else {
                        out->calls[ct].ax25_call[6] &= ~AX25_HBIT;
                        out->repeated[ct]            = 0;
                }
        }
}

























































    8 



















   14 


    1 

    2 






    3 


    9 




















   15 




    1 











































    5 


    4 
    1 
























   19 



   19 
   19 





   15 
    4 










   17 
    2 












   44 




    1 




    7 
   18 












   14 


   11 





    9 


   10 


















































   16 










   16 

   16 








    1 



    2 
    1 


   13 






   33 






   17 




   16 






   23 

    4 


















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 1992 Darren Senn
 */

/* These are all the functions necessary to implement itimers */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/syscalls.h>
#include <linux/time.h>
#include <linux/sched/signal.h>
#include <linux/sched/cputime.h>
#include <linux/posix-timers.h>
#include <linux/hrtimer.h>
#include <trace/events/timer.h>
#include <linux/compat.h>

#include <linux/uaccess.h>

/**
 * itimer_get_remtime - get remaining time for the timer
 *
 * @timer: the timer to read
 *
 * Returns the delta between the expiry time and now, which can be
 * less than zero or 1usec for an pending expired timer
 */
static struct timespec64 itimer_get_remtime(struct hrtimer *timer)
{
        ktime_t rem = __hrtimer_get_remaining(timer, true);

        /*
         * Racy but safe: if the itimer expires after the above
         * hrtimer_get_remtime() call but before this condition
         * then we return 0 - which is correct.
         */
        if (hrtimer_active(timer)) {
                if (rem <= 0)
                        rem = NSEC_PER_USEC;
        } else
                rem = 0;

        return ktime_to_timespec64(rem);
}

static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id,
                           struct itimerspec64 *const value)
{
        u64 val, interval;
        struct cpu_itimer *it = &tsk->signal->it[clock_id];

        spin_lock_irq(&tsk->sighand->siglock);

        val = it->expires;
        interval = it->incr;
        if (val) {
                u64 t, samples[CPUCLOCK_MAX];

                thread_group_sample_cputime(tsk, samples);
                t = samples[clock_id];

                if (val < t)
                        /* about to fire */
                        val = TICK_NSEC;
                else
                        val -= t;
        }

        spin_unlock_irq(&tsk->sighand->siglock);

        value->it_value = ns_to_timespec64(val);
        value->it_interval = ns_to_timespec64(interval);
}

static int do_getitimer(int which, struct itimerspec64 *value)
{
        struct task_struct *tsk = current;

        switch (which) {
        case ITIMER_REAL:
                spin_lock_irq(&tsk->sighand->siglock);
                value->it_value = itimer_get_remtime(&tsk->signal->real_timer);
                value->it_interval =
                        ktime_to_timespec64(tsk->signal->it_real_incr);
                spin_unlock_irq(&tsk->sighand->siglock);
                break;
        case ITIMER_VIRTUAL:
                get_cpu_itimer(tsk, CPUCLOCK_VIRT, value);
                break;
        case ITIMER_PROF:
                get_cpu_itimer(tsk, CPUCLOCK_PROF, value);
                break;
        default:
                return(-EINVAL);
        }
        return 0;
}

static int put_itimerval(struct __kernel_old_itimerval __user *o,
                         const struct itimerspec64 *i)
{
        struct __kernel_old_itimerval v;

        v.it_interval.tv_sec = i->it_interval.tv_sec;
        v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC;
        v.it_value.tv_sec = i->it_value.tv_sec;
        v.it_value.tv_usec = i->it_value.tv_nsec / NSEC_PER_USEC;
        return copy_to_user(o, &v, sizeof(struct __kernel_old_itimerval)) ? -EFAULT : 0;
}


SYSCALL_DEFINE2(getitimer, int, which, struct __kernel_old_itimerval __user *, value)
{
        struct itimerspec64 get_buffer;
        int error = do_getitimer(which, &get_buffer);

        if (!error && put_itimerval(value, &get_buffer))
                error = -EFAULT;
        return error;
}

#if defined(CONFIG_COMPAT) || defined(CONFIG_ALPHA)
struct old_itimerval32 {
        struct old_timeval32        it_interval;
        struct old_timeval32        it_value;
};

static int put_old_itimerval32(struct old_itimerval32 __user *o,
                               const struct itimerspec64 *i)
{
        struct old_itimerval32 v32;

        v32.it_interval.tv_sec = i->it_interval.tv_sec;
        v32.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC;
        v32.it_value.tv_sec = i->it_value.tv_sec;
        v32.it_value.tv_usec = i->it_value.tv_nsec / NSEC_PER_USEC;
        return copy_to_user(o, &v32, sizeof(struct old_itimerval32)) ? -EFAULT : 0;
}

COMPAT_SYSCALL_DEFINE2(getitimer, int, which,
                       struct old_itimerval32 __user *, value)
{
        struct itimerspec64 get_buffer;
        int error = do_getitimer(which, &get_buffer);

        if (!error && put_old_itimerval32(value, &get_buffer))
                error = -EFAULT;
        return error;
}
#endif

/*
 * Invoked from dequeue_signal() when SIG_ALRM is delivered.
 *
 * Restart the ITIMER_REAL timer if it is armed as periodic timer.  Doing
 * this in the signal delivery path instead of self rearming prevents a DoS
 * with small increments in the high reolution timer case and reduces timer
 * noise in general.
 */
void posixtimer_rearm_itimer(struct task_struct *tsk)
{
        struct hrtimer *tmr = &tsk->signal->real_timer;

        if (!hrtimer_is_queued(tmr) && tsk->signal->it_real_incr != 0) {
                hrtimer_forward(tmr, tmr->base->get_time(),
                                tsk->signal->it_real_incr);
                hrtimer_restart(tmr);
        }
}

/*
 * Interval timers are restarted in the signal delivery path.  See
 * posixtimer_rearm_itimer().
 */
enum hrtimer_restart it_real_fn(struct hrtimer *timer)
{
        struct signal_struct *sig =
                container_of(timer, struct signal_struct, real_timer);
        struct pid *leader_pid = sig->pids[PIDTYPE_TGID];

        trace_itimer_expire(ITIMER_REAL, leader_pid, 0);
        kill_pid_info(SIGALRM, SEND_SIG_PRIV, leader_pid);

        return HRTIMER_NORESTART;
}

static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id,
                           const struct itimerspec64 *const value,
                           struct itimerspec64 *const ovalue)
{
        u64 oval, nval, ointerval, ninterval;
        struct cpu_itimer *it = &tsk->signal->it[clock_id];

        nval = timespec64_to_ns(&value->it_value);
        ninterval = timespec64_to_ns(&value->it_interval);

        spin_lock_irq(&tsk->sighand->siglock);

        oval = it->expires;
        ointerval = it->incr;
        if (oval || nval) {
                if (nval > 0)
                        nval += TICK_NSEC;
                set_process_cpu_timer(tsk, clock_id, &nval, &oval);
        }
        it->expires = nval;
        it->incr = ninterval;
        trace_itimer_state(clock_id == CPUCLOCK_VIRT ?
                           ITIMER_VIRTUAL : ITIMER_PROF, value, nval);

        spin_unlock_irq(&tsk->sighand->siglock);

        if (ovalue) {
                ovalue->it_value = ns_to_timespec64(oval);
                ovalue->it_interval = ns_to_timespec64(ointerval);
        }
}

/*
 * Returns true if the timeval is in canonical form
 */
#define timeval_valid(t) \
        (((t)->tv_sec >= 0) && (((unsigned long) (t)->tv_usec) < USEC_PER_SEC))

static int do_setitimer(int which, struct itimerspec64 *value,
                        struct itimerspec64 *ovalue)
{
        struct task_struct *tsk = current;
        struct hrtimer *timer;
        ktime_t expires;

        switch (which) {
        case ITIMER_REAL:
again:
                spin_lock_irq(&tsk->sighand->siglock);
                timer = &tsk->signal->real_timer;
                if (ovalue) {
                        ovalue->it_value = itimer_get_remtime(timer);
                        ovalue->it_interval
                                = ktime_to_timespec64(tsk->signal->it_real_incr);
                }
                /* We are sharing ->siglock with it_real_fn() */
                if (hrtimer_try_to_cancel(timer) < 0) {
                        spin_unlock_irq(&tsk->sighand->siglock);
                        hrtimer_cancel_wait_running(timer);
                        goto again;
                }
                expires = timespec64_to_ktime(value->it_value);
                if (expires != 0) {
                        tsk->signal->it_real_incr =
                                timespec64_to_ktime(value->it_interval);
                        hrtimer_start(timer, expires, HRTIMER_MODE_REL);
                } else
                        tsk->signal->it_real_incr = 0;

                trace_itimer_state(ITIMER_REAL, value, 0);
                spin_unlock_irq(&tsk->sighand->siglock);
                break;
        case ITIMER_VIRTUAL:
                set_cpu_itimer(tsk, CPUCLOCK_VIRT, value, ovalue);
                break;
        case ITIMER_PROF:
                set_cpu_itimer(tsk, CPUCLOCK_PROF, value, ovalue);
                break;
        default:
                return -EINVAL;
        }
        return 0;
}

#ifdef CONFIG_SECURITY_SELINUX
void clear_itimer(void)
{
        struct itimerspec64 v = {};
        int i;

        for (i = 0; i < 3; i++)
                do_setitimer(i, &v, NULL);
}
#endif

#ifdef __ARCH_WANT_SYS_ALARM

/**
 * alarm_setitimer - set alarm in seconds
 *
 * @seconds:        number of seconds until alarm
 *                0 disables the alarm
 *
 * Returns the remaining time in seconds of a pending timer or 0 when
 * the timer is not active.
 *
 * On 32 bit machines the seconds value is limited to (INT_MAX/2) to avoid
 * negative timeval settings which would cause immediate expiry.
 */
static unsigned int alarm_setitimer(unsigned int seconds)
{
        struct itimerspec64 it_new, it_old;

#if BITS_PER_LONG < 64
        if (seconds > INT_MAX)
                seconds = INT_MAX;
#endif
        it_new.it_value.tv_sec = seconds;
        it_new.it_value.tv_nsec = 0;
        it_new.it_interval.tv_sec = it_new.it_interval.tv_nsec = 0;

        do_setitimer(ITIMER_REAL, &it_new, &it_old);

        /*
         * We can't return 0 if we have an alarm pending ...  And we'd
         * better return too much than too little anyway
         */
        if ((!it_old.it_value.tv_sec && it_old.it_value.tv_nsec) ||
              it_old.it_value.tv_nsec >= (NSEC_PER_SEC / 2))
                it_old.it_value.tv_sec++;

        return it_old.it_value.tv_sec;
}

/*
 * For backwards compatibility?  This can be done in libc so Alpha
 * and all newer ports shouldn't need it.
 */
SYSCALL_DEFINE1(alarm, unsigned int, seconds)
{
        return alarm_setitimer(seconds);
}

#endif

static int get_itimerval(struct itimerspec64 *o, const struct __kernel_old_itimerval __user *i)
{
        struct __kernel_old_itimerval v;

        if (copy_from_user(&v, i, sizeof(struct __kernel_old_itimerval)))
                return -EFAULT;

        /* Validate the timevals in value. */
        if (!timeval_valid(&v.it_value) ||
            !timeval_valid(&v.it_interval))
                return -EINVAL;

        o->it_interval.tv_sec = v.it_interval.tv_sec;
        o->it_interval.tv_nsec = v.it_interval.tv_usec * NSEC_PER_USEC;
        o->it_value.tv_sec = v.it_value.tv_sec;
        o->it_value.tv_nsec = v.it_value.tv_usec * NSEC_PER_USEC;
        return 0;
}

SYSCALL_DEFINE3(setitimer, int, which, struct __kernel_old_itimerval __user *, value,
                struct __kernel_old_itimerval __user *, ovalue)
{
        struct itimerspec64 set_buffer, get_buffer;
        int error;

        if (value) {
                error = get_itimerval(&set_buffer, value);
                if (error)
                        return error;
        } else {
                memset(&set_buffer, 0, sizeof(set_buffer));
                printk_once(KERN_WARNING "%s calls setitimer() with new_value NULL pointer."
                            " Misfeature support will be removed\n",
                            current->comm);
        }

        error = do_setitimer(which, &set_buffer, ovalue ? &get_buffer : NULL);
        if (error || !ovalue)
                return error;

        if (put_itimerval(ovalue, &get_buffer))
                return -EFAULT;
        return 0;
}

#if defined(CONFIG_COMPAT) || defined(CONFIG_ALPHA)
static int get_old_itimerval32(struct itimerspec64 *o, const struct old_itimerval32 __user *i)
{
        struct old_itimerval32 v32;

        if (copy_from_user(&v32, i, sizeof(struct old_itimerval32)))
                return -EFAULT;

        /* Validate the timevals in value.  */
        if (!timeval_valid(&v32.it_value) ||
            !timeval_valid(&v32.it_interval))
                return -EINVAL;

        o->it_interval.tv_sec = v32.it_interval.tv_sec;
        o->it_interval.tv_nsec = v32.it_interval.tv_usec * NSEC_PER_USEC;
        o->it_value.tv_sec = v32.it_value.tv_sec;
        o->it_value.tv_nsec = v32.it_value.tv_usec * NSEC_PER_USEC;
        return 0;
}

COMPAT_SYSCALL_DEFINE3(setitimer, int, which,
                       struct old_itimerval32 __user *, value,
                       struct old_itimerval32 __user *, ovalue)
{
        struct itimerspec64 set_buffer, get_buffer;
        int error;

        if (value) {
                error = get_old_itimerval32(&set_buffer, value);
                if (error)
                        return error;
        } else {
                memset(&set_buffer, 0, sizeof(set_buffer));
                printk_once(KERN_WARNING "%s calls setitimer() with new_value NULL pointer."
                            " Misfeature support will be removed\n",
                            current->comm);
        }

        error = do_setitimer(which, &set_buffer, ovalue ? &get_buffer : NULL);
        if (error || !ovalue)
                return error;
        if (put_old_itimerval32(ovalue, &get_buffer))
                return -EFAULT;
        return 0;
}
#endif



















































































































































































   13 









    8 


















































   14 


   14 




















    8 







    8 
    8 
    8 




    8 

































































































    1 





































































































































































































































































































































































































































































































































































    1 










    1 

    1 










































































































































   22 





    1 





   18 
   16 









    9 
   17 


    9 
    8 
   10 
    7 














   18 





   18 

   18 






   18 





   11 








   16 




   16 






    7 




   15 






    1 







   14 








    4 
    7 






   17 




   10 





   18 



   15 

   14 




























   36 




   36 










   33 
























   33 
   29 


    4 




   23 




    7 





    1 





   29 












   34 



   25 
   35 

















   32 


























   32 






















   28 



    3 




    7 




   23 





    1 










   28 
   28 
   28 



   27 
    1 


















   66 
   26 
   67 










   59 

















   59 

   58 




   30 


   30 




   59 



   58 
   30 
   59 
   31 



   59 



   67 

   67 










   24 





























































































































































































   47 
   47 








   46 




   47 

   25 














   29 

    1 



    4 
    2 


    1 

    1 



    3 


    3 













   34 











   24 







    1 

   33 
    4 







   34 
   22 

















   59 



   30 


   56 



   32 



   57 
   57 




   26 


   57 




   28 




   57 










    7 



    2 
    4 


    4 












   25 















   22 

    1 

    2 






   25 


    5 

    8 




    3 




    1 









    1 







    1 
    1 



    1 


    3 








    2 
    1 


    3 
    3 
    1 
    1 

    3 

























































   25 






























   44 






   19 
   25 










   44 

    2 


   32 


   24 
    8 


   31 


    8 
   24 
















    7 
    3 
    8 
    2 




















    2 







   12 

    1 
    9 



    3 



   12 










































































    1 




    2 
    2 



    2 
    2 


    2 















    1 











    1 









    1 
    1 




    1 
    1 




    1 


    1 




































    8 




    4 



    7 













   13 

   13 




   13 









   17 




   16 


   13 









































   80 














   80 



    7 
   73 






   50 


   25 












   13 





    7 




















    5 
    5 


    1 






    3 













    1 














   69 














   64 
    7 

   64 





   58 

   58 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
/*
 * usbmidi.c - ALSA USB MIDI driver
 *
 * Copyright (c) 2002-2009 Clemens Ladisch
 * All rights reserved.
 *
 * Based on the OSS usb-midi driver by NAGANO Daisuke,
 *          NetBSD's umidi driver by Takuya SHIOZAKI,
 *          the "USB Device Class Definition for MIDI Devices" by Roland
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions, and the following disclaimer,
 *    without modification.
 * 2. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed and/or modified under the
 * terms of the GNU General Public License as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any later
 * version.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/usb.h>
#include <linux/wait.h>
#include <linux/usb/audio.h>
#include <linux/usb/midi.h>
#include <linux/module.h>

#include <sound/core.h>
#include <sound/control.h>
#include <sound/rawmidi.h>
#include <sound/asequencer.h>
#include "usbaudio.h"
#include "midi.h"
#include "power.h"
#include "helper.h"

/*
 * define this to log all USB packets
 */
/* #define DUMP_PACKETS */

/*
 * how long to wait after some USB errors, so that hub_wq can disconnect() us
 * without too many spurious errors
 */
#define ERROR_DELAY_JIFFIES (HZ / 10)

#define OUTPUT_URBS 7
#define INPUT_URBS 7


MODULE_AUTHOR("Clemens Ladisch <clemens@ladisch.de>");
MODULE_DESCRIPTION("USB Audio/MIDI helper module");
MODULE_LICENSE("Dual BSD/GPL");

struct snd_usb_midi_in_endpoint;
struct snd_usb_midi_out_endpoint;
struct snd_usb_midi_endpoint;

struct usb_protocol_ops {
        void (*input)(struct snd_usb_midi_in_endpoint*, uint8_t*, int);
        void (*output)(struct snd_usb_midi_out_endpoint *ep, struct urb *urb);
        void (*output_packet)(struct urb*, uint8_t, uint8_t, uint8_t, uint8_t);
        void (*init_out_endpoint)(struct snd_usb_midi_out_endpoint *);
        void (*finish_out_endpoint)(struct snd_usb_midi_out_endpoint *);
};

struct snd_usb_midi {
        struct usb_device *dev;
        struct snd_card *card;
        struct usb_interface *iface;
        const struct snd_usb_audio_quirk *quirk;
        struct snd_rawmidi *rmidi;
        const struct usb_protocol_ops *usb_protocol_ops;
        struct list_head list;
        struct timer_list error_timer;
        spinlock_t disc_lock;
        struct rw_semaphore disc_rwsem;
        struct mutex mutex;
        u32 usb_id;
        int next_midi_device;

        struct snd_usb_midi_endpoint {
                struct snd_usb_midi_out_endpoint *out;
                struct snd_usb_midi_in_endpoint *in;
        } endpoints[MIDI_MAX_ENDPOINTS];
        unsigned long input_triggered;
        unsigned int opened[2];
        unsigned char disconnected;
        unsigned char input_running;

        struct snd_kcontrol *roland_load_ctl;
};

struct snd_usb_midi_out_endpoint {
        struct snd_usb_midi *umidi;
        struct out_urb_context {
                struct urb *urb;
                struct snd_usb_midi_out_endpoint *ep;
        } urbs[OUTPUT_URBS];
        unsigned int active_urbs;
        unsigned int drain_urbs;
        int max_transfer;                /* size of urb buffer */
        struct work_struct work;
        unsigned int next_urb;
        spinlock_t buffer_lock;

        struct usbmidi_out_port {
                struct snd_usb_midi_out_endpoint *ep;
                struct snd_rawmidi_substream *substream;
                int active;
                uint8_t cable;                /* cable number << 4 */
                uint8_t state;
#define STATE_UNKNOWN        0
#define STATE_1PARAM        1
#define STATE_2PARAM_1        2
#define STATE_2PARAM_2        3
#define STATE_SYSEX_0        4
#define STATE_SYSEX_1        5
#define STATE_SYSEX_2        6
                uint8_t data[2];
        } ports[0x10];
        int current_port;

        wait_queue_head_t drain_wait;
};

struct snd_usb_midi_in_endpoint {
        struct snd_usb_midi *umidi;
        struct urb *urbs[INPUT_URBS];
        struct usbmidi_in_port {
                struct snd_rawmidi_substream *substream;
                u8 running_status_length;
        } ports[0x10];
        u8 seen_f5;
        bool in_sysex;
        u8 last_cin;
        u8 error_resubmit;
        int current_port;
};

static void snd_usbmidi_do_output(struct snd_usb_midi_out_endpoint *ep);

static const uint8_t snd_usbmidi_cin_length[] = {
        0, 0, 2, 3, 3, 1, 2, 3, 3, 3, 3, 3, 2, 2, 3, 1
};

/*
 * Submits the URB, with error handling.
 */
static int snd_usbmidi_submit_urb(struct urb *urb, gfp_t flags)
{
        int err = usb_submit_urb(urb, flags);
        if (err < 0 && err != -ENODEV)
                dev_err(&urb->dev->dev, "usb_submit_urb: %d\n", err);
        return err;
}

/*
 * Error handling for URB completion functions.
 */
static int snd_usbmidi_urb_error(const struct urb *urb)
{
        switch (urb->status) {
        /* manually unlinked, or device gone */
        case -ENOENT:
        case -ECONNRESET:
        case -ESHUTDOWN:
        case -ENODEV:
                return -ENODEV;
        /* errors that might occur during unplugging */
        case -EPROTO:
        case -ETIME:
        case -EILSEQ:
                return -EIO;
        default:
                dev_err(&urb->dev->dev, "urb status %d\n", urb->status);
                return 0; /* continue */
        }
}

/*
 * Receives a chunk of MIDI data.
 */
static void snd_usbmidi_input_data(struct snd_usb_midi_in_endpoint *ep,
                                   int portidx, uint8_t *data, int length)
{
        struct usbmidi_in_port *port = &ep->ports[portidx];

        if (!port->substream) {
                dev_dbg(&ep->umidi->dev->dev, "unexpected port %d!\n", portidx);
                return;
        }
        if (!test_bit(port->substream->number, &ep->umidi->input_triggered))
                return;
        snd_rawmidi_receive(port->substream, data, length);
}

#ifdef DUMP_PACKETS
static void dump_urb(const char *type, const u8 *data, int length)
{
        pr_debug("%s packet: [", type);
        for (; length > 0; ++data, --length)
                pr_cont(" %02x", *data);
        pr_cont(" ]\n");
}
#else
#define dump_urb(type, data, length) /* nothing */
#endif

/*
 * Processes the data read from the device.
 */
static void snd_usbmidi_in_urb_complete(struct urb *urb)
{
        struct snd_usb_midi_in_endpoint *ep = urb->context;

        if (urb->status == 0) {
                dump_urb("received", urb->transfer_buffer, urb->actual_length);
                ep->umidi->usb_protocol_ops->input(ep, urb->transfer_buffer,
                                                   urb->actual_length);
        } else {
                int err = snd_usbmidi_urb_error(urb);
                if (err < 0) {
                        if (err != -ENODEV) {
                                ep->error_resubmit = 1;
                                mod_timer(&ep->umidi->error_timer,
                                          jiffies + ERROR_DELAY_JIFFIES);
                        }
                        return;
                }
        }

        urb->dev = ep->umidi->dev;
        snd_usbmidi_submit_urb(urb, GFP_ATOMIC);
}

static void snd_usbmidi_out_urb_complete(struct urb *urb)
{
        struct out_urb_context *context = urb->context;
        struct snd_usb_midi_out_endpoint *ep = context->ep;
        unsigned int urb_index;
        unsigned long flags;

        spin_lock_irqsave(&ep->buffer_lock, flags);
        urb_index = context - ep->urbs;
        ep->active_urbs &= ~(1 << urb_index);
        if (unlikely(ep->drain_urbs)) {
                ep->drain_urbs &= ~(1 << urb_index);
                wake_up(&ep->drain_wait);
        }
        spin_unlock_irqrestore(&ep->buffer_lock, flags);
        if (urb->status < 0) {
                int err = snd_usbmidi_urb_error(urb);
                if (err < 0) {
                        if (err != -ENODEV)
                                mod_timer(&ep->umidi->error_timer,
                                          jiffies + ERROR_DELAY_JIFFIES);
                        return;
                }
        }
        snd_usbmidi_do_output(ep);
}

/*
 * This is called when some data should be transferred to the device
 * (from one or more substreams).
 */
static void snd_usbmidi_do_output(struct snd_usb_midi_out_endpoint *ep)
{
        unsigned int urb_index;
        struct urb *urb;
        unsigned long flags;

        spin_lock_irqsave(&ep->buffer_lock, flags);
        if (ep->umidi->disconnected) {
                spin_unlock_irqrestore(&ep->buffer_lock, flags);
                return;
        }

        urb_index = ep->next_urb;
        for (;;) {
                if (!(ep->active_urbs & (1 << urb_index))) {
                        urb = ep->urbs[urb_index].urb;
                        urb->transfer_buffer_length = 0;
                        ep->umidi->usb_protocol_ops->output(ep, urb);
                        if (urb->transfer_buffer_length == 0)
                                break;

                        dump_urb("sending", urb->transfer_buffer,
                                 urb->transfer_buffer_length);
                        urb->dev = ep->umidi->dev;
                        if (snd_usbmidi_submit_urb(urb, GFP_ATOMIC) < 0)
                                break;
                        ep->active_urbs |= 1 << urb_index;
                }
                if (++urb_index >= OUTPUT_URBS)
                        urb_index = 0;
                if (urb_index == ep->next_urb)
                        break;
        }
        ep->next_urb = urb_index;
        spin_unlock_irqrestore(&ep->buffer_lock, flags);
}

static void snd_usbmidi_out_work(struct work_struct *work)
{
        struct snd_usb_midi_out_endpoint *ep =
                container_of(work, struct snd_usb_midi_out_endpoint, work);

        snd_usbmidi_do_output(ep);
}

/* called after transfers had been interrupted due to some USB error */
static void snd_usbmidi_error_timer(struct timer_list *t)
{
        struct snd_usb_midi *umidi = from_timer(umidi, t, error_timer);
        unsigned int i, j;

        spin_lock(&umidi->disc_lock);
        if (umidi->disconnected) {
                spin_unlock(&umidi->disc_lock);
                return;
        }
        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                struct snd_usb_midi_in_endpoint *in = umidi->endpoints[i].in;
                if (in && in->error_resubmit) {
                        in->error_resubmit = 0;
                        for (j = 0; j < INPUT_URBS; ++j) {
                                if (atomic_read(&in->urbs[j]->use_count))
                                        continue;
                                in->urbs[j]->dev = umidi->dev;
                                snd_usbmidi_submit_urb(in->urbs[j], GFP_ATOMIC);
                        }
                }
                if (umidi->endpoints[i].out)
                        snd_usbmidi_do_output(umidi->endpoints[i].out);
        }
        spin_unlock(&umidi->disc_lock);
}

/* helper function to send static data that may not DMA-able */
static int send_bulk_static_data(struct snd_usb_midi_out_endpoint *ep,
                                 const void *data, int len)
{
        int err = 0;
        void *buf = kmemdup(data, len, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;
        dump_urb("sending", buf, len);
        if (ep->urbs[0].urb)
                err = usb_bulk_msg(ep->umidi->dev, ep->urbs[0].urb->pipe,
                                   buf, len, NULL, 250);
        kfree(buf);
        return err;
}

/*
 * Standard USB MIDI protocol: see the spec.
 * Midiman protocol: like the standard protocol, but the control byte is the
 * fourth byte in each packet, and uses length instead of CIN.
 */

static void snd_usbmidi_standard_input(struct snd_usb_midi_in_endpoint *ep,
                                       uint8_t *buffer, int buffer_length)
{
        int i;

        for (i = 0; i + 3 < buffer_length; i += 4)
                if (buffer[i] != 0) {
                        int cable = buffer[i] >> 4;
                        int length = snd_usbmidi_cin_length[buffer[i] & 0x0f];
                        snd_usbmidi_input_data(ep, cable, &buffer[i + 1],
                                               length);
                }
}

static void snd_usbmidi_midiman_input(struct snd_usb_midi_in_endpoint *ep,
                                      uint8_t *buffer, int buffer_length)
{
        int i;

        for (i = 0; i + 3 < buffer_length; i += 4)
                if (buffer[i + 3] != 0) {
                        int port = buffer[i + 3] >> 4;
                        int length = buffer[i + 3] & 3;
                        snd_usbmidi_input_data(ep, port, &buffer[i], length);
                }
}

/*
 * Buggy M-Audio device: running status on input results in a packet that has
 * the data bytes but not the status byte and that is marked with CIN 4.
 */
static void snd_usbmidi_maudio_broken_running_status_input(
                                        struct snd_usb_midi_in_endpoint *ep,
                                        uint8_t *buffer, int buffer_length)
{
        int i;

        for (i = 0; i + 3 < buffer_length; i += 4)
                if (buffer[i] != 0) {
                        int cable = buffer[i] >> 4;
                        u8 cin = buffer[i] & 0x0f;
                        struct usbmidi_in_port *port = &ep->ports[cable];
                        int length;

                        length = snd_usbmidi_cin_length[cin];
                        if (cin == 0xf && buffer[i + 1] >= 0xf8)
                                ; /* realtime msg: no running status change */
                        else if (cin >= 0x8 && cin <= 0xe)
                                /* channel msg */
                                port->running_status_length = length - 1;
                        else if (cin == 0x4 &&
                                 port->running_status_length != 0 &&
                                 buffer[i + 1] < 0x80)
                                /* CIN 4 that is not a SysEx */
                                length = port->running_status_length;
                        else
                                /*
                                 * All other msgs cannot begin running status.
                                 * (A channel msg sent as two or three CIN 0xF
                                 * packets could in theory, but this device
                                 * doesn't use this format.)
                                 */
                                port->running_status_length = 0;
                        snd_usbmidi_input_data(ep, cable, &buffer[i + 1],
                                               length);
                }
}

/*
 * QinHeng CH345 is buggy: every second packet inside a SysEx has not CIN 4
 * but the previously seen CIN, but still with three data bytes.
 */
static void ch345_broken_sysex_input(struct snd_usb_midi_in_endpoint *ep,
                                     uint8_t *buffer, int buffer_length)
{
        unsigned int i, cin, length;

        for (i = 0; i + 3 < buffer_length; i += 4) {
                if (buffer[i] == 0 && i > 0)
                        break;
                cin = buffer[i] & 0x0f;
                if (ep->in_sysex &&
                    cin == ep->last_cin &&
                    (buffer[i + 1 + (cin == 0x6)] & 0x80) == 0)
                        cin = 0x4;
#if 0
                if (buffer[i + 1] == 0x90) {
                        /*
                         * Either a corrupted running status or a real note-on
                         * message; impossible to detect reliably.
                         */
                }
#endif
                length = snd_usbmidi_cin_length[cin];
                snd_usbmidi_input_data(ep, 0, &buffer[i + 1], length);
                ep->in_sysex = cin == 0x4;
                if (!ep->in_sysex)
                        ep->last_cin = cin;
        }
}

/*
 * CME protocol: like the standard protocol, but SysEx commands are sent as a
 * single USB packet preceded by a 0x0F byte.
 */
static void snd_usbmidi_cme_input(struct snd_usb_midi_in_endpoint *ep,
                                  uint8_t *buffer, int buffer_length)
{
        if (buffer_length < 2 || (buffer[0] & 0x0f) != 0x0f)
                snd_usbmidi_standard_input(ep, buffer, buffer_length);
        else
                snd_usbmidi_input_data(ep, buffer[0] >> 4,
                                       &buffer[1], buffer_length - 1);
}

/*
 * Adds one USB MIDI packet to the output buffer.
 */
static void snd_usbmidi_output_standard_packet(struct urb *urb, uint8_t p0,
                                               uint8_t p1, uint8_t p2,
                                               uint8_t p3)
{

        uint8_t *buf =
                (uint8_t *)urb->transfer_buffer + urb->transfer_buffer_length;
        buf[0] = p0;
        buf[1] = p1;
        buf[2] = p2;
        buf[3] = p3;
        urb->transfer_buffer_length += 4;
}

/*
 * Adds one Midiman packet to the output buffer.
 */
static void snd_usbmidi_output_midiman_packet(struct urb *urb, uint8_t p0,
                                              uint8_t p1, uint8_t p2,
                                              uint8_t p3)
{

        uint8_t *buf =
                (uint8_t *)urb->transfer_buffer + urb->transfer_buffer_length;
        buf[0] = p1;
        buf[1] = p2;
        buf[2] = p3;
        buf[3] = (p0 & 0xf0) | snd_usbmidi_cin_length[p0 & 0x0f];
        urb->transfer_buffer_length += 4;
}

/*
 * Converts MIDI commands to USB MIDI packets.
 */
static void snd_usbmidi_transmit_byte(struct usbmidi_out_port *port,
                                      uint8_t b, struct urb *urb)
{
        uint8_t p0 = port->cable;
        void (*output_packet)(struct urb*, uint8_t, uint8_t, uint8_t, uint8_t) =
                port->ep->umidi->usb_protocol_ops->output_packet;

        if (b >= 0xf8) {
                output_packet(urb, p0 | 0x0f, b, 0, 0);
        } else if (b >= 0xf0) {
                switch (b) {
                case 0xf0:
                        port->data[0] = b;
                        port->state = STATE_SYSEX_1;
                        break;
                case 0xf1:
                case 0xf3:
                        port->data[0] = b;
                        port->state = STATE_1PARAM;
                        break;
                case 0xf2:
                        port->data[0] = b;
                        port->state = STATE_2PARAM_1;
                        break;
                case 0xf4:
                case 0xf5:
                        port->state = STATE_UNKNOWN;
                        break;
                case 0xf6:
                        output_packet(urb, p0 | 0x05, 0xf6, 0, 0);
                        port->state = STATE_UNKNOWN;
                        break;
                case 0xf7:
                        switch (port->state) {
                        case STATE_SYSEX_0:
                                output_packet(urb, p0 | 0x05, 0xf7, 0, 0);
                                break;
                        case STATE_SYSEX_1:
                                output_packet(urb, p0 | 0x06, port->data[0],
                                              0xf7, 0);
                                break;
                        case STATE_SYSEX_2:
                                output_packet(urb, p0 | 0x07, port->data[0],
                                              port->data[1], 0xf7);
                                break;
                        }
                        port->state = STATE_UNKNOWN;
                        break;
                }
        } else if (b >= 0x80) {
                port->data[0] = b;
                if (b >= 0xc0 && b <= 0xdf)
                        port->state = STATE_1PARAM;
                else
                        port->state = STATE_2PARAM_1;
        } else { /* b < 0x80 */
                switch (port->state) {
                case STATE_1PARAM:
                        if (port->data[0] < 0xf0) {
                                p0 |= port->data[0] >> 4;
                        } else {
                                p0 |= 0x02;
                                port->state = STATE_UNKNOWN;
                        }
                        output_packet(urb, p0, port->data[0], b, 0);
                        break;
                case STATE_2PARAM_1:
                        port->data[1] = b;
                        port->state = STATE_2PARAM_2;
                        break;
                case STATE_2PARAM_2:
                        if (port->data[0] < 0xf0) {
                                p0 |= port->data[0] >> 4;
                                port->state = STATE_2PARAM_1;
                        } else {
                                p0 |= 0x03;
                                port->state = STATE_UNKNOWN;
                        }
                        output_packet(urb, p0, port->data[0], port->data[1], b);
                        break;
                case STATE_SYSEX_0:
                        port->data[0] = b;
                        port->state = STATE_SYSEX_1;
                        break;
                case STATE_SYSEX_1:
                        port->data[1] = b;
                        port->state = STATE_SYSEX_2;
                        break;
                case STATE_SYSEX_2:
                        output_packet(urb, p0 | 0x04, port->data[0],
                                      port->data[1], b);
                        port->state = STATE_SYSEX_0;
                        break;
                }
        }
}

static void snd_usbmidi_standard_output(struct snd_usb_midi_out_endpoint *ep,
                                        struct urb *urb)
{
        int p;

        /* FIXME: lower-numbered ports can starve higher-numbered ports */
        for (p = 0; p < 0x10; ++p) {
                struct usbmidi_out_port *port = &ep->ports[p];
                if (!port->active)
                        continue;
                while (urb->transfer_buffer_length + 3 < ep->max_transfer) {
                        uint8_t b;
                        if (snd_rawmidi_transmit(port->substream, &b, 1) != 1) {
                                port->active = 0;
                                break;
                        }
                        snd_usbmidi_transmit_byte(port, b, urb);
                }
        }
}

static const struct usb_protocol_ops snd_usbmidi_standard_ops = {
        .input = snd_usbmidi_standard_input,
        .output = snd_usbmidi_standard_output,
        .output_packet = snd_usbmidi_output_standard_packet,
};

static const struct usb_protocol_ops snd_usbmidi_midiman_ops = {
        .input = snd_usbmidi_midiman_input,
        .output = snd_usbmidi_standard_output,
        .output_packet = snd_usbmidi_output_midiman_packet,
};

static const
struct usb_protocol_ops snd_usbmidi_maudio_broken_running_status_ops = {
        .input = snd_usbmidi_maudio_broken_running_status_input,
        .output = snd_usbmidi_standard_output,
        .output_packet = snd_usbmidi_output_standard_packet,
};

static const struct usb_protocol_ops snd_usbmidi_cme_ops = {
        .input = snd_usbmidi_cme_input,
        .output = snd_usbmidi_standard_output,
        .output_packet = snd_usbmidi_output_standard_packet,
};

static const struct usb_protocol_ops snd_usbmidi_ch345_broken_sysex_ops = {
        .input = ch345_broken_sysex_input,
        .output = snd_usbmidi_standard_output,
        .output_packet = snd_usbmidi_output_standard_packet,
};

/*
 * AKAI MPD16 protocol:
 *
 * For control port (endpoint 1):
 * ==============================
 * One or more chunks consisting of first byte of (0x10 | msg_len) and then a
 * SysEx message (msg_len=9 bytes long).
 *
 * For data port (endpoint 2):
 * ===========================
 * One or more chunks consisting of first byte of (0x20 | msg_len) and then a
 * MIDI message (msg_len bytes long)
 *
 * Messages sent: Active Sense, Note On, Poly Pressure, Control Change.
 */
static void snd_usbmidi_akai_input(struct snd_usb_midi_in_endpoint *ep,
                                   uint8_t *buffer, int buffer_length)
{
        unsigned int pos = 0;
        unsigned int len = (unsigned int)buffer_length;
        while (pos < len) {
                unsigned int port = (buffer[pos] >> 4) - 1;
                unsigned int msg_len = buffer[pos] & 0x0f;
                pos++;
                if (pos + msg_len <= len && port < 2)
                        snd_usbmidi_input_data(ep, 0, &buffer[pos], msg_len);
                pos += msg_len;
        }
}

#define MAX_AKAI_SYSEX_LEN 9

static void snd_usbmidi_akai_output(struct snd_usb_midi_out_endpoint *ep,
                                    struct urb *urb)
{
        uint8_t *msg;
        int pos, end, count, buf_end;
        uint8_t tmp[MAX_AKAI_SYSEX_LEN];
        struct snd_rawmidi_substream *substream = ep->ports[0].substream;

        if (!ep->ports[0].active)
                return;

        msg = urb->transfer_buffer + urb->transfer_buffer_length;
        buf_end = ep->max_transfer - MAX_AKAI_SYSEX_LEN - 1;

        /* only try adding more data when there's space for at least 1 SysEx */
        while (urb->transfer_buffer_length < buf_end) {
                count = snd_rawmidi_transmit_peek(substream,
                                                  tmp, MAX_AKAI_SYSEX_LEN);
                if (!count) {
                        ep->ports[0].active = 0;
                        return;
                }
                /* try to skip non-SysEx data */
                for (pos = 0; pos < count && tmp[pos] != 0xF0; pos++)
                        ;

                if (pos > 0) {
                        snd_rawmidi_transmit_ack(substream, pos);
                        continue;
                }

                /* look for the start or end marker */
                for (end = 1; end < count && tmp[end] < 0xF0; end++)
                        ;

                /* next SysEx started before the end of current one */
                if (end < count && tmp[end] == 0xF0) {
                        /* it's incomplete - drop it */
                        snd_rawmidi_transmit_ack(substream, end);
                        continue;
                }
                /* SysEx complete */
                if (end < count && tmp[end] == 0xF7) {
                        /* queue it, ack it, and get the next one */
                        count = end + 1;
                        msg[0] = 0x10 | count;
                        memcpy(&msg[1], tmp, count);
                        snd_rawmidi_transmit_ack(substream, count);
                        urb->transfer_buffer_length += count + 1;
                        msg += count + 1;
                        continue;
                }
                /* less than 9 bytes and no end byte - wait for more */
                if (count < MAX_AKAI_SYSEX_LEN) {
                        ep->ports[0].active = 0;
                        return;
                }
                /* 9 bytes and no end marker in sight - malformed, skip it */
                snd_rawmidi_transmit_ack(substream, count);
        }
}

static const struct usb_protocol_ops snd_usbmidi_akai_ops = {
        .input = snd_usbmidi_akai_input,
        .output = snd_usbmidi_akai_output,
};

/*
 * Novation USB MIDI protocol: number of data bytes is in the first byte
 * (when receiving) (+1!) or in the second byte (when sending); data begins
 * at the third byte.
 */

static void snd_usbmidi_novation_input(struct snd_usb_midi_in_endpoint *ep,
                                       uint8_t *buffer, int buffer_length)
{
        if (buffer_length < 2 || !buffer[0] || buffer_length < buffer[0] + 1)
                return;
        snd_usbmidi_input_data(ep, 0, &buffer[2], buffer[0] - 1);
}

static void snd_usbmidi_novation_output(struct snd_usb_midi_out_endpoint *ep,
                                        struct urb *urb)
{
        uint8_t *transfer_buffer;
        int count;

        if (!ep->ports[0].active)
                return;
        transfer_buffer = urb->transfer_buffer;
        count = snd_rawmidi_transmit(ep->ports[0].substream,
                                     &transfer_buffer[2],
                                     ep->max_transfer - 2);
        if (count < 1) {
                ep->ports[0].active = 0;
                return;
        }
        transfer_buffer[0] = 0;
        transfer_buffer[1] = count;
        urb->transfer_buffer_length = 2 + count;
}

static const struct usb_protocol_ops snd_usbmidi_novation_ops = {
        .input = snd_usbmidi_novation_input,
        .output = snd_usbmidi_novation_output,
};

/*
 * "raw" protocol: just move raw MIDI bytes from/to the endpoint
 */

static void snd_usbmidi_raw_input(struct snd_usb_midi_in_endpoint *ep,
                                  uint8_t *buffer, int buffer_length)
{
        snd_usbmidi_input_data(ep, 0, buffer, buffer_length);
}

static void snd_usbmidi_raw_output(struct snd_usb_midi_out_endpoint *ep,
                                   struct urb *urb)
{
        int count;

        if (!ep->ports[0].active)
                return;
        count = snd_rawmidi_transmit(ep->ports[0].substream,
                                     urb->transfer_buffer,
                                     ep->max_transfer);
        if (count < 1) {
                ep->ports[0].active = 0;
                return;
        }
        urb->transfer_buffer_length = count;
}

static const struct usb_protocol_ops snd_usbmidi_raw_ops = {
        .input = snd_usbmidi_raw_input,
        .output = snd_usbmidi_raw_output,
};

/*
 * FTDI protocol: raw MIDI bytes, but input packets have two modem status bytes.
 */

static void snd_usbmidi_ftdi_input(struct snd_usb_midi_in_endpoint *ep,
                                   uint8_t *buffer, int buffer_length)
{
        if (buffer_length > 2)
                snd_usbmidi_input_data(ep, 0, buffer + 2, buffer_length - 2);
}

static const struct usb_protocol_ops snd_usbmidi_ftdi_ops = {
        .input = snd_usbmidi_ftdi_input,
        .output = snd_usbmidi_raw_output,
};

static void snd_usbmidi_us122l_input(struct snd_usb_midi_in_endpoint *ep,
                                     uint8_t *buffer, int buffer_length)
{
        if (buffer_length != 9)
                return;
        buffer_length = 8;
        while (buffer_length && buffer[buffer_length - 1] == 0xFD)
                buffer_length--;
        if (buffer_length)
                snd_usbmidi_input_data(ep, 0, buffer, buffer_length);
}

static void snd_usbmidi_us122l_output(struct snd_usb_midi_out_endpoint *ep,
                                      struct urb *urb)
{
        int count;

        if (!ep->ports[0].active)
                return;
        switch (snd_usb_get_speed(ep->umidi->dev)) {
        case USB_SPEED_HIGH:
        case USB_SPEED_SUPER:
        case USB_SPEED_SUPER_PLUS:
                count = 1;
                break;
        default:
                count = 2;
        }
        count = snd_rawmidi_transmit(ep->ports[0].substream,
                                     urb->transfer_buffer,
                                     count);
        if (count < 1) {
                ep->ports[0].active = 0;
                return;
        }

        memset(urb->transfer_buffer + count, 0xFD, ep->max_transfer - count);
        urb->transfer_buffer_length = ep->max_transfer;
}

static const struct usb_protocol_ops snd_usbmidi_122l_ops = {
        .input = snd_usbmidi_us122l_input,
        .output = snd_usbmidi_us122l_output,
};

/*
 * Emagic USB MIDI protocol: raw MIDI with "F5 xx" port switching.
 */

static void snd_usbmidi_emagic_init_out(struct snd_usb_midi_out_endpoint *ep)
{
        static const u8 init_data[] = {
                /* initialization magic: "get version" */
                0xf0,
                0x00, 0x20, 0x31,        /* Emagic */
                0x64,                        /* Unitor8 */
                0x0b,                        /* version number request */
                0x00,                        /* command version */
                0x00,                        /* EEPROM, box 0 */
                0xf7
        };
        send_bulk_static_data(ep, init_data, sizeof(init_data));
        /* while we're at it, pour on more magic */
        send_bulk_static_data(ep, init_data, sizeof(init_data));
}

static void snd_usbmidi_emagic_finish_out(struct snd_usb_midi_out_endpoint *ep)
{
        static const u8 finish_data[] = {
                /* switch to patch mode with last preset */
                0xf0,
                0x00, 0x20, 0x31,        /* Emagic */
                0x64,                        /* Unitor8 */
                0x10,                        /* patch switch command */
                0x00,                        /* command version */
                0x7f,                        /* to all boxes */
                0x40,                        /* last preset in EEPROM */
                0xf7
        };
        send_bulk_static_data(ep, finish_data, sizeof(finish_data));
}

static void snd_usbmidi_emagic_input(struct snd_usb_midi_in_endpoint *ep,
                                     uint8_t *buffer, int buffer_length)
{
        int i;

        /* FF indicates end of valid data */
        for (i = 0; i < buffer_length; ++i)
                if (buffer[i] == 0xff) {
                        buffer_length = i;
                        break;
                }

        /* handle F5 at end of last buffer */
        if (ep->seen_f5)
                goto switch_port;

        while (buffer_length > 0) {
                /* determine size of data until next F5 */
                for (i = 0; i < buffer_length; ++i)
                        if (buffer[i] == 0xf5)
                                break;
                snd_usbmidi_input_data(ep, ep->current_port, buffer, i);
                buffer += i;
                buffer_length -= i;

                if (buffer_length <= 0)
                        break;
                /* assert(buffer[0] == 0xf5); */
                ep->seen_f5 = 1;
                ++buffer;
                --buffer_length;

        switch_port:
                if (buffer_length <= 0)
                        break;
                if (buffer[0] < 0x80) {
                        ep->current_port = (buffer[0] - 1) & 15;
                        ++buffer;
                        --buffer_length;
                }
                ep->seen_f5 = 0;
        }
}

static void snd_usbmidi_emagic_output(struct snd_usb_midi_out_endpoint *ep,
                                      struct urb *urb)
{
        int port0 = ep->current_port;
        uint8_t *buf = urb->transfer_buffer;
        int buf_free = ep->max_transfer;
        int length, i;

        for (i = 0; i < 0x10; ++i) {
                /* round-robin, starting at the last current port */
                int portnum = (port0 + i) & 15;
                struct usbmidi_out_port *port = &ep->ports[portnum];

                if (!port->active)
                        continue;
                if (snd_rawmidi_transmit_peek(port->substream, buf, 1) != 1) {
                        port->active = 0;
                        continue;
                }

                if (portnum != ep->current_port) {
                        if (buf_free < 2)
                                break;
                        ep->current_port = portnum;
                        buf[0] = 0xf5;
                        buf[1] = (portnum + 1) & 15;
                        buf += 2;
                        buf_free -= 2;
                }

                if (buf_free < 1)
                        break;
                length = snd_rawmidi_transmit(port->substream, buf, buf_free);
                if (length > 0) {
                        buf += length;
                        buf_free -= length;
                        if (buf_free < 1)
                                break;
                }
        }
        if (buf_free < ep->max_transfer && buf_free > 0) {
                *buf = 0xff;
                --buf_free;
        }
        urb->transfer_buffer_length = ep->max_transfer - buf_free;
}

static const struct usb_protocol_ops snd_usbmidi_emagic_ops = {
        .input = snd_usbmidi_emagic_input,
        .output = snd_usbmidi_emagic_output,
        .init_out_endpoint = snd_usbmidi_emagic_init_out,
        .finish_out_endpoint = snd_usbmidi_emagic_finish_out,
};


static void update_roland_altsetting(struct snd_usb_midi *umidi)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;
        int is_light_load;

        intf = umidi->iface;
        is_light_load = intf->cur_altsetting != intf->altsetting;
        if (umidi->roland_load_ctl->private_value == is_light_load)
                return;
        hostif = &intf->altsetting[umidi->roland_load_ctl->private_value];
        intfd = get_iface_desc(hostif);
        snd_usbmidi_input_stop(&umidi->list);
        usb_set_interface(umidi->dev, intfd->bInterfaceNumber,
                          intfd->bAlternateSetting);
        snd_usbmidi_input_start(&umidi->list);
}

static int substream_open(struct snd_rawmidi_substream *substream, int dir,
                          int open)
{
        struct snd_usb_midi *umidi = substream->rmidi->private_data;
        struct snd_kcontrol *ctl;

        down_read(&umidi->disc_rwsem);
        if (umidi->disconnected) {
                up_read(&umidi->disc_rwsem);
                return open ? -ENODEV : 0;
        }

        mutex_lock(&umidi->mutex);
        if (open) {
                if (!umidi->opened[0] && !umidi->opened[1]) {
                        if (umidi->roland_load_ctl) {
                                ctl = umidi->roland_load_ctl;
                                ctl->vd[0].access |=
                                        SNDRV_CTL_ELEM_ACCESS_INACTIVE;
                                snd_ctl_notify(umidi->card,
                                       SNDRV_CTL_EVENT_MASK_INFO, &ctl->id);
                                update_roland_altsetting(umidi);
                        }
                }
                umidi->opened[dir]++;
                if (umidi->opened[1])
                        snd_usbmidi_input_start(&umidi->list);
        } else {
                umidi->opened[dir]--;
                if (!umidi->opened[1])
                        snd_usbmidi_input_stop(&umidi->list);
                if (!umidi->opened[0] && !umidi->opened[1]) {
                        if (umidi->roland_load_ctl) {
                                ctl = umidi->roland_load_ctl;
                                ctl->vd[0].access &=
                                        ~SNDRV_CTL_ELEM_ACCESS_INACTIVE;
                                snd_ctl_notify(umidi->card,
                                       SNDRV_CTL_EVENT_MASK_INFO, &ctl->id);
                        }
                }
        }
        mutex_unlock(&umidi->mutex);
        up_read(&umidi->disc_rwsem);
        return 0;
}

static int snd_usbmidi_output_open(struct snd_rawmidi_substream *substream)
{
        struct snd_usb_midi *umidi = substream->rmidi->private_data;
        struct usbmidi_out_port *port = NULL;
        int i, j;

        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i)
                if (umidi->endpoints[i].out)
                        for (j = 0; j < 0x10; ++j)
                                if (umidi->endpoints[i].out->ports[j].substream == substream) {
                                        port = &umidi->endpoints[i].out->ports[j];
                                        break;
                                }
        if (!port)
                return -ENXIO;

        substream->runtime->private_data = port;
        port->state = STATE_UNKNOWN;
        return substream_open(substream, 0, 1);
}

static int snd_usbmidi_output_close(struct snd_rawmidi_substream *substream)
{
        struct usbmidi_out_port *port = substream->runtime->private_data;

        cancel_work_sync(&port->ep->work);
        return substream_open(substream, 0, 0);
}

static void snd_usbmidi_output_trigger(struct snd_rawmidi_substream *substream,
                                       int up)
{
        struct usbmidi_out_port *port =
                (struct usbmidi_out_port *)substream->runtime->private_data;

        port->active = up;
        if (up) {
                if (port->ep->umidi->disconnected) {
                        /* gobble up remaining bytes to prevent wait in
                         * snd_rawmidi_drain_output */
                        snd_rawmidi_proceed(substream);
                        return;
                }
                queue_work(system_highpri_wq, &port->ep->work);
        }
}

static void snd_usbmidi_output_drain(struct snd_rawmidi_substream *substream)
{
        struct usbmidi_out_port *port = substream->runtime->private_data;
        struct snd_usb_midi_out_endpoint *ep = port->ep;
        unsigned int drain_urbs;
        DEFINE_WAIT(wait);
        long timeout = msecs_to_jiffies(50);

        if (ep->umidi->disconnected)
                return;
        /*
         * The substream buffer is empty, but some data might still be in the
         * currently active URBs, so we have to wait for those to complete.
         */
        spin_lock_irq(&ep->buffer_lock);
        drain_urbs = ep->active_urbs;
        if (drain_urbs) {
                ep->drain_urbs |= drain_urbs;
                do {
                        prepare_to_wait(&ep->drain_wait, &wait,
                                        TASK_UNINTERRUPTIBLE);
                        spin_unlock_irq(&ep->buffer_lock);
                        timeout = schedule_timeout(timeout);
                        spin_lock_irq(&ep->buffer_lock);
                        drain_urbs &= ep->drain_urbs;
                } while (drain_urbs && timeout);
                finish_wait(&ep->drain_wait, &wait);
        }
        port->active = 0;
        spin_unlock_irq(&ep->buffer_lock);
}

static int snd_usbmidi_input_open(struct snd_rawmidi_substream *substream)
{
        return substream_open(substream, 1, 1);
}

static int snd_usbmidi_input_close(struct snd_rawmidi_substream *substream)
{
        return substream_open(substream, 1, 0);
}

static void snd_usbmidi_input_trigger(struct snd_rawmidi_substream *substream,
                                      int up)
{
        struct snd_usb_midi *umidi = substream->rmidi->private_data;

        if (up)
                set_bit(substream->number, &umidi->input_triggered);
        else
                clear_bit(substream->number, &umidi->input_triggered);
}

static const struct snd_rawmidi_ops snd_usbmidi_output_ops = {
        .open = snd_usbmidi_output_open,
        .close = snd_usbmidi_output_close,
        .trigger = snd_usbmidi_output_trigger,
        .drain = snd_usbmidi_output_drain,
};

static const struct snd_rawmidi_ops snd_usbmidi_input_ops = {
        .open = snd_usbmidi_input_open,
        .close = snd_usbmidi_input_close,
        .trigger = snd_usbmidi_input_trigger
};

static void free_urb_and_buffer(struct snd_usb_midi *umidi, struct urb *urb,
                                unsigned int buffer_length)
{
        usb_free_coherent(umidi->dev, buffer_length,
                          urb->transfer_buffer, urb->transfer_dma);
        usb_free_urb(urb);
}

/*
 * Frees an input endpoint.
 * May be called when ep hasn't been initialized completely.
 */
static void snd_usbmidi_in_endpoint_delete(struct snd_usb_midi_in_endpoint *ep)
{
        unsigned int i;

        for (i = 0; i < INPUT_URBS; ++i)
                if (ep->urbs[i])
                        free_urb_and_buffer(ep->umidi, ep->urbs[i],
                                            ep->urbs[i]->transfer_buffer_length);
        kfree(ep);
}

/*
 * Creates an input endpoint.
 */
static int snd_usbmidi_in_endpoint_create(struct snd_usb_midi *umidi,
                                          struct snd_usb_midi_endpoint_info *ep_info,
                                          struct snd_usb_midi_endpoint *rep)
{
        struct snd_usb_midi_in_endpoint *ep;
        void *buffer;
        unsigned int pipe;
        int length;
        unsigned int i;
        int err;

        rep->in = NULL;
        ep = kzalloc(sizeof(*ep), GFP_KERNEL);
        if (!ep)
                return -ENOMEM;
        ep->umidi = umidi;

        for (i = 0; i < INPUT_URBS; ++i) {
                ep->urbs[i] = usb_alloc_urb(0, GFP_KERNEL);
                if (!ep->urbs[i]) {
                        err = -ENOMEM;
                        goto error;
                }
        }
        if (ep_info->in_interval)
                pipe = usb_rcvintpipe(umidi->dev, ep_info->in_ep);
        else
                pipe = usb_rcvbulkpipe(umidi->dev, ep_info->in_ep);
        length = usb_maxpacket(umidi->dev, pipe);
        for (i = 0; i < INPUT_URBS; ++i) {
                buffer = usb_alloc_coherent(umidi->dev, length, GFP_KERNEL,
                                            &ep->urbs[i]->transfer_dma);
                if (!buffer) {
                        err = -ENOMEM;
                        goto error;
                }
                if (ep_info->in_interval)
                        usb_fill_int_urb(ep->urbs[i], umidi->dev,
                                         pipe, buffer, length,
                                         snd_usbmidi_in_urb_complete,
                                         ep, ep_info->in_interval);
                else
                        usb_fill_bulk_urb(ep->urbs[i], umidi->dev,
                                          pipe, buffer, length,
                                          snd_usbmidi_in_urb_complete, ep);
                ep->urbs[i]->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
                err = usb_urb_ep_type_check(ep->urbs[i]);
                if (err < 0) {
                        dev_err(&umidi->dev->dev, "invalid MIDI in EP %x\n",
                                ep_info->in_ep);
                        goto error;
                }
        }

        rep->in = ep;
        return 0;

 error:
        snd_usbmidi_in_endpoint_delete(ep);
        return err;
}

/*
 * Frees an output endpoint.
 * May be called when ep hasn't been initialized completely.
 */
static void snd_usbmidi_out_endpoint_clear(struct snd_usb_midi_out_endpoint *ep)
{
        unsigned int i;

        for (i = 0; i < OUTPUT_URBS; ++i)
                if (ep->urbs[i].urb) {
                        free_urb_and_buffer(ep->umidi, ep->urbs[i].urb,
                                            ep->max_transfer);
                        ep->urbs[i].urb = NULL;
                }
}

static void snd_usbmidi_out_endpoint_delete(struct snd_usb_midi_out_endpoint *ep)
{
        snd_usbmidi_out_endpoint_clear(ep);
        kfree(ep);
}

/*
 * Creates an output endpoint, and initializes output ports.
 */
static int snd_usbmidi_out_endpoint_create(struct snd_usb_midi *umidi,
                                           struct snd_usb_midi_endpoint_info *ep_info,
                                           struct snd_usb_midi_endpoint *rep)
{
        struct snd_usb_midi_out_endpoint *ep;
        unsigned int i;
        unsigned int pipe;
        void *buffer;
        int err;

        rep->out = NULL;
        ep = kzalloc(sizeof(*ep), GFP_KERNEL);
        if (!ep)
                return -ENOMEM;
        ep->umidi = umidi;

        for (i = 0; i < OUTPUT_URBS; ++i) {
                ep->urbs[i].urb = usb_alloc_urb(0, GFP_KERNEL);
                if (!ep->urbs[i].urb) {
                        err = -ENOMEM;
                        goto error;
                }
                ep->urbs[i].ep = ep;
        }
        if (ep_info->out_interval)
                pipe = usb_sndintpipe(umidi->dev, ep_info->out_ep);
        else
                pipe = usb_sndbulkpipe(umidi->dev, ep_info->out_ep);
        switch (umidi->usb_id) {
        default:
                ep->max_transfer = usb_maxpacket(umidi->dev, pipe);
                break;
                /*
                 * Various chips declare a packet size larger than 4 bytes, but
                 * do not actually work with larger packets:
                 */
        case USB_ID(0x0a67, 0x5011): /* Medeli DD305 */
        case USB_ID(0x0a92, 0x1020): /* ESI M4U */
        case USB_ID(0x1430, 0x474b): /* RedOctane GH MIDI INTERFACE */
        case USB_ID(0x15ca, 0x0101): /* Textech USB Midi Cable */
        case USB_ID(0x15ca, 0x1806): /* Textech USB Midi Cable */
        case USB_ID(0x1a86, 0x752d): /* QinHeng CH345 "USB2.0-MIDI" */
        case USB_ID(0xfc08, 0x0101): /* Unknown vendor Cable */
                ep->max_transfer = 4;
                break;
                /*
                 * Some devices only work with 9 bytes packet size:
                 */
        case USB_ID(0x0644, 0x800e): /* Tascam US-122L */
        case USB_ID(0x0644, 0x800f): /* Tascam US-144 */
                ep->max_transfer = 9;
                break;
        }
        for (i = 0; i < OUTPUT_URBS; ++i) {
                buffer = usb_alloc_coherent(umidi->dev,
                                            ep->max_transfer, GFP_KERNEL,
                                            &ep->urbs[i].urb->transfer_dma);
                if (!buffer) {
                        err = -ENOMEM;
                        goto error;
                }
                if (ep_info->out_interval)
                        usb_fill_int_urb(ep->urbs[i].urb, umidi->dev,
                                         pipe, buffer, ep->max_transfer,
                                         snd_usbmidi_out_urb_complete,
                                         &ep->urbs[i], ep_info->out_interval);
                else
                        usb_fill_bulk_urb(ep->urbs[i].urb, umidi->dev,
                                          pipe, buffer, ep->max_transfer,
                                          snd_usbmidi_out_urb_complete,
                                          &ep->urbs[i]);
                err = usb_urb_ep_type_check(ep->urbs[i].urb);
                if (err < 0) {
                        dev_err(&umidi->dev->dev, "invalid MIDI out EP %x\n",
                                ep_info->out_ep);
                        goto error;
                }
                ep->urbs[i].urb->transfer_flags = URB_NO_TRANSFER_DMA_MAP;
        }

        spin_lock_init(&ep->buffer_lock);
        INIT_WORK(&ep->work, snd_usbmidi_out_work);
        init_waitqueue_head(&ep->drain_wait);

        for (i = 0; i < 0x10; ++i)
                if (ep_info->out_cables & (1 << i)) {
                        ep->ports[i].ep = ep;
                        ep->ports[i].cable = i << 4;
                }

        if (umidi->usb_protocol_ops->init_out_endpoint)
                umidi->usb_protocol_ops->init_out_endpoint(ep);

        rep->out = ep;
        return 0;

 error:
        snd_usbmidi_out_endpoint_delete(ep);
        return err;
}

/*
 * Frees everything.
 */
static void snd_usbmidi_free(struct snd_usb_midi *umidi)
{
        int i;

        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                struct snd_usb_midi_endpoint *ep = &umidi->endpoints[i];
                if (ep->out)
                        snd_usbmidi_out_endpoint_delete(ep->out);
                if (ep->in)
                        snd_usbmidi_in_endpoint_delete(ep->in);
        }
        mutex_destroy(&umidi->mutex);
        kfree(umidi);
}

/*
 * Unlinks all URBs (must be done before the usb_device is deleted).
 */
void snd_usbmidi_disconnect(struct list_head *p)
{
        struct snd_usb_midi *umidi;
        unsigned int i, j;

        umidi = list_entry(p, struct snd_usb_midi, list);
        /*
         * an URB's completion handler may start the timer and
         * a timer may submit an URB. To reliably break the cycle
         * a flag under lock must be used
         */
        down_write(&umidi->disc_rwsem);
        spin_lock_irq(&umidi->disc_lock);
        umidi->disconnected = 1;
        spin_unlock_irq(&umidi->disc_lock);
        up_write(&umidi->disc_rwsem);

        del_timer_sync(&umidi->error_timer);

        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                struct snd_usb_midi_endpoint *ep = &umidi->endpoints[i];
                if (ep->out)
                        cancel_work_sync(&ep->out->work);
                if (ep->out) {
                        for (j = 0; j < OUTPUT_URBS; ++j)
                                usb_kill_urb(ep->out->urbs[j].urb);
                        if (umidi->usb_protocol_ops->finish_out_endpoint)
                                umidi->usb_protocol_ops->finish_out_endpoint(ep->out);
                        ep->out->active_urbs = 0;
                        if (ep->out->drain_urbs) {
                                ep->out->drain_urbs = 0;
                                wake_up(&ep->out->drain_wait);
                        }
                }
                if (ep->in)
                        for (j = 0; j < INPUT_URBS; ++j)
                                usb_kill_urb(ep->in->urbs[j]);
                /* free endpoints here; later call can result in Oops */
                if (ep->out)
                        snd_usbmidi_out_endpoint_clear(ep->out);
                if (ep->in) {
                        snd_usbmidi_in_endpoint_delete(ep->in);
                        ep->in = NULL;
                }
        }
}
EXPORT_SYMBOL(snd_usbmidi_disconnect);

static void snd_usbmidi_rawmidi_free(struct snd_rawmidi *rmidi)
{
        struct snd_usb_midi *umidi = rmidi->private_data;
        snd_usbmidi_free(umidi);
}

static struct snd_rawmidi_substream *snd_usbmidi_find_substream(struct snd_usb_midi *umidi,
                                                                int stream,
                                                                int number)
{
        struct snd_rawmidi_substream *substream;

        list_for_each_entry(substream, &umidi->rmidi->streams[stream].substreams,
                            list) {
                if (substream->number == number)
                        return substream;
        }
        return NULL;
}

/*
 * This list specifies names for ports that do not fit into the standard
 * "(product) MIDI (n)" schema because they aren't external MIDI ports,
 * such as internal control or synthesizer ports.
 */
static struct port_info {
        u32 id;
        short int port;
        short int voices;
        const char *name;
        unsigned int seq_flags;
} snd_usbmidi_port_info[] = {
#define PORT_INFO(vendor, product, num, name_, voices_, flags) \
        { .id = USB_ID(vendor, product), \
          .port = num, .voices = voices_, \
          .name = name_, .seq_flags = flags }
#define EXTERNAL_PORT(vendor, product, num, name) \
        PORT_INFO(vendor, product, num, name, 0, \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC | \
                  SNDRV_SEQ_PORT_TYPE_HARDWARE | \
                  SNDRV_SEQ_PORT_TYPE_PORT)
#define CONTROL_PORT(vendor, product, num, name) \
        PORT_INFO(vendor, product, num, name, 0, \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC | \
                  SNDRV_SEQ_PORT_TYPE_HARDWARE)
#define GM_SYNTH_PORT(vendor, product, num, name, voices) \
        PORT_INFO(vendor, product, num, name, voices, \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GM | \
                  SNDRV_SEQ_PORT_TYPE_HARDWARE | \
                  SNDRV_SEQ_PORT_TYPE_SYNTHESIZER)
#define ROLAND_SYNTH_PORT(vendor, product, num, name, voices) \
        PORT_INFO(vendor, product, num, name, voices, \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GM | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GM2 | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GS | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_XG | \
                  SNDRV_SEQ_PORT_TYPE_HARDWARE | \
                  SNDRV_SEQ_PORT_TYPE_SYNTHESIZER)
#define SOUNDCANVAS_PORT(vendor, product, num, name, voices) \
        PORT_INFO(vendor, product, num, name, voices, \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GM | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GM2 | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_GS | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_XG | \
                  SNDRV_SEQ_PORT_TYPE_MIDI_MT32 | \
                  SNDRV_SEQ_PORT_TYPE_HARDWARE | \
                  SNDRV_SEQ_PORT_TYPE_SYNTHESIZER)
        /* Yamaha MOTIF XF */
        GM_SYNTH_PORT(0x0499, 0x105c, 0, "%s Tone Generator", 128),
        CONTROL_PORT(0x0499, 0x105c, 1, "%s Remote Control"),
        EXTERNAL_PORT(0x0499, 0x105c, 2, "%s Thru"),
        CONTROL_PORT(0x0499, 0x105c, 3, "%s Editor"),
        /* Roland UA-100 */
        CONTROL_PORT(0x0582, 0x0000, 2, "%s Control"),
        /* Roland SC-8850 */
        SOUNDCANVAS_PORT(0x0582, 0x0003, 0, "%s Part A", 128),
        SOUNDCANVAS_PORT(0x0582, 0x0003, 1, "%s Part B", 128),
        SOUNDCANVAS_PORT(0x0582, 0x0003, 2, "%s Part C", 128),
        SOUNDCANVAS_PORT(0x0582, 0x0003, 3, "%s Part D", 128),
        EXTERNAL_PORT(0x0582, 0x0003, 4, "%s MIDI 1"),
        EXTERNAL_PORT(0x0582, 0x0003, 5, "%s MIDI 2"),
        /* Roland U-8 */
        EXTERNAL_PORT(0x0582, 0x0004, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x0004, 1, "%s Control"),
        /* Roland SC-8820 */
        SOUNDCANVAS_PORT(0x0582, 0x0007, 0, "%s Part A", 64),
        SOUNDCANVAS_PORT(0x0582, 0x0007, 1, "%s Part B", 64),
        EXTERNAL_PORT(0x0582, 0x0007, 2, "%s MIDI"),
        /* Roland SK-500 */
        SOUNDCANVAS_PORT(0x0582, 0x000b, 0, "%s Part A", 64),
        SOUNDCANVAS_PORT(0x0582, 0x000b, 1, "%s Part B", 64),
        EXTERNAL_PORT(0x0582, 0x000b, 2, "%s MIDI"),
        /* Roland SC-D70 */
        SOUNDCANVAS_PORT(0x0582, 0x000c, 0, "%s Part A", 64),
        SOUNDCANVAS_PORT(0x0582, 0x000c, 1, "%s Part B", 64),
        EXTERNAL_PORT(0x0582, 0x000c, 2, "%s MIDI"),
        /* Edirol UM-880 */
        CONTROL_PORT(0x0582, 0x0014, 8, "%s Control"),
        /* Edirol SD-90 */
        ROLAND_SYNTH_PORT(0x0582, 0x0016, 0, "%s Part A", 128),
        ROLAND_SYNTH_PORT(0x0582, 0x0016, 1, "%s Part B", 128),
        EXTERNAL_PORT(0x0582, 0x0016, 2, "%s MIDI 1"),
        EXTERNAL_PORT(0x0582, 0x0016, 3, "%s MIDI 2"),
        /* Edirol UM-550 */
        CONTROL_PORT(0x0582, 0x0023, 5, "%s Control"),
        /* Edirol SD-20 */
        ROLAND_SYNTH_PORT(0x0582, 0x0027, 0, "%s Part A", 64),
        ROLAND_SYNTH_PORT(0x0582, 0x0027, 1, "%s Part B", 64),
        EXTERNAL_PORT(0x0582, 0x0027, 2, "%s MIDI"),
        /* Edirol SD-80 */
        ROLAND_SYNTH_PORT(0x0582, 0x0029, 0, "%s Part A", 128),
        ROLAND_SYNTH_PORT(0x0582, 0x0029, 1, "%s Part B", 128),
        EXTERNAL_PORT(0x0582, 0x0029, 2, "%s MIDI 1"),
        EXTERNAL_PORT(0x0582, 0x0029, 3, "%s MIDI 2"),
        /* Edirol UA-700 */
        EXTERNAL_PORT(0x0582, 0x002b, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x002b, 1, "%s Control"),
        /* Roland VariOS */
        EXTERNAL_PORT(0x0582, 0x002f, 0, "%s MIDI"),
        EXTERNAL_PORT(0x0582, 0x002f, 1, "%s External MIDI"),
        EXTERNAL_PORT(0x0582, 0x002f, 2, "%s Sync"),
        /* Edirol PCR */
        EXTERNAL_PORT(0x0582, 0x0033, 0, "%s MIDI"),
        EXTERNAL_PORT(0x0582, 0x0033, 1, "%s 1"),
        EXTERNAL_PORT(0x0582, 0x0033, 2, "%s 2"),
        /* BOSS GS-10 */
        EXTERNAL_PORT(0x0582, 0x003b, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x003b, 1, "%s Control"),
        /* Edirol UA-1000 */
        EXTERNAL_PORT(0x0582, 0x0044, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x0044, 1, "%s Control"),
        /* Edirol UR-80 */
        EXTERNAL_PORT(0x0582, 0x0048, 0, "%s MIDI"),
        EXTERNAL_PORT(0x0582, 0x0048, 1, "%s 1"),
        EXTERNAL_PORT(0x0582, 0x0048, 2, "%s 2"),
        /* Edirol PCR-A */
        EXTERNAL_PORT(0x0582, 0x004d, 0, "%s MIDI"),
        EXTERNAL_PORT(0x0582, 0x004d, 1, "%s 1"),
        EXTERNAL_PORT(0x0582, 0x004d, 2, "%s 2"),
        /* BOSS GT-PRO */
        CONTROL_PORT(0x0582, 0x0089, 0, "%s Control"),
        /* Edirol UM-3EX */
        CONTROL_PORT(0x0582, 0x009a, 3, "%s Control"),
        /* Roland VG-99 */
        CONTROL_PORT(0x0582, 0x00b2, 0, "%s Control"),
        EXTERNAL_PORT(0x0582, 0x00b2, 1, "%s MIDI"),
        /* Cakewalk Sonar V-Studio 100 */
        EXTERNAL_PORT(0x0582, 0x00eb, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x00eb, 1, "%s Control"),
        /* Roland VB-99 */
        CONTROL_PORT(0x0582, 0x0102, 0, "%s Control"),
        EXTERNAL_PORT(0x0582, 0x0102, 1, "%s MIDI"),
        /* Roland A-PRO */
        EXTERNAL_PORT(0x0582, 0x010f, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x010f, 1, "%s 1"),
        CONTROL_PORT(0x0582, 0x010f, 2, "%s 2"),
        /* Roland SD-50 */
        ROLAND_SYNTH_PORT(0x0582, 0x0114, 0, "%s Synth", 128),
        EXTERNAL_PORT(0x0582, 0x0114, 1, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x0114, 2, "%s Control"),
        /* Roland OCTA-CAPTURE */
        EXTERNAL_PORT(0x0582, 0x0120, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x0120, 1, "%s Control"),
        EXTERNAL_PORT(0x0582, 0x0121, 0, "%s MIDI"),
        CONTROL_PORT(0x0582, 0x0121, 1, "%s Control"),
        /* Roland SPD-SX */
        CONTROL_PORT(0x0582, 0x0145, 0, "%s Control"),
        EXTERNAL_PORT(0x0582, 0x0145, 1, "%s MIDI"),
        /* Roland A-Series */
        CONTROL_PORT(0x0582, 0x0156, 0, "%s Keyboard"),
        EXTERNAL_PORT(0x0582, 0x0156, 1, "%s MIDI"),
        /* Roland INTEGRA-7 */
        ROLAND_SYNTH_PORT(0x0582, 0x015b, 0, "%s Synth", 128),
        CONTROL_PORT(0x0582, 0x015b, 1, "%s Control"),
        /* M-Audio MidiSport 8x8 */
        CONTROL_PORT(0x0763, 0x1031, 8, "%s Control"),
        CONTROL_PORT(0x0763, 0x1033, 8, "%s Control"),
        /* MOTU Fastlane */
        EXTERNAL_PORT(0x07fd, 0x0001, 0, "%s MIDI A"),
        EXTERNAL_PORT(0x07fd, 0x0001, 1, "%s MIDI B"),
        /* Emagic Unitor8/AMT8/MT4 */
        EXTERNAL_PORT(0x086a, 0x0001, 8, "%s Broadcast"),
        EXTERNAL_PORT(0x086a, 0x0002, 8, "%s Broadcast"),
        EXTERNAL_PORT(0x086a, 0x0003, 4, "%s Broadcast"),
        /* Akai MPD16 */
        CONTROL_PORT(0x09e8, 0x0062, 0, "%s Control"),
        PORT_INFO(0x09e8, 0x0062, 1, "%s MIDI", 0,
                SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC |
                SNDRV_SEQ_PORT_TYPE_HARDWARE),
        /* Access Music Virus TI */
        EXTERNAL_PORT(0x133e, 0x0815, 0, "%s MIDI"),
        PORT_INFO(0x133e, 0x0815, 1, "%s Synth", 0,
                SNDRV_SEQ_PORT_TYPE_MIDI_GENERIC |
                SNDRV_SEQ_PORT_TYPE_HARDWARE |
                SNDRV_SEQ_PORT_TYPE_SYNTHESIZER),
};

static struct port_info *find_port_info(struct snd_usb_midi *umidi, int number)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(snd_usbmidi_port_info); ++i) {
                if (snd_usbmidi_port_info[i].id == umidi->usb_id &&
                    snd_usbmidi_port_info[i].port == number)
                        return &snd_usbmidi_port_info[i];
        }
        return NULL;
}

static void snd_usbmidi_get_port_info(struct snd_rawmidi *rmidi, int number,
                                      struct snd_seq_port_info *seq_port_info)
{
        struct snd_usb_midi *umidi = rmidi->private_data;
        struct port_info *port_info;

        /* TODO: read port flags from descriptors */
        port_info = find_port_info(umidi, number);
        if (port_info) {
                seq_port_info->type = port_info->seq_flags;
                seq_port_info->midi_voices = port_info->voices;
        }
}

/* return iJack for the corresponding jackID */
static int find_usb_ijack(struct usb_host_interface *hostif, uint8_t jack_id)
{
        unsigned char *extra = hostif->extra;
        int extralen = hostif->extralen;
        struct usb_descriptor_header *h;
        struct usb_midi_out_jack_descriptor *outjd;
        struct usb_midi_in_jack_descriptor *injd;
        size_t sz;

        while (extralen > 4) {
                h = (struct usb_descriptor_header *)extra;
                if (h->bDescriptorType != USB_DT_CS_INTERFACE)
                        goto next;

                outjd = (struct usb_midi_out_jack_descriptor *)h;
                if (h->bLength >= sizeof(*outjd) &&
                    outjd->bDescriptorSubtype == UAC_MIDI_OUT_JACK &&
                    outjd->bJackID == jack_id) {
                        sz = USB_DT_MIDI_OUT_SIZE(outjd->bNrInputPins);
                        if (outjd->bLength < sz)
                                goto next;
                        return *(extra + sz - 1);
                }

                injd = (struct usb_midi_in_jack_descriptor *)h;
                if (injd->bLength >= sizeof(*injd) &&
                    injd->bDescriptorSubtype == UAC_MIDI_IN_JACK &&
                    injd->bJackID == jack_id)
                        return injd->iJack;

next:
                if (!extra[0])
                        break;
                extralen -= extra[0];
                extra += extra[0];
        }
        return 0;
}

static void snd_usbmidi_init_substream(struct snd_usb_midi *umidi,
                                       int stream, int number, int jack_id,
                                       struct snd_rawmidi_substream **rsubstream)
{
        struct port_info *port_info;
        const char *name_format;
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        uint8_t jack_name_buf[32];
        uint8_t *default_jack_name = "MIDI";
        uint8_t *jack_name = default_jack_name;
        uint8_t iJack;
        int res;

        struct snd_rawmidi_substream *substream =
                snd_usbmidi_find_substream(umidi, stream, number);
        if (!substream) {
                dev_err(&umidi->dev->dev, "substream %d:%d not found\n", stream,
                        number);
                return;
        }

        intf = umidi->iface;
        if (intf && jack_id >= 0) {
                hostif = intf->cur_altsetting;
                iJack = find_usb_ijack(hostif, jack_id);
                if (iJack != 0) {
                        res = usb_string(umidi->dev, iJack, jack_name_buf,
                          ARRAY_SIZE(jack_name_buf));
                        if (res)
                                jack_name = jack_name_buf;
                }
        }

        port_info = find_port_info(umidi, number);
        name_format = port_info ? port_info->name :
                (jack_name != default_jack_name  ? "%s %s" : "%s %s %d");
        snprintf(substream->name, sizeof(substream->name),
                 name_format, umidi->card->shortname, jack_name, number + 1);

        *rsubstream = substream;
}

/*
 * Creates the endpoints and their ports.
 */
static int snd_usbmidi_create_endpoints(struct snd_usb_midi *umidi,
                                        struct snd_usb_midi_endpoint_info *endpoints)
{
        int i, j, err;
        int out_ports = 0, in_ports = 0;

        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                if (endpoints[i].out_cables) {
                        err = snd_usbmidi_out_endpoint_create(umidi,
                                                              &endpoints[i],
                                                              &umidi->endpoints[i]);
                        if (err < 0)
                                return err;
                }
                if (endpoints[i].in_cables) {
                        err = snd_usbmidi_in_endpoint_create(umidi,
                                                             &endpoints[i],
                                                             &umidi->endpoints[i]);
                        if (err < 0)
                                return err;
                }

                for (j = 0; j < 0x10; ++j) {
                        if (endpoints[i].out_cables & (1 << j)) {
                                snd_usbmidi_init_substream(umidi,
                                                           SNDRV_RAWMIDI_STREAM_OUTPUT,
                                                           out_ports,
                                                           endpoints[i].assoc_out_jacks[j],
                                                           &umidi->endpoints[i].out->ports[j].substream);
                                ++out_ports;
                        }
                        if (endpoints[i].in_cables & (1 << j)) {
                                snd_usbmidi_init_substream(umidi,
                                                           SNDRV_RAWMIDI_STREAM_INPUT,
                                                           in_ports,
                                                           endpoints[i].assoc_in_jacks[j],
                                                           &umidi->endpoints[i].in->ports[j].substream);
                                ++in_ports;
                        }
                }
        }
        dev_dbg(&umidi->dev->dev, "created %d output and %d input ports\n",
                    out_ports, in_ports);
        return 0;
}

static struct usb_ms_endpoint_descriptor *find_usb_ms_endpoint_descriptor(
                                        struct usb_host_endpoint *hostep)
{
        unsigned char *extra = hostep->extra;
        int extralen = hostep->extralen;

        while (extralen > 3) {
                struct usb_ms_endpoint_descriptor *ms_ep =
                                (struct usb_ms_endpoint_descriptor *)extra;

                if (ms_ep->bLength > 3 &&
                    ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT &&
                    ms_ep->bDescriptorSubtype == UAC_MS_GENERAL)
                        return ms_ep;
                if (!extra[0])
                        break;
                extralen -= extra[0];
                extra += extra[0];
        }
        return NULL;
}

/*
 * Returns MIDIStreaming device capabilities.
 */
static int snd_usbmidi_get_ms_info(struct snd_usb_midi *umidi,
                                   struct snd_usb_midi_endpoint_info *endpoints)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;
        struct usb_ms_header_descriptor *ms_header;
        struct usb_host_endpoint *hostep;
        struct usb_endpoint_descriptor *ep;
        struct usb_ms_endpoint_descriptor *ms_ep;
        int i, j, epidx;

        intf = umidi->iface;
        if (!intf)
                return -ENXIO;
        hostif = &intf->altsetting[0];
        intfd = get_iface_desc(hostif);
        ms_header = (struct usb_ms_header_descriptor *)hostif->extra;
        if (hostif->extralen >= 7 &&
            ms_header->bLength >= 7 &&
            ms_header->bDescriptorType == USB_DT_CS_INTERFACE &&
            ms_header->bDescriptorSubtype == UAC_HEADER)
                dev_dbg(&umidi->dev->dev, "MIDIStreaming version %02x.%02x\n",
                            ((uint8_t *)&ms_header->bcdMSC)[1], ((uint8_t *)&ms_header->bcdMSC)[0]);
        else
                dev_warn(&umidi->dev->dev,
                         "MIDIStreaming interface descriptor not found\n");

        epidx = 0;
        for (i = 0; i < intfd->bNumEndpoints; ++i) {
                hostep = &hostif->endpoint[i];
                ep = get_ep_desc(hostep);
                if (!usb_endpoint_xfer_bulk(ep) && !usb_endpoint_xfer_int(ep))
                        continue;
                ms_ep = find_usb_ms_endpoint_descriptor(hostep);
                if (!ms_ep)
                        continue;
                if (ms_ep->bLength <= sizeof(*ms_ep))
                        continue;
                if (ms_ep->bNumEmbMIDIJack > 0x10)
                        continue;
                if (ms_ep->bLength < sizeof(*ms_ep) + ms_ep->bNumEmbMIDIJack)
                        continue;
                if (usb_endpoint_dir_out(ep)) {
                        if (endpoints[epidx].out_ep) {
                                if (++epidx >= MIDI_MAX_ENDPOINTS) {
                                        dev_warn(&umidi->dev->dev,
                                                 "too many endpoints\n");
                                        break;
                                }
                        }
                        endpoints[epidx].out_ep = usb_endpoint_num(ep);
                        if (usb_endpoint_xfer_int(ep))
                                endpoints[epidx].out_interval = ep->bInterval;
                        else if (snd_usb_get_speed(umidi->dev) == USB_SPEED_LOW)
                                /*
                                 * Low speed bulk transfers don't exist, so
                                 * force interrupt transfers for devices like
                                 * ESI MIDI Mate that try to use them anyway.
                                 */
                                endpoints[epidx].out_interval = 1;
                        endpoints[epidx].out_cables =
                                (1 << ms_ep->bNumEmbMIDIJack) - 1;
                        for (j = 0; j < ms_ep->bNumEmbMIDIJack; ++j)
                                endpoints[epidx].assoc_out_jacks[j] = ms_ep->baAssocJackID[j];
                        for (; j < ARRAY_SIZE(endpoints[epidx].assoc_out_jacks); ++j)
                                endpoints[epidx].assoc_out_jacks[j] = -1;
                        dev_dbg(&umidi->dev->dev, "EP %02X: %d jack(s)\n",
                                ep->bEndpointAddress, ms_ep->bNumEmbMIDIJack);
                } else {
                        if (endpoints[epidx].in_ep) {
                                if (++epidx >= MIDI_MAX_ENDPOINTS) {
                                        dev_warn(&umidi->dev->dev,
                                                 "too many endpoints\n");
                                        break;
                                }
                        }
                        endpoints[epidx].in_ep = usb_endpoint_num(ep);
                        if (usb_endpoint_xfer_int(ep))
                                endpoints[epidx].in_interval = ep->bInterval;
                        else if (snd_usb_get_speed(umidi->dev) == USB_SPEED_LOW)
                                endpoints[epidx].in_interval = 1;
                        endpoints[epidx].in_cables =
                                (1 << ms_ep->bNumEmbMIDIJack) - 1;
                        for (j = 0; j < ms_ep->bNumEmbMIDIJack; ++j)
                                endpoints[epidx].assoc_in_jacks[j] = ms_ep->baAssocJackID[j];
                        for (; j < ARRAY_SIZE(endpoints[epidx].assoc_in_jacks); ++j)
                                endpoints[epidx].assoc_in_jacks[j] = -1;
                        dev_dbg(&umidi->dev->dev, "EP %02X: %d jack(s)\n",
                                ep->bEndpointAddress, ms_ep->bNumEmbMIDIJack);
                }
        }
        return 0;
}

static int roland_load_info(struct snd_kcontrol *kcontrol,
                            struct snd_ctl_elem_info *info)
{
        static const char *const names[] = { "High Load", "Light Load" };

        return snd_ctl_enum_info(info, 1, 2, names);
}

static int roland_load_get(struct snd_kcontrol *kcontrol,
                           struct snd_ctl_elem_value *value)
{
        value->value.enumerated.item[0] = kcontrol->private_value;
        return 0;
}

static int roland_load_put(struct snd_kcontrol *kcontrol,
                           struct snd_ctl_elem_value *value)
{
        struct snd_usb_midi *umidi = kcontrol->private_data;
        int changed;

        if (value->value.enumerated.item[0] > 1)
                return -EINVAL;
        mutex_lock(&umidi->mutex);
        changed = value->value.enumerated.item[0] != kcontrol->private_value;
        if (changed)
                kcontrol->private_value = value->value.enumerated.item[0];
        mutex_unlock(&umidi->mutex);
        return changed;
}

static const struct snd_kcontrol_new roland_load_ctl = {
        .iface = SNDRV_CTL_ELEM_IFACE_MIXER,
        .name = "MIDI Input Mode",
        .info = roland_load_info,
        .get = roland_load_get,
        .put = roland_load_put,
        .private_value = 1,
};

/*
 * On Roland devices, use the second alternate setting to be able to use
 * the interrupt input endpoint.
 */
static void snd_usbmidi_switch_roland_altsetting(struct snd_usb_midi *umidi)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;

        intf = umidi->iface;
        if (!intf || intf->num_altsetting != 2)
                return;

        hostif = &intf->altsetting[1];
        intfd = get_iface_desc(hostif);
       /* If either or both of the endpoints support interrupt transfer,
        * then use the alternate setting
        */
        if (intfd->bNumEndpoints != 2 ||
            !((get_endpoint(hostif, 0)->bmAttributes &
               USB_ENDPOINT_XFERTYPE_MASK) == USB_ENDPOINT_XFER_INT ||
              (get_endpoint(hostif, 1)->bmAttributes &
               USB_ENDPOINT_XFERTYPE_MASK) == USB_ENDPOINT_XFER_INT))
                return;

        dev_dbg(&umidi->dev->dev, "switching to altsetting %d with int ep\n",
                    intfd->bAlternateSetting);
        usb_set_interface(umidi->dev, intfd->bInterfaceNumber,
                          intfd->bAlternateSetting);

        umidi->roland_load_ctl = snd_ctl_new1(&roland_load_ctl, umidi);
        if (snd_ctl_add(umidi->card, umidi->roland_load_ctl) < 0)
                umidi->roland_load_ctl = NULL;
}

/*
 * Try to find any usable endpoints in the interface.
 */
static int snd_usbmidi_detect_endpoints(struct snd_usb_midi *umidi,
                                        struct snd_usb_midi_endpoint_info *endpoint,
                                        int max_endpoints)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;
        struct usb_endpoint_descriptor *epd;
        int i, out_eps = 0, in_eps = 0;

        if (USB_ID_VENDOR(umidi->usb_id) == 0x0582)
                snd_usbmidi_switch_roland_altsetting(umidi);

        if (endpoint[0].out_ep || endpoint[0].in_ep)
                return 0;

        intf = umidi->iface;
        if (!intf || intf->num_altsetting < 1)
                return -ENOENT;
        hostif = intf->cur_altsetting;
        intfd = get_iface_desc(hostif);

        for (i = 0; i < intfd->bNumEndpoints; ++i) {
                epd = get_endpoint(hostif, i);
                if (!usb_endpoint_xfer_bulk(epd) &&
                    !usb_endpoint_xfer_int(epd))
                        continue;
                if (out_eps < max_endpoints &&
                    usb_endpoint_dir_out(epd)) {
                        endpoint[out_eps].out_ep = usb_endpoint_num(epd);
                        if (usb_endpoint_xfer_int(epd))
                                endpoint[out_eps].out_interval = epd->bInterval;
                        ++out_eps;
                }
                if (in_eps < max_endpoints &&
                    usb_endpoint_dir_in(epd)) {
                        endpoint[in_eps].in_ep = usb_endpoint_num(epd);
                        if (usb_endpoint_xfer_int(epd))
                                endpoint[in_eps].in_interval = epd->bInterval;
                        ++in_eps;
                }
        }
        return (out_eps || in_eps) ? 0 : -ENOENT;
}

/*
 * Detects the endpoints for one-port-per-endpoint protocols.
 */
static int snd_usbmidi_detect_per_port_endpoints(struct snd_usb_midi *umidi,
                                                 struct snd_usb_midi_endpoint_info *endpoints)
{
        int err, i;

        err = snd_usbmidi_detect_endpoints(umidi, endpoints, MIDI_MAX_ENDPOINTS);
        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                if (endpoints[i].out_ep)
                        endpoints[i].out_cables = 0x0001;
                if (endpoints[i].in_ep)
                        endpoints[i].in_cables = 0x0001;
        }
        return err;
}

/*
 * Detects the endpoints and ports of Yamaha devices.
 */
static int snd_usbmidi_detect_yamaha(struct snd_usb_midi *umidi,
                                     struct snd_usb_midi_endpoint_info *endpoint)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;
        uint8_t *cs_desc;

        intf = umidi->iface;
        if (!intf)
                return -ENOENT;
        hostif = intf->altsetting;
        intfd = get_iface_desc(hostif);
        if (intfd->bNumEndpoints < 1)
                return -ENOENT;

        /*
         * For each port there is one MIDI_IN/OUT_JACK descriptor, not
         * necessarily with any useful contents.  So simply count 'em.
         */
        for (cs_desc = hostif->extra;
             cs_desc < hostif->extra + hostif->extralen && cs_desc[0] >= 2;
             cs_desc += cs_desc[0]) {
                if (cs_desc[1] == USB_DT_CS_INTERFACE) {
                        if (cs_desc[2] == UAC_MIDI_IN_JACK)
                                endpoint->in_cables =
                                        (endpoint->in_cables << 1) | 1;
                        else if (cs_desc[2] == UAC_MIDI_OUT_JACK)
                                endpoint->out_cables =
                                        (endpoint->out_cables << 1) | 1;
                }
        }
        if (!endpoint->in_cables && !endpoint->out_cables)
                return -ENOENT;

        return snd_usbmidi_detect_endpoints(umidi, endpoint, 1);
}

/*
 * Detects the endpoints and ports of Roland devices.
 */
static int snd_usbmidi_detect_roland(struct snd_usb_midi *umidi,
                                     struct snd_usb_midi_endpoint_info *endpoint)
{
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        u8 *cs_desc;

        intf = umidi->iface;
        if (!intf)
                return -ENOENT;
        hostif = intf->altsetting;
        /*
         * Some devices have a descriptor <06 24 F1 02 <inputs> <outputs>>,
         * some have standard class descriptors, or both kinds, or neither.
         */
        for (cs_desc = hostif->extra;
             cs_desc < hostif->extra + hostif->extralen && cs_desc[0] >= 2;
             cs_desc += cs_desc[0]) {
                if (cs_desc[0] >= 6 &&
                    cs_desc[1] == USB_DT_CS_INTERFACE &&
                    cs_desc[2] == 0xf1 &&
                    cs_desc[3] == 0x02) {
                        if (cs_desc[4] > 0x10 || cs_desc[5] > 0x10)
                                continue;
                        endpoint->in_cables  = (1 << cs_desc[4]) - 1;
                        endpoint->out_cables = (1 << cs_desc[5]) - 1;
                        return snd_usbmidi_detect_endpoints(umidi, endpoint, 1);
                } else if (cs_desc[0] >= 7 &&
                           cs_desc[1] == USB_DT_CS_INTERFACE &&
                           cs_desc[2] == UAC_HEADER) {
                        return snd_usbmidi_get_ms_info(umidi, endpoint);
                }
        }

        return -ENODEV;
}

/*
 * Creates the endpoints and their ports for Midiman devices.
 */
static int snd_usbmidi_create_endpoints_midiman(struct snd_usb_midi *umidi,
                                                struct snd_usb_midi_endpoint_info *endpoint)
{
        struct snd_usb_midi_endpoint_info ep_info;
        struct usb_interface *intf;
        struct usb_host_interface *hostif;
        struct usb_interface_descriptor *intfd;
        struct usb_endpoint_descriptor *epd;
        int cable, err;

        intf = umidi->iface;
        if (!intf)
                return -ENOENT;
        hostif = intf->altsetting;
        intfd = get_iface_desc(hostif);
        /*
         * The various MidiSport devices have more or less random endpoint
         * numbers, so we have to identify the endpoints by their index in
         * the descriptor array, like the driver for that other OS does.
         *
         * There is one interrupt input endpoint for all input ports, one
         * bulk output endpoint for even-numbered ports, and one for odd-
         * numbered ports.  Both bulk output endpoints have corresponding
         * input bulk endpoints (at indices 1 and 3) which aren't used.
         */
        if (intfd->bNumEndpoints < (endpoint->out_cables > 0x0001 ? 5 : 3)) {
                dev_dbg(&umidi->dev->dev, "not enough endpoints\n");
                return -ENOENT;
        }

        epd = get_endpoint(hostif, 0);
        if (!usb_endpoint_dir_in(epd) || !usb_endpoint_xfer_int(epd)) {
                dev_dbg(&umidi->dev->dev, "endpoint[0] isn't interrupt\n");
                return -ENXIO;
        }
        epd = get_endpoint(hostif, 2);
        if (!usb_endpoint_dir_out(epd) || !usb_endpoint_xfer_bulk(epd)) {
                dev_dbg(&umidi->dev->dev, "endpoint[2] isn't bulk output\n");
                return -ENXIO;
        }
        if (endpoint->out_cables > 0x0001) {
                epd = get_endpoint(hostif, 4);
                if (!usb_endpoint_dir_out(epd) ||
                    !usb_endpoint_xfer_bulk(epd)) {
                        dev_dbg(&umidi->dev->dev,
                                "endpoint[4] isn't bulk output\n");
                        return -ENXIO;
                }
        }

        ep_info.out_ep = get_endpoint(hostif, 2)->bEndpointAddress &
                USB_ENDPOINT_NUMBER_MASK;
        ep_info.out_interval = 0;
        ep_info.out_cables = endpoint->out_cables & 0x5555;
        err = snd_usbmidi_out_endpoint_create(umidi, &ep_info,
                                              &umidi->endpoints[0]);
        if (err < 0)
                return err;

        ep_info.in_ep = get_endpoint(hostif, 0)->bEndpointAddress &
                USB_ENDPOINT_NUMBER_MASK;
        ep_info.in_interval = get_endpoint(hostif, 0)->bInterval;
        ep_info.in_cables = endpoint->in_cables;
        err = snd_usbmidi_in_endpoint_create(umidi, &ep_info,
                                             &umidi->endpoints[0]);
        if (err < 0)
                return err;

        if (endpoint->out_cables > 0x0001) {
                ep_info.out_ep = get_endpoint(hostif, 4)->bEndpointAddress &
                        USB_ENDPOINT_NUMBER_MASK;
                ep_info.out_cables = endpoint->out_cables & 0xaaaa;
                err = snd_usbmidi_out_endpoint_create(umidi, &ep_info,
                                                      &umidi->endpoints[1]);
                if (err < 0)
                        return err;
        }

        for (cable = 0; cable < 0x10; ++cable) {
                if (endpoint->out_cables & (1 << cable))
                        snd_usbmidi_init_substream(umidi,
                                                   SNDRV_RAWMIDI_STREAM_OUTPUT,
                                                   cable,
                                                   -1 /* prevent trying to find jack */,
                                                   &umidi->endpoints[cable & 1].out->ports[cable].substream);
                if (endpoint->in_cables & (1 << cable))
                        snd_usbmidi_init_substream(umidi,
                                                   SNDRV_RAWMIDI_STREAM_INPUT,
                                                   cable,
                                                   -1 /* prevent trying to find jack */,
                                                   &umidi->endpoints[0].in->ports[cable].substream);
        }
        return 0;
}

static const struct snd_rawmidi_global_ops snd_usbmidi_ops = {
        .get_port_info = snd_usbmidi_get_port_info,
};

static int snd_usbmidi_create_rawmidi(struct snd_usb_midi *umidi,
                                      int out_ports, int in_ports)
{
        struct snd_rawmidi *rmidi;
        int err;

        err = snd_rawmidi_new(umidi->card, "USB MIDI",
                              umidi->next_midi_device++,
                              out_ports, in_ports, &rmidi);
        if (err < 0)
                return err;
        strcpy(rmidi->name, umidi->card->shortname);
        rmidi->info_flags = SNDRV_RAWMIDI_INFO_OUTPUT |
                            SNDRV_RAWMIDI_INFO_INPUT |
                            SNDRV_RAWMIDI_INFO_DUPLEX;
        rmidi->ops = &snd_usbmidi_ops;
        rmidi->private_data = umidi;
        rmidi->private_free = snd_usbmidi_rawmidi_free;
        snd_rawmidi_set_ops(rmidi, SNDRV_RAWMIDI_STREAM_OUTPUT,
                            &snd_usbmidi_output_ops);
        snd_rawmidi_set_ops(rmidi, SNDRV_RAWMIDI_STREAM_INPUT,
                            &snd_usbmidi_input_ops);

        umidi->rmidi = rmidi;
        return 0;
}

/*
 * Temporarily stop input.
 */
void snd_usbmidi_input_stop(struct list_head *p)
{
        struct snd_usb_midi *umidi;
        unsigned int i, j;

        umidi = list_entry(p, struct snd_usb_midi, list);
        if (!umidi->input_running)
                return;
        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                struct snd_usb_midi_endpoint *ep = &umidi->endpoints[i];
                if (ep->in)
                        for (j = 0; j < INPUT_URBS; ++j)
                                usb_kill_urb(ep->in->urbs[j]);
        }
        umidi->input_running = 0;
}
EXPORT_SYMBOL(snd_usbmidi_input_stop);

static void snd_usbmidi_input_start_ep(struct snd_usb_midi *umidi,
                                       struct snd_usb_midi_in_endpoint *ep)
{
        unsigned int i;
        unsigned long flags;

        if (!ep)
                return;
        for (i = 0; i < INPUT_URBS; ++i) {
                struct urb *urb = ep->urbs[i];
                spin_lock_irqsave(&umidi->disc_lock, flags);
                if (!atomic_read(&urb->use_count)) {
                        urb->dev = ep->umidi->dev;
                        snd_usbmidi_submit_urb(urb, GFP_ATOMIC);
                }
                spin_unlock_irqrestore(&umidi->disc_lock, flags);
        }
}

/*
 * Resume input after a call to snd_usbmidi_input_stop().
 */
void snd_usbmidi_input_start(struct list_head *p)
{
        struct snd_usb_midi *umidi;
        int i;

        umidi = list_entry(p, struct snd_usb_midi, list);
        if (umidi->input_running || !umidi->opened[1])
                return;
        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i)
                snd_usbmidi_input_start_ep(umidi, umidi->endpoints[i].in);
        umidi->input_running = 1;
}
EXPORT_SYMBOL(snd_usbmidi_input_start);

/*
 * Prepare for suspend. Typically called from the USB suspend callback.
 */
void snd_usbmidi_suspend(struct list_head *p)
{
        struct snd_usb_midi *umidi;

        umidi = list_entry(p, struct snd_usb_midi, list);
        mutex_lock(&umidi->mutex);
        snd_usbmidi_input_stop(p);
        mutex_unlock(&umidi->mutex);
}
EXPORT_SYMBOL(snd_usbmidi_suspend);

/*
 * Resume. Typically called from the USB resume callback.
 */
void snd_usbmidi_resume(struct list_head *p)
{
        struct snd_usb_midi *umidi;

        umidi = list_entry(p, struct snd_usb_midi, list);
        mutex_lock(&umidi->mutex);
        snd_usbmidi_input_start(p);
        mutex_unlock(&umidi->mutex);
}
EXPORT_SYMBOL(snd_usbmidi_resume);

/*
 * Creates and registers everything needed for a MIDI streaming interface.
 */
int __snd_usbmidi_create(struct snd_card *card,
                         struct usb_interface *iface,
                         struct list_head *midi_list,
                         const struct snd_usb_audio_quirk *quirk,
                         unsigned int usb_id,
                         unsigned int *num_rawmidis)
{
        struct snd_usb_midi *umidi;
        struct snd_usb_midi_endpoint_info endpoints[MIDI_MAX_ENDPOINTS];
        int out_ports, in_ports;
        int i, err;

        umidi = kzalloc(sizeof(*umidi), GFP_KERNEL);
        if (!umidi)
                return -ENOMEM;
        umidi->dev = interface_to_usbdev(iface);
        umidi->card = card;
        umidi->iface = iface;
        umidi->quirk = quirk;
        umidi->usb_protocol_ops = &snd_usbmidi_standard_ops;
        if (num_rawmidis)
                umidi->next_midi_device = *num_rawmidis;
        spin_lock_init(&umidi->disc_lock);
        init_rwsem(&umidi->disc_rwsem);
        mutex_init(&umidi->mutex);
        if (!usb_id)
                usb_id = USB_ID(le16_to_cpu(umidi->dev->descriptor.idVendor),
                               le16_to_cpu(umidi->dev->descriptor.idProduct));
        umidi->usb_id = usb_id;
        timer_setup(&umidi->error_timer, snd_usbmidi_error_timer, 0);

        /* detect the endpoint(s) to use */
        memset(endpoints, 0, sizeof(endpoints));
        switch (quirk ? quirk->type : QUIRK_MIDI_STANDARD_INTERFACE) {
        case QUIRK_MIDI_STANDARD_INTERFACE:
                err = snd_usbmidi_get_ms_info(umidi, endpoints);
                if (umidi->usb_id == USB_ID(0x0763, 0x0150)) /* M-Audio Uno */
                        umidi->usb_protocol_ops =
                                &snd_usbmidi_maudio_broken_running_status_ops;
                break;
        case QUIRK_MIDI_US122L:
                umidi->usb_protocol_ops = &snd_usbmidi_122l_ops;
                fallthrough;
        case QUIRK_MIDI_FIXED_ENDPOINT:
                memcpy(&endpoints[0], quirk->data,
                       sizeof(struct snd_usb_midi_endpoint_info));
                err = snd_usbmidi_detect_endpoints(umidi, &endpoints[0], 1);
                break;
        case QUIRK_MIDI_YAMAHA:
                err = snd_usbmidi_detect_yamaha(umidi, &endpoints[0]);
                break;
        case QUIRK_MIDI_ROLAND:
                err = snd_usbmidi_detect_roland(umidi, &endpoints[0]);
                break;
        case QUIRK_MIDI_MIDIMAN:
                umidi->usb_protocol_ops = &snd_usbmidi_midiman_ops;
                memcpy(&endpoints[0], quirk->data,
                       sizeof(struct snd_usb_midi_endpoint_info));
                err = 0;
                break;
        case QUIRK_MIDI_NOVATION:
                umidi->usb_protocol_ops = &snd_usbmidi_novation_ops;
                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                break;
        case QUIRK_MIDI_RAW_BYTES:
                umidi->usb_protocol_ops = &snd_usbmidi_raw_ops;
                /*
                 * Interface 1 contains isochronous endpoints, but with the same
                 * numbers as in interface 0.  Since it is interface 1 that the
                 * USB core has most recently seen, these descriptors are now
                 * associated with the endpoint numbers.  This will foul up our
                 * attempts to submit bulk/interrupt URBs to the endpoints in
                 * interface 0, so we have to make sure that the USB core looks
                 * again at interface 0 by calling usb_set_interface() on it.
                 */
                if (umidi->usb_id == USB_ID(0x07fd, 0x0001)) /* MOTU Fastlane */
                        usb_set_interface(umidi->dev, 0, 0);
                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                break;
        case QUIRK_MIDI_EMAGIC:
                umidi->usb_protocol_ops = &snd_usbmidi_emagic_ops;
                memcpy(&endpoints[0], quirk->data,
                       sizeof(struct snd_usb_midi_endpoint_info));
                err = snd_usbmidi_detect_endpoints(umidi, &endpoints[0], 1);
                break;
        case QUIRK_MIDI_CME:
                umidi->usb_protocol_ops = &snd_usbmidi_cme_ops;
                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                break;
        case QUIRK_MIDI_AKAI:
                umidi->usb_protocol_ops = &snd_usbmidi_akai_ops;
                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                /* endpoint 1 is input-only */
                endpoints[1].out_cables = 0;
                break;
        case QUIRK_MIDI_FTDI:
                umidi->usb_protocol_ops = &snd_usbmidi_ftdi_ops;

                /* set baud rate to 31250 (48 MHz / 16 / 96) */
                err = usb_control_msg(umidi->dev, usb_sndctrlpipe(umidi->dev, 0),
                                      3, 0x40, 0x60, 0, NULL, 0, 1000);
                if (err < 0)
                        break;

                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                break;
        case QUIRK_MIDI_CH345:
                umidi->usb_protocol_ops = &snd_usbmidi_ch345_broken_sysex_ops;
                err = snd_usbmidi_detect_per_port_endpoints(umidi, endpoints);
                break;
        default:
                dev_err(&umidi->dev->dev, "invalid quirk type %d\n",
                        quirk->type);
                err = -ENXIO;
                break;
        }
        if (err < 0)
                goto free_midi;

        /* create rawmidi device */
        out_ports = 0;
        in_ports = 0;
        for (i = 0; i < MIDI_MAX_ENDPOINTS; ++i) {
                out_ports += hweight16(endpoints[i].out_cables);
                in_ports += hweight16(endpoints[i].in_cables);
        }
        err = snd_usbmidi_create_rawmidi(umidi, out_ports, in_ports);
        if (err < 0)
                goto free_midi;

        /* create endpoint/port structures */
        if (quirk && quirk->type == QUIRK_MIDI_MIDIMAN)
                err = snd_usbmidi_create_endpoints_midiman(umidi, &endpoints[0]);
        else
                err = snd_usbmidi_create_endpoints(umidi, endpoints);
        if (err < 0)
                goto exit;

        usb_autopm_get_interface_no_resume(umidi->iface);

        list_add_tail(&umidi->list, midi_list);
        if (num_rawmidis)
                *num_rawmidis = umidi->next_midi_device;
        return 0;

free_midi:
        kfree(umidi);
exit:
        return err;
}
EXPORT_SYMBOL(__snd_usbmidi_create);
















































   11 




    2 



















   11 




    2 




























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
/* SPDX-License-Identifier: GPL-2.0 */
/* Based on net/mac80211/trace.h */

#undef TRACE_SYSTEM
#define TRACE_SYSTEM mac802154

#if !defined(__MAC802154_DRIVER_TRACE) || defined(TRACE_HEADER_MULTI_READ)
#define __MAC802154_DRIVER_TRACE

#include <linux/tracepoint.h>

#include <net/mac802154.h>
#include "ieee802154_i.h"

#define MAXNAME                32
#define LOCAL_ENTRY        __array(char, wpan_phy_name, MAXNAME)
#define LOCAL_ASSIGN        strscpy(__entry->wpan_phy_name, \
                                wpan_phy_name(local->hw.phy), MAXNAME)
#define LOCAL_PR_FMT        "%s"
#define LOCAL_PR_ARG        __entry->wpan_phy_name

#define CCA_ENTRY __field(enum nl802154_cca_modes, cca_mode) \
                  __field(enum nl802154_cca_opts, cca_opt)
#define CCA_ASSIGN \
        do {                                     \
                (__entry->cca_mode) = cca->mode; \
                (__entry->cca_opt) = cca->opt;   \
        } while (0)
#define CCA_PR_FMT "cca_mode: %d, cca_opt: %d"
#define CCA_PR_ARG __entry->cca_mode, __entry->cca_opt

#define BOOL_TO_STR(bo) (bo) ? "true" : "false"

/* Tracing for driver callbacks */

DECLARE_EVENT_CLASS(local_only_evt4,
        TP_PROTO(struct ieee802154_local *local),
        TP_ARGS(local),
        TP_STRUCT__entry(
                LOCAL_ENTRY
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
        ),
        TP_printk(LOCAL_PR_FMT, LOCAL_PR_ARG)
);

DEFINE_EVENT(local_only_evt4, 802154_drv_return_void,
        TP_PROTO(struct ieee802154_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(802154_drv_return_int,
        TP_PROTO(struct ieee802154_local *local, int ret),
        TP_ARGS(local, ret),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(int, ret)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->ret = ret;
        ),
        TP_printk(LOCAL_PR_FMT ", returned: %d", LOCAL_PR_ARG,
                  __entry->ret)
);

DEFINE_EVENT(local_only_evt4, 802154_drv_start,
        TP_PROTO(struct ieee802154_local *local),
        TP_ARGS(local)
);

DEFINE_EVENT(local_only_evt4, 802154_drv_stop,
        TP_PROTO(struct ieee802154_local *local),
        TP_ARGS(local)
);

TRACE_EVENT(802154_drv_set_channel,
        TP_PROTO(struct ieee802154_local *local, u8 page, u8 channel),
        TP_ARGS(local, page, channel),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u8, page)
                __field(u8, channel)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->page = page;
                __entry->channel = channel;
        ),
        TP_printk(LOCAL_PR_FMT ", page: %d, channel: %d", LOCAL_PR_ARG,
                  __entry->page, __entry->channel)
);

TRACE_EVENT(802154_drv_set_cca_mode,
        TP_PROTO(struct ieee802154_local *local,
                 const struct wpan_phy_cca *cca),
        TP_ARGS(local, cca),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                CCA_ENTRY
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                CCA_ASSIGN;
        ),
        TP_printk(LOCAL_PR_FMT ", " CCA_PR_FMT, LOCAL_PR_ARG,
                  CCA_PR_ARG)
);

TRACE_EVENT(802154_drv_set_cca_ed_level,
        TP_PROTO(struct ieee802154_local *local, s32 mbm),
        TP_ARGS(local, mbm),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(s32, mbm)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->mbm = mbm;
        ),
        TP_printk(LOCAL_PR_FMT ", ed level: %d", LOCAL_PR_ARG,
                  __entry->mbm)
);

TRACE_EVENT(802154_drv_set_tx_power,
        TP_PROTO(struct ieee802154_local *local, s32 power),
        TP_ARGS(local, power),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(s32, power)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->power = power;
        ),
        TP_printk(LOCAL_PR_FMT ", mbm: %d", LOCAL_PR_ARG,
                 __entry->power)
);

TRACE_EVENT(802154_drv_set_lbt_mode,
        TP_PROTO(struct ieee802154_local *local, bool mode),
        TP_ARGS(local, mode),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, mode)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->mode = mode;
        ),
        TP_printk(LOCAL_PR_FMT ", lbt mode: %s", LOCAL_PR_ARG,
                  BOOL_TO_STR(__entry->mode))
);

TRACE_EVENT(802154_drv_set_short_addr,
        TP_PROTO(struct ieee802154_local *local, __le16 short_addr),
        TP_ARGS(local, short_addr),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(__le16, short_addr)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->short_addr = short_addr;
        ),
        TP_printk(LOCAL_PR_FMT ", short addr: 0x%04x", LOCAL_PR_ARG,
                  le16_to_cpu(__entry->short_addr))
);

TRACE_EVENT(802154_drv_set_pan_id,
        TP_PROTO(struct ieee802154_local *local, __le16 pan_id),
        TP_ARGS(local, pan_id),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(__le16, pan_id)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->pan_id = pan_id;
        ),
        TP_printk(LOCAL_PR_FMT ", pan id: 0x%04x", LOCAL_PR_ARG,
                  le16_to_cpu(__entry->pan_id))
);

TRACE_EVENT(802154_drv_set_extended_addr,
        TP_PROTO(struct ieee802154_local *local, __le64 extended_addr),
        TP_ARGS(local, extended_addr),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(__le64, extended_addr)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->extended_addr = extended_addr;
        ),
        TP_printk(LOCAL_PR_FMT ", extended addr: 0x%llx", LOCAL_PR_ARG,
                  le64_to_cpu(__entry->extended_addr))
);

TRACE_EVENT(802154_drv_set_pan_coord,
        TP_PROTO(struct ieee802154_local *local, bool is_coord),
        TP_ARGS(local, is_coord),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, is_coord)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->is_coord = is_coord;
        ),
        TP_printk(LOCAL_PR_FMT ", is_coord: %s", LOCAL_PR_ARG,
                  BOOL_TO_STR(__entry->is_coord))
);

TRACE_EVENT(802154_drv_set_csma_params,
        TP_PROTO(struct ieee802154_local *local, u8 min_be, u8 max_be,
                 u8 max_csma_backoffs),
        TP_ARGS(local, min_be, max_be, max_csma_backoffs),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(u8, min_be)
                __field(u8, max_be)
                __field(u8, max_csma_backoffs)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN,
                __entry->min_be = min_be;
                __entry->max_be = max_be;
                __entry->max_csma_backoffs = max_csma_backoffs;
        ),
        TP_printk(LOCAL_PR_FMT ", min be: %d, max be: %d, max csma backoffs: %d",
                  LOCAL_PR_ARG, __entry->min_be, __entry->max_be,
                  __entry->max_csma_backoffs)
);

TRACE_EVENT(802154_drv_set_max_frame_retries,
        TP_PROTO(struct ieee802154_local *local, s8 max_frame_retries),
        TP_ARGS(local, max_frame_retries),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(s8, max_frame_retries)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->max_frame_retries = max_frame_retries;
        ),
        TP_printk(LOCAL_PR_FMT ", max frame retries: %d", LOCAL_PR_ARG,
                  __entry->max_frame_retries)
);

TRACE_EVENT(802154_drv_set_promiscuous_mode,
        TP_PROTO(struct ieee802154_local *local, bool on),
        TP_ARGS(local, on),
        TP_STRUCT__entry(
                LOCAL_ENTRY
                __field(bool, on)
        ),
        TP_fast_assign(
                LOCAL_ASSIGN;
                __entry->on = on;
        ),
        TP_printk(LOCAL_PR_FMT ", promiscuous mode: %s", LOCAL_PR_ARG,
                  BOOL_TO_STR(__entry->on))
);

TRACE_EVENT(802154_new_scan_event,
        TP_PROTO(struct ieee802154_coord_desc *desc),
        TP_ARGS(desc),
        TP_STRUCT__entry(
                __field(__le16, pan_id)
                __field(__le64, addr)
                __field(u8, channel)
                __field(u8, page)
        ),
        TP_fast_assign(
                __entry->page = desc->page;
                __entry->channel = desc->channel;
                __entry->pan_id = desc->addr.pan_id;
                __entry->addr = desc->addr.extended_addr;
        ),
        TP_printk("panid: %u, coord_addr: 0x%llx, page: %u, channel: %u",
                  __le16_to_cpu(__entry->pan_id), __le64_to_cpu(__entry->addr),
                  __entry->page, __entry->channel)
);

DEFINE_EVENT(802154_new_scan_event, 802154_scan_event,
        TP_PROTO(struct ieee802154_coord_desc *desc),
        TP_ARGS(desc)
);

#endif /* !__MAC802154_DRIVER_TRACE || TRACE_HEADER_MULTI_READ */

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace
#include <trace/define_trace.h>




























  373 







































  367 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM snd_pcm

#if !defined(_PCM_PARAMS_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _PCM_PARAMS_TRACE_H

#include <linux/tracepoint.h>

#define HW_PARAM_ENTRY(param) {SNDRV_PCM_HW_PARAM_##param, #param}
#define hw_param_labels                        \
        HW_PARAM_ENTRY(ACCESS),                \
        HW_PARAM_ENTRY(FORMAT),                \
        HW_PARAM_ENTRY(SUBFORMAT),        \
        HW_PARAM_ENTRY(SAMPLE_BITS),        \
        HW_PARAM_ENTRY(FRAME_BITS),        \
        HW_PARAM_ENTRY(CHANNELS),        \
        HW_PARAM_ENTRY(RATE),                \
        HW_PARAM_ENTRY(PERIOD_TIME),        \
        HW_PARAM_ENTRY(PERIOD_SIZE),        \
        HW_PARAM_ENTRY(PERIOD_BYTES),        \
        HW_PARAM_ENTRY(PERIODS),        \
        HW_PARAM_ENTRY(BUFFER_TIME),        \
        HW_PARAM_ENTRY(BUFFER_SIZE),        \
        HW_PARAM_ENTRY(BUFFER_BYTES),        \
        HW_PARAM_ENTRY(TICK_TIME)

TRACE_EVENT(hw_mask_param,
        TP_PROTO(struct snd_pcm_substream *substream, snd_pcm_hw_param_t type, int index, const struct snd_mask *prev, const struct snd_mask *curr),
        TP_ARGS(substream, type, index, prev, curr),
        TP_STRUCT__entry(
                __field(int, card)
                __field(int, device)
                __field(int, subdevice)
                __field(int, direction)
                __field(snd_pcm_hw_param_t, type)
                __field(int, index)
                __field(int, total)
                __array(__u32, prev_bits, 8)
                __array(__u32, curr_bits, 8)
        ),
        TP_fast_assign(
                __entry->card = substream->pcm->card->number;
                __entry->device = substream->pcm->device;
                __entry->subdevice = substream->number;
                __entry->direction = substream->stream;
                __entry->type = type;
                __entry->index = index;
                __entry->total = substream->runtime->hw_constraints.rules_num;
                memcpy(__entry->prev_bits, prev->bits, sizeof(__u32) * 8);
                memcpy(__entry->curr_bits, curr->bits, sizeof(__u32) * 8);
        ),
        TP_printk("pcmC%dD%d%s:%d %03d/%03d %s %08x%08x%08x%08x %08x%08x%08x%08x",
                  __entry->card,
                  __entry->device,
                  __entry->direction ? "c" : "p",
                  __entry->subdevice,
                  __entry->index,
                  __entry->total,
                  __print_symbolic(__entry->type, hw_param_labels),
                  __entry->prev_bits[3], __entry->prev_bits[2],
                  __entry->prev_bits[1], __entry->prev_bits[0],
                  __entry->curr_bits[3], __entry->curr_bits[2],
                  __entry->curr_bits[1], __entry->curr_bits[0]
        )
);

TRACE_EVENT(hw_interval_param,
        TP_PROTO(struct snd_pcm_substream *substream, snd_pcm_hw_param_t type, int index, const struct snd_interval *prev, const struct snd_interval *curr),
        TP_ARGS(substream, type, index, prev, curr),
        TP_STRUCT__entry(
                __field(int, card)
                __field(int, device)
                __field(int, subdevice)
                __field(int, direction)
                __field(snd_pcm_hw_param_t, type)
                __field(int, index)
                __field(int, total)
                __field(unsigned int, prev_min)
                __field(unsigned int, prev_max)
                __field(unsigned int, prev_openmin)
                __field(unsigned int, prev_openmax)
                __field(unsigned int, prev_integer)
                __field(unsigned int, prev_empty)
                __field(unsigned int, curr_min)
                __field(unsigned int, curr_max)
                __field(unsigned int, curr_openmin)
                __field(unsigned int, curr_openmax)
                __field(unsigned int, curr_integer)
                __field(unsigned int, curr_empty)
        ),
        TP_fast_assign(
                __entry->card = substream->pcm->card->number;
                __entry->device = substream->pcm->device;
                __entry->subdevice = substream->number;
                __entry->direction = substream->stream;
                __entry->type = type;
                __entry->index = index;
                __entry->total = substream->runtime->hw_constraints.rules_num;
                __entry->prev_min = prev->min;
                __entry->prev_max = prev->max;
                __entry->prev_openmin = prev->openmin;
                __entry->prev_openmax = prev->openmax;
                __entry->prev_integer = prev->integer;
                __entry->prev_empty = prev->empty;
                __entry->curr_min = curr->min;
                __entry->curr_max = curr->max;
                __entry->curr_openmin = curr->openmin;
                __entry->curr_openmax = curr->openmax;
                __entry->curr_integer = curr->integer;
                __entry->curr_empty = curr->empty;
        ),
        TP_printk("pcmC%dD%d%s:%d %03d/%03d %s %d %d %s%u %u%s %d %d %s%u %u%s",
                  __entry->card,
                  __entry->device,
                  __entry->direction ? "c" : "p",
                  __entry->subdevice,
                  __entry->index,
                  __entry->total,
                  __print_symbolic(__entry->type, hw_param_labels),
                  __entry->prev_empty,
                  __entry->prev_integer,
                  __entry->prev_openmin ? "(" : "[",
                  __entry->prev_min,
                  __entry->prev_max,
                  __entry->prev_openmax ? ")" : "]",
                  __entry->curr_empty,
                  __entry->curr_integer,
                  __entry->curr_openmin ? "(" : "[",
                  __entry->curr_min,
                  __entry->curr_max,
                  __entry->curr_openmax ? ")" : "]"
        )
);

#endif /* _PCM_PARAMS_TRACE_H */

/* This part must be outside protection */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE pcm_param_trace
#include <trace/define_trace.h>




























































































































































































































































































































































































































































































































































































































































    2 





    2 





























   12 





   13 





































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 Facebook */

#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <linux/rcupdate_trace.h>

struct bpf_iter_target_info {
        struct list_head list;
        const struct bpf_iter_reg *reg_info;
        u32 btf_id;        /* cached value */
};

struct bpf_iter_link {
        struct bpf_link link;
        struct bpf_iter_aux_info aux;
        struct bpf_iter_target_info *tinfo;
};

struct bpf_iter_priv_data {
        struct bpf_iter_target_info *tinfo;
        const struct bpf_iter_seq_info *seq_info;
        struct bpf_prog *prog;
        u64 session_id;
        u64 seq_num;
        bool done_stop;
        u8 target_private[] __aligned(8);
};

static struct list_head targets = LIST_HEAD_INIT(targets);
static DEFINE_MUTEX(targets_mutex);

/* protect bpf_iter_link changes */
static DEFINE_MUTEX(link_mutex);

/* incremented on every opened seq_file */
static atomic64_t session_id;

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
                            const struct bpf_iter_seq_info *seq_info);

static void bpf_iter_inc_seq_num(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->seq_num++;
}

static void bpf_iter_dec_seq_num(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->seq_num--;
}

static void bpf_iter_done_stop(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->done_stop = true;
}

static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
{
        return tinfo->reg_info->feature & BPF_ITER_RESCHED;
}

static bool bpf_iter_support_resched(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        return bpf_iter_target_support_resched(iter_priv->tinfo);
}

/* maximum visited objects before bailing out */
#define MAX_ITER_OBJECTS        1000000

/* bpf_seq_read, a customized and simpler version for bpf iterator.
 * The following are differences from seq_read():
 *  . fixed buffer size (PAGE_SIZE)
 *  . assuming NULL ->llseek()
 *  . stop() may call bpf program, handling potential overflow there
 */
static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
                            loff_t *ppos)
{
        struct seq_file *seq = file->private_data;
        size_t n, offs, copied = 0;
        int err = 0, num_objs = 0;
        bool can_resched;
        void *p;

        mutex_lock(&seq->lock);

        if (!seq->buf) {
                seq->size = PAGE_SIZE << 3;
                seq->buf = kvmalloc(seq->size, GFP_KERNEL);
                if (!seq->buf) {
                        err = -ENOMEM;
                        goto done;
                }
        }

        if (seq->count) {
                n = min(seq->count, size);
                err = copy_to_user(buf, seq->buf + seq->from, n);
                if (err) {
                        err = -EFAULT;
                        goto done;
                }
                seq->count -= n;
                seq->from += n;
                copied = n;
                goto done;
        }

        seq->from = 0;
        p = seq->op->start(seq, &seq->index);
        if (!p)
                goto stop;
        if (IS_ERR(p)) {
                err = PTR_ERR(p);
                seq->op->stop(seq, p);
                seq->count = 0;
                goto done;
        }

        err = seq->op->show(seq, p);
        if (err > 0) {
                /* object is skipped, decrease seq_num, so next
                 * valid object can reuse the same seq_num.
                 */
                bpf_iter_dec_seq_num(seq);
                seq->count = 0;
        } else if (err < 0 || seq_has_overflowed(seq)) {
                if (!err)
                        err = -E2BIG;
                seq->op->stop(seq, p);
                seq->count = 0;
                goto done;
        }

        can_resched = bpf_iter_support_resched(seq);
        while (1) {
                loff_t pos = seq->index;

                num_objs++;
                offs = seq->count;
                p = seq->op->next(seq, p, &seq->index);
                if (pos == seq->index) {
                        pr_info_ratelimited("buggy seq_file .next function %ps "
                                "did not updated position index\n",
                                seq->op->next);
                        seq->index++;
                }

                if (IS_ERR_OR_NULL(p))
                        break;

                /* got a valid next object, increase seq_num */
                bpf_iter_inc_seq_num(seq);

                if (seq->count >= size)
                        break;

                if (num_objs >= MAX_ITER_OBJECTS) {
                        if (offs == 0) {
                                err = -EAGAIN;
                                seq->op->stop(seq, p);
                                goto done;
                        }
                        break;
                }

                err = seq->op->show(seq, p);
                if (err > 0) {
                        bpf_iter_dec_seq_num(seq);
                        seq->count = offs;
                } else if (err < 0 || seq_has_overflowed(seq)) {
                        seq->count = offs;
                        if (offs == 0) {
                                if (!err)
                                        err = -E2BIG;
                                seq->op->stop(seq, p);
                                goto done;
                        }
                        break;
                }

                if (can_resched)
                        cond_resched();
        }
stop:
        offs = seq->count;
        if (IS_ERR(p)) {
                seq->op->stop(seq, NULL);
                err = PTR_ERR(p);
                goto done;
        }
        /* bpf program called if !p */
        seq->op->stop(seq, p);
        if (!p) {
                if (!seq_has_overflowed(seq)) {
                        bpf_iter_done_stop(seq);
                } else {
                        seq->count = offs;
                        if (offs == 0) {
                                err = -E2BIG;
                                goto done;
                        }
                }
        }

        n = min(seq->count, size);
        err = copy_to_user(buf, seq->buf, n);
        if (err) {
                err = -EFAULT;
                goto done;
        }
        copied = n;
        seq->count -= n;
        seq->from = n;
done:
        if (!copied)
                copied = err;
        else
                *ppos += copied;
        mutex_unlock(&seq->lock);
        return copied;
}

static const struct bpf_iter_seq_info *
__get_seq_info(struct bpf_iter_link *link)
{
        const struct bpf_iter_seq_info *seq_info;

        if (link->aux.map) {
                seq_info = link->aux.map->ops->iter_seq_info;
                if (seq_info)
                        return seq_info;
        }

        return link->tinfo->reg_info->seq_info;
}

static int iter_open(struct inode *inode, struct file *file)
{
        struct bpf_iter_link *link = inode->i_private;

        return prepare_seq_file(file, link, __get_seq_info(link));
}

static int iter_release(struct inode *inode, struct file *file)
{
        struct bpf_iter_priv_data *iter_priv;
        struct seq_file *seq;

        seq = file->private_data;
        if (!seq)
                return 0;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);

        if (iter_priv->seq_info->fini_seq_private)
                iter_priv->seq_info->fini_seq_private(seq->private);

        bpf_prog_put(iter_priv->prog);
        seq->private = iter_priv;

        return seq_release_private(inode, file);
}

const struct file_operations bpf_iter_fops = {
        .open                = iter_open,
        .read                = bpf_seq_read,
        .release        = iter_release,
};

/* The argument reg_info will be cached in bpf_iter_target_info.
 * The common practice is to declare target reg_info as
 * a const static variable and passed as an argument to
 * bpf_iter_reg_target().
 */
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
{
        struct bpf_iter_target_info *tinfo;

        tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
        if (!tinfo)
                return -ENOMEM;

        tinfo->reg_info = reg_info;
        INIT_LIST_HEAD(&tinfo->list);

        mutex_lock(&targets_mutex);
        list_add(&tinfo->list, &targets);
        mutex_unlock(&targets_mutex);

        return 0;
}

void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
{
        struct bpf_iter_target_info *tinfo;
        bool found = false;

        mutex_lock(&targets_mutex);
        list_for_each_entry(tinfo, &targets, list) {
                if (reg_info == tinfo->reg_info) {
                        list_del(&tinfo->list);
                        kfree(tinfo);
                        found = true;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        WARN_ON(found == false);
}

static void cache_btf_id(struct bpf_iter_target_info *tinfo,
                         struct bpf_prog *prog)
{
        tinfo->btf_id = prog->aux->attach_btf_id;
}

bool bpf_iter_prog_supported(struct bpf_prog *prog)
{
        const char *attach_fname = prog->aux->attach_func_name;
        struct bpf_iter_target_info *tinfo = NULL, *iter;
        u32 prog_btf_id = prog->aux->attach_btf_id;
        const char *prefix = BPF_ITER_FUNC_PREFIX;
        int prefix_len = strlen(prefix);

        if (strncmp(attach_fname, prefix, prefix_len))
                return false;

        mutex_lock(&targets_mutex);
        list_for_each_entry(iter, &targets, list) {
                if (iter->btf_id && iter->btf_id == prog_btf_id) {
                        tinfo = iter;
                        break;
                }
                if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) {
                        cache_btf_id(iter, prog);
                        tinfo = iter;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        if (tinfo) {
                prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
                prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
        }

        return tinfo != NULL;
}

const struct bpf_func_proto *
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_iter_target_info *tinfo;
        const struct bpf_func_proto *fn = NULL;

        mutex_lock(&targets_mutex);
        list_for_each_entry(tinfo, &targets, list) {
                if (tinfo->btf_id == prog->aux->attach_btf_id) {
                        const struct bpf_iter_reg *reg_info;

                        reg_info = tinfo->reg_info;
                        if (reg_info->get_func_proto)
                                fn = reg_info->get_func_proto(func_id, prog);
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        return fn;
}

static void bpf_iter_link_release(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);

        if (iter_link->tinfo->reg_info->detach_target)
                iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
}

static void bpf_iter_link_dealloc(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);

        kfree(iter_link);
}

static int bpf_iter_link_replace(struct bpf_link *link,
                                 struct bpf_prog *new_prog,
                                 struct bpf_prog *old_prog)
{
        int ret = 0;

        mutex_lock(&link_mutex);
        if (old_prog && link->prog != old_prog) {
                ret = -EPERM;
                goto out_unlock;
        }

        if (link->prog->type != new_prog->type ||
            link->prog->expected_attach_type != new_prog->expected_attach_type ||
            link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
                ret = -EINVAL;
                goto out_unlock;
        }

        old_prog = xchg(&link->prog, new_prog);
        bpf_prog_put(old_prog);

out_unlock:
        mutex_unlock(&link_mutex);
        return ret;
}

static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
                                      struct seq_file *seq)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);
        bpf_iter_show_fdinfo_t show_fdinfo;

        seq_printf(seq,
                   "target_name:\t%s\n",
                   iter_link->tinfo->reg_info->target);

        show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
        if (show_fdinfo)
                show_fdinfo(&iter_link->aux, seq);
}

static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
                                        struct bpf_link_info *info)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);
        char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
        bpf_iter_fill_link_info_t fill_link_info;
        u32 ulen = info->iter.target_name_len;
        const char *target_name;
        u32 target_len;

        if (!ulen ^ !ubuf)
                return -EINVAL;

        target_name = iter_link->tinfo->reg_info->target;
        target_len =  strlen(target_name);
        info->iter.target_name_len = target_len + 1;

        if (ubuf) {
                if (ulen >= target_len + 1) {
                        if (copy_to_user(ubuf, target_name, target_len + 1))
                                return -EFAULT;
                } else {
                        char zero = '\0';

                        if (copy_to_user(ubuf, target_name, ulen - 1))
                                return -EFAULT;
                        if (put_user(zero, ubuf + ulen - 1))
                                return -EFAULT;
                        return -ENOSPC;
                }
        }

        fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
        if (fill_link_info)
                return fill_link_info(&iter_link->aux, info);

        return 0;
}

static const struct bpf_link_ops bpf_iter_link_lops = {
        .release = bpf_iter_link_release,
        .dealloc = bpf_iter_link_dealloc,
        .update_prog = bpf_iter_link_replace,
        .show_fdinfo = bpf_iter_link_show_fdinfo,
        .fill_link_info = bpf_iter_link_fill_link_info,
};

bool bpf_link_is_iter(struct bpf_link *link)
{
        return link->ops == &bpf_iter_link_lops;
}

int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
                         struct bpf_prog *prog)
{
        struct bpf_iter_target_info *tinfo = NULL, *iter;
        struct bpf_link_primer link_primer;
        union bpf_iter_link_info linfo;
        struct bpf_iter_link *link;
        u32 prog_btf_id, linfo_len;
        bpfptr_t ulinfo;
        int err;

        if (attr->link_create.target_fd || attr->link_create.flags)
                return -EINVAL;

        memset(&linfo, 0, sizeof(union bpf_iter_link_info));

        ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
        linfo_len = attr->link_create.iter_info_len;
        if (bpfptr_is_null(ulinfo) ^ !linfo_len)
                return -EINVAL;

        if (!bpfptr_is_null(ulinfo)) {
                err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
                                               linfo_len);
                if (err)
                        return err;
                linfo_len = min_t(u32, linfo_len, sizeof(linfo));
                if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
                        return -EFAULT;
        }

        prog_btf_id = prog->aux->attach_btf_id;
        mutex_lock(&targets_mutex);
        list_for_each_entry(iter, &targets, list) {
                if (iter->btf_id == prog_btf_id) {
                        tinfo = iter;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);
        if (!tinfo)
                return -ENOENT;

        /* Only allow sleepable program for resched-able iterator */
        if (prog->sleepable && !bpf_iter_target_support_resched(tinfo))
                return -EINVAL;

        link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
        if (!link)
                return -ENOMEM;

        bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
        link->tinfo = tinfo;

        err = bpf_link_prime(&link->link, &link_primer);
        if (err) {
                kfree(link);
                return err;
        }

        if (tinfo->reg_info->attach_target) {
                err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
                if (err) {
                        bpf_link_cleanup(&link_primer);
                        return err;
                }
        }

        return bpf_link_settle(&link_primer);
}

static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
                          struct bpf_iter_target_info *tinfo,
                          const struct bpf_iter_seq_info *seq_info,
                          struct bpf_prog *prog)
{
        priv_data->tinfo = tinfo;
        priv_data->seq_info = seq_info;
        priv_data->prog = prog;
        priv_data->session_id = atomic64_inc_return(&session_id);
        priv_data->seq_num = 0;
        priv_data->done_stop = false;
}

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
                            const struct bpf_iter_seq_info *seq_info)
{
        struct bpf_iter_priv_data *priv_data;
        struct bpf_iter_target_info *tinfo;
        struct bpf_prog *prog;
        u32 total_priv_dsize;
        struct seq_file *seq;
        int err = 0;

        mutex_lock(&link_mutex);
        prog = link->link.prog;
        bpf_prog_inc(prog);
        mutex_unlock(&link_mutex);

        tinfo = link->tinfo;
        total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
                           seq_info->seq_priv_size;
        priv_data = __seq_open_private(file, seq_info->seq_ops,
                                       total_priv_dsize);
        if (!priv_data) {
                err = -ENOMEM;
                goto release_prog;
        }

        if (seq_info->init_seq_private) {
                err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
                if (err)
                        goto release_seq_file;
        }

        init_seq_meta(priv_data, tinfo, seq_info, prog);
        seq = file->private_data;
        seq->private = priv_data->target_private;

        return 0;

release_seq_file:
        seq_release_private(file->f_inode, file);
        file->private_data = NULL;
release_prog:
        bpf_prog_put(prog);
        return err;
}

int bpf_iter_new_fd(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link;
        struct file *file;
        unsigned int flags;
        int err, fd;

        if (link->ops != &bpf_iter_link_lops)
                return -EINVAL;

        flags = O_RDONLY | O_CLOEXEC;
        fd = get_unused_fd_flags(flags);
        if (fd < 0)
                return fd;

        file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
        if (IS_ERR(file)) {
                err = PTR_ERR(file);
                goto free_fd;
        }

        iter_link = container_of(link, struct bpf_iter_link, link);
        err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
        if (err)
                goto free_file;

        fd_install(fd, file);
        return fd;

free_file:
        fput(file);
free_fd:
        put_unused_fd(fd);
        return err;
}

struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
{
        struct bpf_iter_priv_data *iter_priv;
        struct seq_file *seq;
        void *seq_priv;

        seq = meta->seq;
        if (seq->file->f_op != &bpf_iter_fops)
                return NULL;

        seq_priv = seq->private;
        iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
                                 target_private);

        if (in_stop && iter_priv->done_stop)
                return NULL;

        meta->session_id = iter_priv->session_id;
        meta->seq_num = iter_priv->seq_num;

        return iter_priv->prog;
}

int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
{
        struct bpf_run_ctx run_ctx, *old_run_ctx;
        int ret;

        if (prog->sleepable) {
                rcu_read_lock_trace();
                migrate_disable();
                might_fault();
                old_run_ctx = bpf_set_run_ctx(&run_ctx);
                ret = bpf_prog_run(prog, ctx);
                bpf_reset_run_ctx(old_run_ctx);
                migrate_enable();
                rcu_read_unlock_trace();
        } else {
                rcu_read_lock();
                migrate_disable();
                old_run_ctx = bpf_set_run_ctx(&run_ctx);
                ret = bpf_prog_run(prog, ctx);
                bpf_reset_run_ctx(old_run_ctx);
                migrate_enable();
                rcu_read_unlock();
        }

        /* bpf program can only return 0 or 1:
         *  0 : okay
         *  1 : retry the same object
         * The bpf_iter_run_prog() return value
         * will be seq_ops->show() return value.
         */
        return ret == 0 ? 0 : -EAGAIN;
}

BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
           void *, callback_ctx, u64, flags)
{
        return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
}

const struct bpf_func_proto bpf_for_each_map_elem_proto = {
        .func                = bpf_for_each_map_elem,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_FUNC,
        .arg3_type        = ARG_PTR_TO_STACK_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
           u64, flags)
{
        bpf_callback_t callback = (bpf_callback_t)callback_fn;
        u64 ret;
        u32 i;

        /* Note: these safety checks are also verified when bpf_loop
         * is inlined, be careful to modify this code in sync. See
         * function verifier.c:inline_bpf_loop.
         */
        if (flags)
                return -EINVAL;
        if (nr_loops > BPF_MAX_LOOPS)
                return -E2BIG;

        for (i = 0; i < nr_loops; i++) {
                ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0);
                /* return value: 0 - continue, 1 - stop and return */
                if (ret)
                        return i + 1;
        }

        return i;
}

const struct bpf_func_proto bpf_loop_proto = {
        .func                = bpf_loop,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_ANYTHING,
        .arg2_type        = ARG_PTR_TO_FUNC,
        .arg3_type        = ARG_PTR_TO_STACK_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};

struct bpf_iter_num_kern {
        int cur; /* current value, inclusive */
        int end; /* final value, exclusive */
} __aligned(8);

__bpf_kfunc_start_defs();

__bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end)
{
        struct bpf_iter_num_kern *s = (void *)it;

        BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num));
        BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num));

        /* start == end is legit, it's an empty range and we'll just get NULL
         * on first (and any subsequent) bpf_iter_num_next() call
         */
        if (start > end) {
                s->cur = s->end = 0;
                return -EINVAL;
        }

        /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */
        if ((s64)end - (s64)start > BPF_MAX_LOOPS) {
                s->cur = s->end = 0;
                return -E2BIG;
        }

        /* user will call bpf_iter_num_next() first,
         * which will set s->cur to exactly start value;
         * underflow shouldn't matter
         */
        s->cur = start - 1;
        s->end = end;

        return 0;
}

__bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it)
{
        struct bpf_iter_num_kern *s = (void *)it;

        /* check failed initialization or if we are done (same behavior);
         * need to be careful about overflow, so convert to s64 for checks,
         * e.g., if s->cur == s->end == INT_MAX, we can't just do
         * s->cur + 1 >= s->end
         */
        if ((s64)(s->cur + 1) >= s->end) {
                s->cur = s->end = 0;
                return NULL;
        }

        s->cur++;

        return &s->cur;
}

__bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
{
        struct bpf_iter_num_kern *s = (void *)it;

        s->cur = s->end = 0;
}

__bpf_kfunc_end_defs();




















































































































































    3 






    3 


    3 


    3 




    1 







    3 







    1 



    2 













    3 





    3 






    3 






























   11 






   11 







    1 



    1 



















































    9 
    9 


    9 



    9 



   11 
   11 


   11 








    6 







    2 















    1 



    3 









    2 










    2 
























   11 
   11 































    1 




   10 







   11 

    1 






    2 


    7 









    6 

    2 









    6 
    2 
    5 









    3 



    2 

    2 








    2 











    9 


















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Berkeley Packet Filter based traffic classifier
 *
 * Might be used to classify traffic through flexible, user-defined and
 * possibly JIT-ed BPF filters for traffic control as an alternative to
 * ematches.
 *
 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <linux/idr.h>

#include <net/rtnetlink.h>
#include <net/pkt_cls.h>
#include <net/sock.h>
#include <net/tc_wrapper.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
MODULE_DESCRIPTION("TC BPF based classifier");

#define CLS_BPF_NAME_LEN        256
#define CLS_BPF_SUPPORTED_GEN_FLAGS                \
        (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)

struct cls_bpf_head {
        struct list_head plist;
        struct idr handle_idr;
        struct rcu_head rcu;
};

struct cls_bpf_prog {
        struct bpf_prog *filter;
        struct list_head link;
        struct tcf_result res;
        bool exts_integrated;
        u32 gen_flags;
        unsigned int in_hw_count;
        struct tcf_exts exts;
        u32 handle;
        u16 bpf_num_ops;
        struct sock_filter *bpf_ops;
        const char *bpf_name;
        struct tcf_proto *tp;
        struct rcu_work rwork;
};

static const struct nla_policy bpf_policy[TCA_BPF_MAX + 1] = {
        [TCA_BPF_CLASSID]        = { .type = NLA_U32 },
        [TCA_BPF_FLAGS]                = { .type = NLA_U32 },
        [TCA_BPF_FLAGS_GEN]        = { .type = NLA_U32 },
        [TCA_BPF_FD]                = { .type = NLA_U32 },
        [TCA_BPF_NAME]                = { .type = NLA_NUL_STRING,
                                    .len = CLS_BPF_NAME_LEN },
        [TCA_BPF_OPS_LEN]        = { .type = NLA_U16 },
        [TCA_BPF_OPS]                = { .type = NLA_BINARY,
                                    .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
};

static int cls_bpf_exec_opcode(int code)
{
        switch (code) {
        case TC_ACT_OK:
        case TC_ACT_SHOT:
        case TC_ACT_STOLEN:
        case TC_ACT_TRAP:
        case TC_ACT_REDIRECT:
        case TC_ACT_UNSPEC:
                return code;
        default:
                return TC_ACT_UNSPEC;
        }
}

TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb,
                                       const struct tcf_proto *tp,
                                       struct tcf_result *res)
{
        struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
        bool at_ingress = skb_at_tc_ingress(skb);
        struct cls_bpf_prog *prog;
        int ret = -1;

        list_for_each_entry_rcu(prog, &head->plist, link) {
                int filter_res;

                qdisc_skb_cb(skb)->tc_classid = prog->res.classid;

                if (tc_skip_sw(prog->gen_flags)) {
                        filter_res = prog->exts_integrated ? TC_ACT_UNSPEC : 0;
                } else if (at_ingress) {
                        /* It is safe to push/pull even if skb_shared() */
                        __skb_push(skb, skb->mac_len);
                        bpf_compute_data_pointers(skb);
                        filter_res = bpf_prog_run(prog->filter, skb);
                        __skb_pull(skb, skb->mac_len);
                } else {
                        bpf_compute_data_pointers(skb);
                        filter_res = bpf_prog_run(prog->filter, skb);
                }
                if (unlikely(!skb->tstamp && skb->tstamp_type))
                        skb->tstamp_type = SKB_CLOCK_REALTIME;

                if (prog->exts_integrated) {
                        res->class   = 0;
                        res->classid = TC_H_MAJ(prog->res.classid) |
                                       qdisc_skb_cb(skb)->tc_classid;

                        ret = cls_bpf_exec_opcode(filter_res);
                        if (ret == TC_ACT_UNSPEC)
                                continue;
                        break;
                }

                if (filter_res == 0)
                        continue;
                if (filter_res != -1) {
                        res->class   = 0;
                        res->classid = filter_res;
                } else {
                        *res = prog->res;
                }

                ret = tcf_exts_exec(skb, &prog->exts, res);
                if (ret < 0)
                        continue;

                break;
        }

        return ret;
}

static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog)
{
        return !prog->bpf_ops;
}

static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog,
                               struct cls_bpf_prog *oldprog,
                               struct netlink_ext_ack *extack)
{
        struct tcf_block *block = tp->chain->block;
        struct tc_cls_bpf_offload cls_bpf = {};
        struct cls_bpf_prog *obj;
        bool skip_sw;
        int err;

        skip_sw = prog && tc_skip_sw(prog->gen_flags);
        obj = prog ?: oldprog;

        tc_cls_common_offload_init(&cls_bpf.common, tp, obj->gen_flags, extack);
        cls_bpf.command = TC_CLSBPF_OFFLOAD;
        cls_bpf.exts = &obj->exts;
        cls_bpf.prog = prog ? prog->filter : NULL;
        cls_bpf.oldprog = oldprog ? oldprog->filter : NULL;
        cls_bpf.name = obj->bpf_name;
        cls_bpf.exts_integrated = obj->exts_integrated;

        if (oldprog && prog)
                err = tc_setup_cb_replace(block, tp, TC_SETUP_CLSBPF, &cls_bpf,
                                          skip_sw, &oldprog->gen_flags,
                                          &oldprog->in_hw_count,
                                          &prog->gen_flags, &prog->in_hw_count,
                                          true);
        else if (prog)
                err = tc_setup_cb_add(block, tp, TC_SETUP_CLSBPF, &cls_bpf,
                                      skip_sw, &prog->gen_flags,
                                      &prog->in_hw_count, true);
        else
                err = tc_setup_cb_destroy(block, tp, TC_SETUP_CLSBPF, &cls_bpf,
                                          skip_sw, &oldprog->gen_flags,
                                          &oldprog->in_hw_count, true);

        if (prog && err) {
                cls_bpf_offload_cmd(tp, oldprog, prog, extack);
                return err;
        }

        if (prog && skip_sw && !(prog->gen_flags & TCA_CLS_FLAGS_IN_HW))
                return -EINVAL;

        return 0;
}

static u32 cls_bpf_flags(u32 flags)
{
        return flags & CLS_BPF_SUPPORTED_GEN_FLAGS;
}

static int cls_bpf_offload(struct tcf_proto *tp, struct cls_bpf_prog *prog,
                           struct cls_bpf_prog *oldprog,
                           struct netlink_ext_ack *extack)
{
        if (prog && oldprog &&
            cls_bpf_flags(prog->gen_flags) !=
            cls_bpf_flags(oldprog->gen_flags))
                return -EINVAL;

        if (prog && tc_skip_hw(prog->gen_flags))
                prog = NULL;
        if (oldprog && tc_skip_hw(oldprog->gen_flags))
                oldprog = NULL;
        if (!prog && !oldprog)
                return 0;

        return cls_bpf_offload_cmd(tp, prog, oldprog, extack);
}

static void cls_bpf_stop_offload(struct tcf_proto *tp,
                                 struct cls_bpf_prog *prog,
                                 struct netlink_ext_ack *extack)
{
        int err;

        err = cls_bpf_offload_cmd(tp, NULL, prog, extack);
        if (err)
                pr_err("Stopping hardware offload failed: %d\n", err);
}

static void cls_bpf_offload_update_stats(struct tcf_proto *tp,
                                         struct cls_bpf_prog *prog)
{
        struct tcf_block *block = tp->chain->block;
        struct tc_cls_bpf_offload cls_bpf = {};

        tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags, NULL);
        cls_bpf.command = TC_CLSBPF_STATS;
        cls_bpf.exts = &prog->exts;
        cls_bpf.prog = prog->filter;
        cls_bpf.name = prog->bpf_name;
        cls_bpf.exts_integrated = prog->exts_integrated;

        tc_setup_cb_call(block, TC_SETUP_CLSBPF, &cls_bpf, false, true);
}

static int cls_bpf_init(struct tcf_proto *tp)
{
        struct cls_bpf_head *head;

        head = kzalloc(sizeof(*head), GFP_KERNEL);
        if (head == NULL)
                return -ENOBUFS;

        INIT_LIST_HEAD_RCU(&head->plist);
        idr_init(&head->handle_idr);
        rcu_assign_pointer(tp->root, head);

        return 0;
}

static void cls_bpf_free_parms(struct cls_bpf_prog *prog)
{
        if (cls_bpf_is_ebpf(prog))
                bpf_prog_put(prog->filter);
        else
                bpf_prog_destroy(prog->filter);

        kfree(prog->bpf_name);
        kfree(prog->bpf_ops);
}

static void __cls_bpf_delete_prog(struct cls_bpf_prog *prog)
{
        tcf_exts_destroy(&prog->exts);
        tcf_exts_put_net(&prog->exts);

        cls_bpf_free_parms(prog);
        kfree(prog);
}

static void cls_bpf_delete_prog_work(struct work_struct *work)
{
        struct cls_bpf_prog *prog = container_of(to_rcu_work(work),
                                                 struct cls_bpf_prog,
                                                 rwork);
        rtnl_lock();
        __cls_bpf_delete_prog(prog);
        rtnl_unlock();
}

static void __cls_bpf_delete(struct tcf_proto *tp, struct cls_bpf_prog *prog,
                             struct netlink_ext_ack *extack)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);

        idr_remove(&head->handle_idr, prog->handle);
        cls_bpf_stop_offload(tp, prog, extack);
        list_del_rcu(&prog->link);
        tcf_unbind_filter(tp, &prog->res);
        if (tcf_exts_get_net(&prog->exts))
                tcf_queue_work(&prog->rwork, cls_bpf_delete_prog_work);
        else
                __cls_bpf_delete_prog(prog);
}

static int cls_bpf_delete(struct tcf_proto *tp, void *arg, bool *last,
                          bool rtnl_held, struct netlink_ext_ack *extack)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);

        __cls_bpf_delete(tp, arg, extack);
        *last = list_empty(&head->plist);
        return 0;
}

static void cls_bpf_destroy(struct tcf_proto *tp, bool rtnl_held,
                            struct netlink_ext_ack *extack)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);
        struct cls_bpf_prog *prog, *tmp;

        list_for_each_entry_safe(prog, tmp, &head->plist, link)
                __cls_bpf_delete(tp, prog, extack);

        idr_destroy(&head->handle_idr);
        kfree_rcu(head, rcu);
}

static void *cls_bpf_get(struct tcf_proto *tp, u32 handle)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);
        struct cls_bpf_prog *prog;

        list_for_each_entry(prog, &head->plist, link) {
                if (prog->handle == handle)
                        return prog;
        }

        return NULL;
}

static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog)
{
        struct sock_filter *bpf_ops;
        struct sock_fprog_kern fprog_tmp;
        struct bpf_prog *fp;
        u16 bpf_size, bpf_num_ops;
        int ret;

        bpf_num_ops = nla_get_u16(tb[TCA_BPF_OPS_LEN]);
        if (bpf_num_ops > BPF_MAXINSNS || bpf_num_ops == 0)
                return -EINVAL;

        bpf_size = bpf_num_ops * sizeof(*bpf_ops);
        if (bpf_size != nla_len(tb[TCA_BPF_OPS]))
                return -EINVAL;

        bpf_ops = kmemdup(nla_data(tb[TCA_BPF_OPS]), bpf_size, GFP_KERNEL);
        if (bpf_ops == NULL)
                return -ENOMEM;

        fprog_tmp.len = bpf_num_ops;
        fprog_tmp.filter = bpf_ops;

        ret = bpf_prog_create(&fp, &fprog_tmp);
        if (ret < 0) {
                kfree(bpf_ops);
                return ret;
        }

        prog->bpf_ops = bpf_ops;
        prog->bpf_num_ops = bpf_num_ops;
        prog->bpf_name = NULL;
        prog->filter = fp;

        return 0;
}

static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog,
                                 u32 gen_flags, const struct tcf_proto *tp)
{
        struct bpf_prog *fp;
        char *name = NULL;
        bool skip_sw;
        u32 bpf_fd;

        bpf_fd = nla_get_u32(tb[TCA_BPF_FD]);
        skip_sw = gen_flags & TCA_CLS_FLAGS_SKIP_SW;

        fp = bpf_prog_get_type_dev(bpf_fd, BPF_PROG_TYPE_SCHED_CLS, skip_sw);
        if (IS_ERR(fp))
                return PTR_ERR(fp);

        if (tb[TCA_BPF_NAME]) {
                name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL);
                if (!name) {
                        bpf_prog_put(fp);
                        return -ENOMEM;
                }
        }

        prog->bpf_ops = NULL;
        prog->bpf_name = name;
        prog->filter = fp;

        if (fp->dst_needed)
                tcf_block_netif_keep_dst(tp->chain->block);

        return 0;
}

static int cls_bpf_change(struct net *net, struct sk_buff *in_skb,
                          struct tcf_proto *tp, unsigned long base,
                          u32 handle, struct nlattr **tca,
                          void **arg, u32 flags,
                          struct netlink_ext_ack *extack)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);
        bool is_bpf, is_ebpf, have_exts = false;
        struct cls_bpf_prog *oldprog = *arg;
        struct nlattr *tb[TCA_BPF_MAX + 1];
        bool bound_to_filter = false;
        struct cls_bpf_prog *prog;
        u32 gen_flags = 0;
        int ret;

        if (tca[TCA_OPTIONS] == NULL)
                return -EINVAL;

        ret = nla_parse_nested_deprecated(tb, TCA_BPF_MAX, tca[TCA_OPTIONS],
                                          bpf_policy, NULL);
        if (ret < 0)
                return ret;

        prog = kzalloc(sizeof(*prog), GFP_KERNEL);
        if (!prog)
                return -ENOBUFS;

        ret = tcf_exts_init(&prog->exts, net, TCA_BPF_ACT, TCA_BPF_POLICE);
        if (ret < 0)
                goto errout;

        if (oldprog) {
                if (handle && oldprog->handle != handle) {
                        ret = -EINVAL;
                        goto errout;
                }
        }

        if (handle == 0) {
                handle = 1;
                ret = idr_alloc_u32(&head->handle_idr, prog, &handle,
                                    INT_MAX, GFP_KERNEL);
        } else if (!oldprog) {
                ret = idr_alloc_u32(&head->handle_idr, prog, &handle,
                                    handle, GFP_KERNEL);
        }

        if (ret)
                goto errout;
        prog->handle = handle;

        is_bpf = tb[TCA_BPF_OPS_LEN] && tb[TCA_BPF_OPS];
        is_ebpf = tb[TCA_BPF_FD];
        if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) {
                ret = -EINVAL;
                goto errout_idr;
        }

        ret = tcf_exts_validate(net, tp, tb, tca[TCA_RATE], &prog->exts,
                                flags, extack);
        if (ret < 0)
                goto errout_idr;

        if (tb[TCA_BPF_FLAGS]) {
                u32 bpf_flags = nla_get_u32(tb[TCA_BPF_FLAGS]);

                if (bpf_flags & ~TCA_BPF_FLAG_ACT_DIRECT) {
                        ret = -EINVAL;
                        goto errout_idr;
                }

                have_exts = bpf_flags & TCA_BPF_FLAG_ACT_DIRECT;
        }
        if (tb[TCA_BPF_FLAGS_GEN]) {
                gen_flags = nla_get_u32(tb[TCA_BPF_FLAGS_GEN]);
                if (gen_flags & ~CLS_BPF_SUPPORTED_GEN_FLAGS ||
                    !tc_flags_valid(gen_flags)) {
                        ret = -EINVAL;
                        goto errout_idr;
                }
        }

        prog->exts_integrated = have_exts;
        prog->gen_flags = gen_flags;

        ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog) :
                cls_bpf_prog_from_efd(tb, prog, gen_flags, tp);
        if (ret < 0)
                goto errout_idr;

        if (tb[TCA_BPF_CLASSID]) {
                prog->res.classid = nla_get_u32(tb[TCA_BPF_CLASSID]);
                tcf_bind_filter(tp, &prog->res, base);
                bound_to_filter = true;
        }

        ret = cls_bpf_offload(tp, prog, oldprog, extack);
        if (ret)
                goto errout_parms;

        if (!tc_in_hw(prog->gen_flags))
                prog->gen_flags |= TCA_CLS_FLAGS_NOT_IN_HW;

        tcf_proto_update_usesw(tp, prog->gen_flags);

        if (oldprog) {
                idr_replace(&head->handle_idr, prog, handle);
                list_replace_rcu(&oldprog->link, &prog->link);
                tcf_unbind_filter(tp, &oldprog->res);
                tcf_exts_get_net(&oldprog->exts);
                tcf_queue_work(&oldprog->rwork, cls_bpf_delete_prog_work);
        } else {
                list_add_rcu(&prog->link, &head->plist);
        }

        *arg = prog;
        return 0;

errout_parms:
        if (bound_to_filter)
                tcf_unbind_filter(tp, &prog->res);
        cls_bpf_free_parms(prog);
errout_idr:
        if (!oldprog)
                idr_remove(&head->handle_idr, prog->handle);
errout:
        tcf_exts_destroy(&prog->exts);
        kfree(prog);
        return ret;
}

static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog *prog,
                                 struct sk_buff *skb)
{
        struct nlattr *nla;

        if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))
                return -EMSGSIZE;

        nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *
                          sizeof(struct sock_filter));
        if (nla == NULL)
                return -EMSGSIZE;

        memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));

        return 0;
}

static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog *prog,
                                  struct sk_buff *skb)
{
        struct nlattr *nla;

        if (prog->bpf_name &&
            nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))
                return -EMSGSIZE;

        if (nla_put_u32(skb, TCA_BPF_ID, prog->filter->aux->id))
                return -EMSGSIZE;

        nla = nla_reserve(skb, TCA_BPF_TAG, sizeof(prog->filter->tag));
        if (nla == NULL)
                return -EMSGSIZE;

        memcpy(nla_data(nla), prog->filter->tag, nla_len(nla));

        return 0;
}

static int cls_bpf_dump(struct net *net, struct tcf_proto *tp, void *fh,
                        struct sk_buff *skb, struct tcmsg *tm, bool rtnl_held)
{
        struct cls_bpf_prog *prog = fh;
        struct nlattr *nest;
        u32 bpf_flags = 0;
        int ret;

        if (prog == NULL)
                return skb->len;

        tm->tcm_handle = prog->handle;

        cls_bpf_offload_update_stats(tp, prog);

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (nest == NULL)
                goto nla_put_failure;

        if (prog->res.classid &&
            nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))
                goto nla_put_failure;

        if (cls_bpf_is_ebpf(prog))
                ret = cls_bpf_dump_ebpf_info(prog, skb);
        else
                ret = cls_bpf_dump_bpf_info(prog, skb);
        if (ret)
                goto nla_put_failure;

        if (tcf_exts_dump(skb, &prog->exts) < 0)
                goto nla_put_failure;

        if (prog->exts_integrated)
                bpf_flags |= TCA_BPF_FLAG_ACT_DIRECT;
        if (bpf_flags && nla_put_u32(skb, TCA_BPF_FLAGS, bpf_flags))
                goto nla_put_failure;
        if (prog->gen_flags &&
            nla_put_u32(skb, TCA_BPF_FLAGS_GEN, prog->gen_flags))
                goto nla_put_failure;

        nla_nest_end(skb, nest);

        if (tcf_exts_dump_stats(skb, &prog->exts) < 0)
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static void cls_bpf_bind_class(void *fh, u32 classid, unsigned long cl,
                               void *q, unsigned long base)
{
        struct cls_bpf_prog *prog = fh;

        tc_cls_bind_class(classid, cl, q, &prog->res, base);
}

static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg,
                         bool rtnl_held)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);
        struct cls_bpf_prog *prog;

        list_for_each_entry(prog, &head->plist, link) {
                if (!tc_cls_stats_dump(tp, arg, prog))
                        break;
        }
}

static int cls_bpf_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,
                             void *cb_priv, struct netlink_ext_ack *extack)
{
        struct cls_bpf_head *head = rtnl_dereference(tp->root);
        struct tcf_block *block = tp->chain->block;
        struct tc_cls_bpf_offload cls_bpf = {};
        struct cls_bpf_prog *prog;
        int err;

        list_for_each_entry(prog, &head->plist, link) {
                if (tc_skip_hw(prog->gen_flags))
                        continue;

                tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags,
                                           extack);
                cls_bpf.command = TC_CLSBPF_OFFLOAD;
                cls_bpf.exts = &prog->exts;
                cls_bpf.prog = add ? prog->filter : NULL;
                cls_bpf.oldprog = add ? NULL : prog->filter;
                cls_bpf.name = prog->bpf_name;
                cls_bpf.exts_integrated = prog->exts_integrated;

                err = tc_setup_cb_reoffload(block, tp, add, cb, TC_SETUP_CLSBPF,
                                            &cls_bpf, cb_priv, &prog->gen_flags,
                                            &prog->in_hw_count);
                if (err)
                        return err;
        }

        return 0;
}

static struct tcf_proto_ops cls_bpf_ops __read_mostly = {
        .kind                =        "bpf",
        .owner                =        THIS_MODULE,
        .classify        =        cls_bpf_classify,
        .init                =        cls_bpf_init,
        .destroy        =        cls_bpf_destroy,
        .get                =        cls_bpf_get,
        .change                =        cls_bpf_change,
        .delete                =        cls_bpf_delete,
        .walk                =        cls_bpf_walk,
        .reoffload        =        cls_bpf_reoffload,
        .dump                =        cls_bpf_dump,
        .bind_class        =        cls_bpf_bind_class,
};
MODULE_ALIAS_NET_CLS("bpf");

static int __init cls_bpf_init_mod(void)
{
        return register_tcf_proto_ops(&cls_bpf_ops);
}

static void __exit cls_bpf_exit_mod(void)
{
        unregister_tcf_proto_ops(&cls_bpf_ops);
}

module_init(cls_bpf_init_mod);
module_exit(cls_bpf_exit_mod);





























































































































































































































   16 





















   16 
   16 






































   13 













   18 













   18 












    4 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Queue of folios definitions
 *
 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * See:
 *
 *        Documentation/core-api/folio_queue.rst
 *
 * for a description of the API.
 */

#ifndef _LINUX_FOLIO_QUEUE_H
#define _LINUX_FOLIO_QUEUE_H

#include <linux/pagevec.h>

/*
 * Segment in a queue of running buffers.  Each segment can hold a number of
 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
 * iterator.  The possibility exists of inserting non-folio elements into the
 * queue (such as gaps).
 *
 * Explicit prev and next pointers are used instead of a list_head to make it
 * easier to add segments to tail and remove them from the head without the
 * need for a lock.
 */
struct folio_queue {
        struct folio_batch        vec;                /* Folios in the queue segment */
        u8                        orders[PAGEVEC_SIZE]; /* Order of each folio */
        struct folio_queue        *next;                /* Next queue segment or NULL */
        struct folio_queue        *prev;                /* Previous queue segment of NULL */
        unsigned long                marks;                /* 1-bit mark per folio */
        unsigned long                marks2;                /* Second 1-bit mark per folio */
        unsigned long                marks3;                /* Third 1-bit mark per folio */
#if PAGEVEC_SIZE > BITS_PER_LONG
#error marks is not big enough
#endif
        unsigned int                rreq_id;
        unsigned int                debug_id;
};

/**
 * folioq_init - Initialise a folio queue segment
 * @folioq: The segment to initialise
 * @rreq_id: The request identifier to use in tracelines.
 *
 * Initialise a folio queue segment and set an identifier to be used in traces.
 *
 * Note that the folio pointers are left uninitialised.
 */
static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id)
{
        folio_batch_init(&folioq->vec);
        folioq->next = NULL;
        folioq->prev = NULL;
        folioq->marks = 0;
        folioq->marks2 = 0;
        folioq->marks3 = 0;
        folioq->rreq_id = rreq_id;
        folioq->debug_id = 0;
}

/**
 * folioq_nr_slots: Query the capacity of a folio queue segment
 * @folioq: The segment to query
 *
 * Query the number of folios that a particular folio queue segment might hold.
 * [!] NOTE: This must not be assumed to be the same for every segment!
 */
static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
{
        return PAGEVEC_SIZE;
}

/**
 * folioq_count: Query the occupancy of a folio queue segment
 * @folioq: The segment to query
 *
 * Query the number of folios that have been added to a folio queue segment.
 * Note that this is not decreased as folios are removed from a segment.
 */
static inline unsigned int folioq_count(struct folio_queue *folioq)
{
        return folio_batch_count(&folioq->vec);
}

/**
 * folioq_full: Query if a folio queue segment is full
 * @folioq: The segment to query
 *
 * Query if a folio queue segment is fully occupied.  Note that this does not
 * change if folios are removed from a segment.
 */
static inline bool folioq_full(struct folio_queue *folioq)
{
        //return !folio_batch_space(&folioq->vec);
        return folioq_count(folioq) >= folioq_nr_slots(folioq);
}

/**
 * folioq_is_marked: Check first folio mark in a folio queue segment
 * @folioq: The segment to query
 * @slot: The slot number of the folio to query
 *
 * Determine if the first mark is set for the folio in the specified slot in a
 * folio queue segment.
 */
static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
{
        return test_bit(slot, &folioq->marks);
}

/**
 * folioq_mark: Set the first mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Set the first mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
{
        set_bit(slot, &folioq->marks);
}

/**
 * folioq_unmark: Clear the first mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Clear the first mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
{
        clear_bit(slot, &folioq->marks);
}

/**
 * folioq_is_marked2: Check second folio mark in a folio queue segment
 * @folioq: The segment to query
 * @slot: The slot number of the folio to query
 *
 * Determine if the second mark is set for the folio in the specified slot in a
 * folio queue segment.
 */
static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
{
        return test_bit(slot, &folioq->marks2);
}

/**
 * folioq_mark2: Set the second mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Set the second mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
{
        set_bit(slot, &folioq->marks2);
}

/**
 * folioq_unmark2: Clear the second mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Clear the second mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
{
        clear_bit(slot, &folioq->marks2);
}

/**
 * folioq_is_marked3: Check third folio mark in a folio queue segment
 * @folioq: The segment to query
 * @slot: The slot number of the folio to query
 *
 * Determine if the third mark is set for the folio in the specified slot in a
 * folio queue segment.
 */
static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot)
{
        return test_bit(slot, &folioq->marks3);
}

/**
 * folioq_mark3: Set the third mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Set the third mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot)
{
        set_bit(slot, &folioq->marks3);
}

/**
 * folioq_unmark3: Clear the third mark on a folio in a folio queue segment
 * @folioq: The segment to modify
 * @slot: The slot number of the folio to modify
 *
 * Clear the third mark for the folio in the specified slot in a folio queue
 * segment.
 */
static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot)
{
        clear_bit(slot, &folioq->marks3);
}

static inline unsigned int __folio_order(struct folio *folio)
{
        if (!folio_test_large(folio))
                return 0;
        return folio->_flags_1 & 0xff;
}

/**
 * folioq_append: Add a folio to a folio queue segment
 * @folioq: The segment to add to
 * @folio: The folio to add
 *
 * Add a folio to the tail of the sequence in a folio queue segment, increasing
 * the occupancy count and returning the slot number for the folio just added.
 * The folio size is extracted and stored in the queue and the marks are left
 * unmodified.
 *
 * Note that it's left up to the caller to check that the segment capacity will
 * not be exceeded and to extend the queue.
 */
static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
{
        unsigned int slot = folioq->vec.nr++;

        folioq->vec.folios[slot] = folio;
        folioq->orders[slot] = __folio_order(folio);
        return slot;
}

/**
 * folioq_append_mark: Add a folio to a folio queue segment
 * @folioq: The segment to add to
 * @folio: The folio to add
 *
 * Add a folio to the tail of the sequence in a folio queue segment, increasing
 * the occupancy count and returning the slot number for the folio just added.
 * The folio size is extracted and stored in the queue, the first mark is set
 * and and the second and third marks are left unmodified.
 *
 * Note that it's left up to the caller to check that the segment capacity will
 * not be exceeded and to extend the queue.
 */
static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
{
        unsigned int slot = folioq->vec.nr++;

        folioq->vec.folios[slot] = folio;
        folioq->orders[slot] = __folio_order(folio);
        folioq_mark(folioq, slot);
        return slot;
}

/**
 * folioq_folio: Get a folio from a folio queue segment
 * @folioq: The segment to access
 * @slot: The folio slot to access
 *
 * Retrieve the folio in the specified slot from a folio queue segment.  Note
 * that no bounds check is made and if the slot hasn't been added into yet, the
 * pointer will be undefined.  If the slot has been cleared, NULL will be
 * returned.
 */
static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
{
        return folioq->vec.folios[slot];
}

/**
 * folioq_folio_order: Get the order of a folio from a folio queue segment
 * @folioq: The segment to access
 * @slot: The folio slot to access
 *
 * Retrieve the order of the folio in the specified slot from a folio queue
 * segment.  Note that no bounds check is made and if the slot hasn't been
 * added into yet, the order returned will be 0.
 */
static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
{
        return folioq->orders[slot];
}

/**
 * folioq_folio_size: Get the size of a folio from a folio queue segment
 * @folioq: The segment to access
 * @slot: The folio slot to access
 *
 * Retrieve the size of the folio in the specified slot from a folio queue
 * segment.  Note that no bounds check is made and if the slot hasn't been
 * added into yet, the size returned will be PAGE_SIZE.
 */
static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
{
        return PAGE_SIZE << folioq_folio_order(folioq, slot);
}

/**
 * folioq_clear: Clear a folio from a folio queue segment
 * @folioq: The segment to clear
 * @slot: The folio slot to clear
 *
 * Clear a folio from a sequence in a folio queue segment and clear its marks.
 * The occupancy count is left unchanged.
 */
static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
{
        folioq->vec.folios[slot] = NULL;
        folioq_unmark(folioq, slot);
        folioq_unmark2(folioq, slot);
        folioq_unmark3(folioq, slot);
}

#endif /* _LINUX_FOLIO_QUEUE_H */





























































































































































































































































































































































































  144 
  143 





























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * kernfs.h - pseudo filesystem decoupled from vfs locking
 */

#ifndef __LINUX_KERNFS_H
#define __LINUX_KERNFS_H

#include <linux/err.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/idr.h>
#include <linux/lockdep.h>
#include <linux/rbtree.h>
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/uidgid.h>
#include <linux/wait.h>
#include <linux/rwsem.h>
#include <linux/cache.h>

struct file;
struct dentry;
struct iattr;
struct seq_file;
struct vm_area_struct;
struct vm_operations_struct;
struct super_block;
struct file_system_type;
struct poll_table_struct;
struct fs_context;

struct kernfs_fs_context;
struct kernfs_open_node;
struct kernfs_iattrs;

/*
 * NR_KERNFS_LOCK_BITS determines size (NR_KERNFS_LOCKS) of hash
 * table of locks.
 * Having a small hash table would impact scalability, since
 * more and more kernfs_node objects will end up using same lock
 * and having a very large hash table would waste memory.
 *
 * At the moment size of hash table of locks is being set based on
 * the number of CPUs as follows:
 *
 * NR_CPU      NR_KERNFS_LOCK_BITS      NR_KERNFS_LOCKS
 *   1                  1                       2
 *  2-3                 2                       4
 *  4-7                 4                       16
 *  8-15                6                       64
 *  16-31               8                       256
 *  32 and more         10                      1024
 *
 * The above relation between NR_CPU and number of locks is based
 * on some internal experimentation which involved booting qemu
 * with different values of smp, performing some sysfs operations
 * on all CPUs and observing how increase in number of locks impacts
 * completion time of these sysfs operations on each CPU.
 */
#ifdef CONFIG_SMP
#define NR_KERNFS_LOCK_BITS (2 * (ilog2(NR_CPUS < 32 ? NR_CPUS : 32)))
#else
#define NR_KERNFS_LOCK_BITS     1
#endif

#define NR_KERNFS_LOCKS     (1 << NR_KERNFS_LOCK_BITS)

/*
 * There's one kernfs_open_file for each open file and one kernfs_open_node
 * for each kernfs_node with one or more open files.
 *
 * filp->private_data points to seq_file whose ->private points to
 * kernfs_open_file.
 *
 * kernfs_open_files are chained at kernfs_open_node->files, which is
 * protected by kernfs_global_locks.open_file_mutex[i].
 *
 * To reduce possible contention in sysfs access, arising due to single
 * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node
 * object address as hash keys to get the index of these locks.
 *
 * Hashed mutexes are safe to use here because operations using these don't
 * rely on global exclusion.
 *
 * In future we intend to replace other global locks with hashed ones as well.
 * kernfs_global_locks acts as a holder for all such hash tables.
 */
struct kernfs_global_locks {
        struct mutex open_file_mutex[NR_KERNFS_LOCKS];
};

enum kernfs_node_type {
        KERNFS_DIR                = 0x0001,
        KERNFS_FILE                = 0x0002,
        KERNFS_LINK                = 0x0004,
};

#define KERNFS_TYPE_MASK                0x000f
#define KERNFS_FLAG_MASK                ~KERNFS_TYPE_MASK
#define KERNFS_MAX_USER_XATTRS                128
#define KERNFS_USER_XATTR_SIZE_LIMIT        (128 << 10)

enum kernfs_node_flag {
        KERNFS_ACTIVATED        = 0x0010,
        KERNFS_NS                = 0x0020,
        KERNFS_HAS_SEQ_SHOW        = 0x0040,
        KERNFS_HAS_MMAP                = 0x0080,
        KERNFS_LOCKDEP                = 0x0100,
        KERNFS_HIDDEN                = 0x0200,
        KERNFS_SUICIDAL                = 0x0400,
        KERNFS_SUICIDED                = 0x0800,
        KERNFS_EMPTY_DIR        = 0x1000,
        KERNFS_HAS_RELEASE        = 0x2000,
        KERNFS_REMOVING                = 0x4000,
};

/* @flags for kernfs_create_root() */
enum kernfs_root_flag {
        /*
         * kernfs_nodes are created in the deactivated state and invisible.
         * They require explicit kernfs_activate() to become visible.  This
         * can be used to make related nodes become visible atomically
         * after all nodes are created successfully.
         */
        KERNFS_ROOT_CREATE_DEACTIVATED                = 0x0001,

        /*
         * For regular files, if the opener has CAP_DAC_OVERRIDE, open(2)
         * succeeds regardless of the RW permissions.  sysfs had an extra
         * layer of enforcement where open(2) fails with -EACCES regardless
         * of CAP_DAC_OVERRIDE if the permission doesn't have the
         * respective read or write access at all (none of S_IRUGO or
         * S_IWUGO) or the respective operation isn't implemented.  The
         * following flag enables that behavior.
         */
        KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK        = 0x0002,

        /*
         * The filesystem supports exportfs operation, so userspace can use
         * fhandle to access nodes of the fs.
         */
        KERNFS_ROOT_SUPPORT_EXPORTOP                = 0x0004,

        /*
         * Support user xattrs to be written to nodes rooted at this root.
         */
        KERNFS_ROOT_SUPPORT_USER_XATTR                = 0x0008,
};

/* type-specific structures for kernfs_node union members */
struct kernfs_elem_dir {
        unsigned long                subdirs;
        /* children rbtree starts here and goes through kn->rb */
        struct rb_root                children;

        /*
         * The kernfs hierarchy this directory belongs to.  This fits
         * better directly in kernfs_node but is here to save space.
         */
        struct kernfs_root        *root;
        /*
         * Monotonic revision counter, used to identify if a directory
         * node has changed during negative dentry revalidation.
         */
        unsigned long                rev;
};

struct kernfs_elem_symlink {
        struct kernfs_node        *target_kn;
};

struct kernfs_elem_attr {
        const struct kernfs_ops        *ops;
        struct kernfs_open_node __rcu        *open;
        loff_t                        size;
        struct kernfs_node        *notify_next;        /* for kernfs_notify() */
};

/*
 * kernfs_node - the building block of kernfs hierarchy.  Each and every
 * kernfs node is represented by single kernfs_node.  Most fields are
 * private to kernfs and shouldn't be accessed directly by kernfs users.
 *
 * As long as count reference is held, the kernfs_node itself is
 * accessible.  Dereferencing elem or any other outer entity requires
 * active reference.
 */
struct kernfs_node {
        atomic_t                count;
        atomic_t                active;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map        dep_map;
#endif
        /*
         * Use kernfs_get_parent() and kernfs_name/path() instead of
         * accessing the following two fields directly.  If the node is
         * never moved to a different parent, it is safe to access the
         * parent directly.
         */
        struct kernfs_node        *parent;
        const char                *name;

        struct rb_node                rb;

        const void                *ns;        /* namespace tag */
        unsigned int                hash;        /* ns + name hash */
        unsigned short                flags;
        umode_t                        mode;

        union {
                struct kernfs_elem_dir                dir;
                struct kernfs_elem_symlink        symlink;
                struct kernfs_elem_attr                attr;
        };

        /*
         * 64bit unique ID.  On 64bit ino setups, id is the ino.  On 32bit,
         * the low 32bits are ino and upper generation.
         */
        u64                        id;

        void                        *priv;
        struct kernfs_iattrs        *iattr;

        struct rcu_head                rcu;
};

/*
 * kernfs_syscall_ops may be specified on kernfs_create_root() to support
 * syscalls.  These optional callbacks are invoked on the matching syscalls
 * and can perform any kernfs operations which don't necessarily have to be
 * the exact operation requested.  An active reference is held for each
 * kernfs_node parameter.
 */
struct kernfs_syscall_ops {
        int (*show_options)(struct seq_file *sf, struct kernfs_root *root);

        int (*mkdir)(struct kernfs_node *parent, const char *name,
                     umode_t mode);
        int (*rmdir)(struct kernfs_node *kn);
        int (*rename)(struct kernfs_node *kn, struct kernfs_node *new_parent,
                      const char *new_name);
        int (*show_path)(struct seq_file *sf, struct kernfs_node *kn,
                         struct kernfs_root *root);
};

struct kernfs_node *kernfs_root_to_node(struct kernfs_root *root);

struct kernfs_open_file {
        /* published fields */
        struct kernfs_node        *kn;
        struct file                *file;
        struct seq_file                *seq_file;
        void                        *priv;

        /* private fields, do not use outside kernfs proper */
        struct mutex                mutex;
        struct mutex                prealloc_mutex;
        int                        event;
        struct list_head        list;
        char                        *prealloc_buf;

        size_t                        atomic_write_len;
        bool                        mmapped:1;
        bool                        released:1;
        const struct vm_operations_struct *vm_ops;
};

struct kernfs_ops {
        /*
         * Optional open/release methods.  Both are called with
         * @of->seq_file populated.
         */
        int (*open)(struct kernfs_open_file *of);
        void (*release)(struct kernfs_open_file *of);

        /*
         * Read is handled by either seq_file or raw_read().
         *
         * If seq_show() is present, seq_file path is active.  Other seq
         * operations are optional and if not implemented, the behavior is
         * equivalent to single_open().  @sf->private points to the
         * associated kernfs_open_file.
         *
         * read() is bounced through kernel buffer and a read larger than
         * PAGE_SIZE results in partial operation of PAGE_SIZE.
         */
        int (*seq_show)(struct seq_file *sf, void *v);

        void *(*seq_start)(struct seq_file *sf, loff_t *ppos);
        void *(*seq_next)(struct seq_file *sf, void *v, loff_t *ppos);
        void (*seq_stop)(struct seq_file *sf, void *v);

        ssize_t (*read)(struct kernfs_open_file *of, char *buf, size_t bytes,
                        loff_t off);

        /*
         * write() is bounced through kernel buffer.  If atomic_write_len
         * is not set, a write larger than PAGE_SIZE results in partial
         * operations of PAGE_SIZE chunks.  If atomic_write_len is set,
         * writes upto the specified size are executed atomically but
         * larger ones are rejected with -E2BIG.
         */
        size_t atomic_write_len;
        /*
         * "prealloc" causes a buffer to be allocated at open for
         * all read/write requests.  As ->seq_show uses seq_read()
         * which does its own allocation, it is incompatible with
         * ->prealloc.  Provide ->read and ->write with ->prealloc.
         */
        bool prealloc;
        ssize_t (*write)(struct kernfs_open_file *of, char *buf, size_t bytes,
                         loff_t off);

        __poll_t (*poll)(struct kernfs_open_file *of,
                         struct poll_table_struct *pt);

        int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma);
        loff_t (*llseek)(struct kernfs_open_file *of, loff_t offset, int whence);
};

/*
 * The kernfs superblock creation/mount parameter context.
 */
struct kernfs_fs_context {
        struct kernfs_root        *root;                /* Root of the hierarchy being mounted */
        void                        *ns_tag;        /* Namespace tag of the mount (or NULL) */
        unsigned long                magic;                /* File system specific magic number */

        /* The following are set/used by kernfs_mount() */
        bool                        new_sb_created;        /* Set to T if we allocated a new sb */
};

#ifdef CONFIG_KERNFS

static inline enum kernfs_node_type kernfs_type(struct kernfs_node *kn)
{
        return kn->flags & KERNFS_TYPE_MASK;
}

static inline ino_t kernfs_id_ino(u64 id)
{
        /* id is ino if ino_t is 64bit; otherwise, low 32bits */
        if (sizeof(ino_t) >= sizeof(u64))
                return id;
        else
                return (u32)id;
}

static inline u32 kernfs_id_gen(u64 id)
{
        /* gen is fixed at 1 if ino_t is 64bit; otherwise, high 32bits */
        if (sizeof(ino_t) >= sizeof(u64))
                return 1;
        else
                return id >> 32;
}

static inline ino_t kernfs_ino(struct kernfs_node *kn)
{
        return kernfs_id_ino(kn->id);
}

static inline ino_t kernfs_gen(struct kernfs_node *kn)
{
        return kernfs_id_gen(kn->id);
}

/**
 * kernfs_enable_ns - enable namespace under a directory
 * @kn: directory of interest, should be empty
 *
 * This is to be called right after @kn is created to enable namespace
 * under it.  All children of @kn must have non-NULL namespace tags and
 * only the ones which match the super_block's tag will be visible.
 */
static inline void kernfs_enable_ns(struct kernfs_node *kn)
{
        WARN_ON_ONCE(kernfs_type(kn) != KERNFS_DIR);
        WARN_ON_ONCE(!RB_EMPTY_ROOT(&kn->dir.children));
        kn->flags |= KERNFS_NS;
}

/**
 * kernfs_ns_enabled - test whether namespace is enabled
 * @kn: the node to test
 *
 * Test whether namespace filtering is enabled for the children of @ns.
 */
static inline bool kernfs_ns_enabled(struct kernfs_node *kn)
{
        return kn->flags & KERNFS_NS;
}

int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen);
int kernfs_path_from_node(struct kernfs_node *root_kn, struct kernfs_node *kn,
                          char *buf, size_t buflen);
void pr_cont_kernfs_name(struct kernfs_node *kn);
void pr_cont_kernfs_path(struct kernfs_node *kn);
struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn);
struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent,
                                           const char *name, const void *ns);
struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent,
                                           const char *path, const void *ns);
void kernfs_get(struct kernfs_node *kn);
void kernfs_put(struct kernfs_node *kn);

struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry);
struct kernfs_root *kernfs_root_from_sb(struct super_block *sb);
struct inode *kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn);

struct dentry *kernfs_node_dentry(struct kernfs_node *kn,
                                  struct super_block *sb);
struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
                                       unsigned int flags, void *priv);
void kernfs_destroy_root(struct kernfs_root *root);

struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent,
                                         const char *name, umode_t mode,
                                         kuid_t uid, kgid_t gid,
                                         void *priv, const void *ns);
struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
                                            const char *name);
struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent,
                                         const char *name, umode_t mode,
                                         kuid_t uid, kgid_t gid,
                                         loff_t size,
                                         const struct kernfs_ops *ops,
                                         void *priv, const void *ns,
                                         struct lock_class_key *key);
struct kernfs_node *kernfs_create_link(struct kernfs_node *parent,
                                       const char *name,
                                       struct kernfs_node *target);
void kernfs_activate(struct kernfs_node *kn);
void kernfs_show(struct kernfs_node *kn, bool show);
void kernfs_remove(struct kernfs_node *kn);
void kernfs_break_active_protection(struct kernfs_node *kn);
void kernfs_unbreak_active_protection(struct kernfs_node *kn);
bool kernfs_remove_self(struct kernfs_node *kn);
int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
                             const void *ns);
int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
                     const char *new_name, const void *new_ns);
int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr);
__poll_t kernfs_generic_poll(struct kernfs_open_file *of,
                             struct poll_table_struct *pt);
void kernfs_notify(struct kernfs_node *kn);

int kernfs_xattr_get(struct kernfs_node *kn, const char *name,
                     void *value, size_t size);
int kernfs_xattr_set(struct kernfs_node *kn, const char *name,
                     const void *value, size_t size, int flags);

const void *kernfs_super_ns(struct super_block *sb);
int kernfs_get_tree(struct fs_context *fc);
void kernfs_free_fs_context(struct fs_context *fc);
void kernfs_kill_sb(struct super_block *sb);

void kernfs_init(void);

struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root,
                                                   u64 id);
#else        /* CONFIG_KERNFS */

static inline enum kernfs_node_type kernfs_type(struct kernfs_node *kn)
{ return 0; }        /* whatever */

static inline void kernfs_enable_ns(struct kernfs_node *kn) { }

static inline bool kernfs_ns_enabled(struct kernfs_node *kn)
{ return false; }

static inline int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)
{ return -ENOSYS; }

static inline int kernfs_path_from_node(struct kernfs_node *root_kn,
                                        struct kernfs_node *kn,
                                        char *buf, size_t buflen)
{ return -ENOSYS; }

static inline void pr_cont_kernfs_name(struct kernfs_node *kn) { }
static inline void pr_cont_kernfs_path(struct kernfs_node *kn) { }

static inline struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)
{ return NULL; }

static inline struct kernfs_node *
kernfs_find_and_get_ns(struct kernfs_node *parent, const char *name,
                       const void *ns)
{ return NULL; }
static inline struct kernfs_node *
kernfs_walk_and_get_ns(struct kernfs_node *parent, const char *path,
                       const void *ns)
{ return NULL; }

static inline void kernfs_get(struct kernfs_node *kn) { }
static inline void kernfs_put(struct kernfs_node *kn) { }

static inline struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry)
{ return NULL; }

static inline struct kernfs_root *kernfs_root_from_sb(struct super_block *sb)
{ return NULL; }

static inline struct inode *
kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn)
{ return NULL; }

static inline struct kernfs_root *
kernfs_create_root(struct kernfs_syscall_ops *scops, unsigned int flags,
                   void *priv)
{ return ERR_PTR(-ENOSYS); }

static inline void kernfs_destroy_root(struct kernfs_root *root) { }

static inline struct kernfs_node *
kernfs_create_dir_ns(struct kernfs_node *parent, const char *name,
                     umode_t mode, kuid_t uid, kgid_t gid,
                     void *priv, const void *ns)
{ return ERR_PTR(-ENOSYS); }

static inline struct kernfs_node *
__kernfs_create_file(struct kernfs_node *parent, const char *name,
                     umode_t mode, kuid_t uid, kgid_t gid,
                     loff_t size, const struct kernfs_ops *ops,
                     void *priv, const void *ns, struct lock_class_key *key)
{ return ERR_PTR(-ENOSYS); }

static inline struct kernfs_node *
kernfs_create_link(struct kernfs_node *parent, const char *name,
                   struct kernfs_node *target)
{ return ERR_PTR(-ENOSYS); }

static inline void kernfs_activate(struct kernfs_node *kn) { }

static inline void kernfs_remove(struct kernfs_node *kn) { }

static inline bool kernfs_remove_self(struct kernfs_node *kn)
{ return false; }

static inline int kernfs_remove_by_name_ns(struct kernfs_node *kn,
                                           const char *name, const void *ns)
{ return -ENOSYS; }

static inline int kernfs_rename_ns(struct kernfs_node *kn,
                                   struct kernfs_node *new_parent,
                                   const char *new_name, const void *new_ns)
{ return -ENOSYS; }

static inline int kernfs_setattr(struct kernfs_node *kn,
                                 const struct iattr *iattr)
{ return -ENOSYS; }

static inline __poll_t kernfs_generic_poll(struct kernfs_open_file *of,
                                           struct poll_table_struct *pt)
{ return -ENOSYS; }

static inline void kernfs_notify(struct kernfs_node *kn) { }

static inline int kernfs_xattr_get(struct kernfs_node *kn, const char *name,
                                   void *value, size_t size)
{ return -ENOSYS; }

static inline int kernfs_xattr_set(struct kernfs_node *kn, const char *name,
                                   const void *value, size_t size, int flags)
{ return -ENOSYS; }

static inline const void *kernfs_super_ns(struct super_block *sb)
{ return NULL; }

static inline int kernfs_get_tree(struct fs_context *fc)
{ return -ENOSYS; }

static inline void kernfs_free_fs_context(struct fs_context *fc) { }

static inline void kernfs_kill_sb(struct super_block *sb) { }

static inline void kernfs_init(void) { }

#endif        /* CONFIG_KERNFS */

/**
 * kernfs_path - build full path of a given node
 * @kn: kernfs_node of interest
 * @buf: buffer to copy @kn's name into
 * @buflen: size of @buf
 *
 * If @kn is NULL result will be "(null)".
 *
 * Returns the length of the full path.  If the full length is equal to or
 * greater than @buflen, @buf contains the truncated path with the trailing
 * '\0'.  On error, -errno is returned.
 */
static inline int kernfs_path(struct kernfs_node *kn, char *buf, size_t buflen)
{
        return kernfs_path_from_node(kn, NULL, buf, buflen);
}

static inline struct kernfs_node *
kernfs_find_and_get(struct kernfs_node *kn, const char *name)
{
        return kernfs_find_and_get_ns(kn, name, NULL);
}

static inline struct kernfs_node *
kernfs_walk_and_get(struct kernfs_node *kn, const char *path)
{
        return kernfs_walk_and_get_ns(kn, path, NULL);
}

static inline struct kernfs_node *
kernfs_create_dir(struct kernfs_node *parent, const char *name, umode_t mode,
                  void *priv)
{
        return kernfs_create_dir_ns(parent, name, mode,
                                    GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
                                    priv, NULL);
}

static inline int kernfs_remove_by_name(struct kernfs_node *parent,
                                        const char *name)
{
        return kernfs_remove_by_name_ns(parent, name, NULL);
}

static inline int kernfs_rename(struct kernfs_node *kn,
                                struct kernfs_node *new_parent,
                                const char *new_name)
{
        return kernfs_rename_ns(kn, new_parent, new_name, NULL);
}

#endif        /* __LINUX_KERNFS_H */





























































































































































































    4 




    4 






















    4 
    2 











    5 


    4 









    5 














    3 






    2 









    1 












    2 




   11 










    2 





    5 






































    6 






    2 
    4 


    2 
    1 
    3 
    4 


    5 


    1 


    5 
    1 


    6 






    3 
    3 









    1 





    5 












    2 

    4 







    3 

    3 




    4 

























    4 



    3 
    1 








































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * USB Synaptics device driver
 *
 *  Copyright (c) 2002 Rob Miller (rob@inpharmatica . co . uk)
 *  Copyright (c) 2003 Ron Lee (ron@debian.org)
 *        cPad driver for kernel 2.4
 *
 *  Copyright (c) 2004 Jan Steinhoff (cpad@jan-steinhoff . de)
 *  Copyright (c) 2004 Ron Lee (ron@debian.org)
 *        rewritten for kernel 2.6
 *
 *  cPad display character device part is not included. It can be found at
 *  http://jan-steinhoff.de/linux/synaptics-usb.html
 *
 * Bases on:        usb_skeleton.c v2.2 by Greg Kroah-Hartman
 *                drivers/hid/usbhid/usbmouse.c by Vojtech Pavlik
 *                drivers/input/mouse/synaptics.c by Peter Osterlund
 *
 * Trademarks are the property of their respective owners.
 */

/*
 * There are three different types of Synaptics USB devices: Touchpads,
 * touchsticks (or trackpoints), and touchscreens. Touchpads are well supported
 * by this driver, touchstick support has not been tested much yet, and
 * touchscreens have not been tested at all.
 *
 * Up to three alternate settings are possible:
 *        setting 0: one int endpoint for relative movement (used by usbhid.ko)
 *        setting 1: one int endpoint for absolute finger position
 *        setting 2 (cPad only): one int endpoint for absolute finger position and
 *                   two bulk endpoints for the display (in/out)
 * This driver uses setting 1.
 */

#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/usb.h>
#include <linux/input.h>
#include <linux/usb/input.h>

#define USB_VENDOR_ID_SYNAPTICS        0x06cb
#define USB_DEVICE_ID_SYNAPTICS_TP        0x0001        /* Synaptics USB TouchPad */
#define USB_DEVICE_ID_SYNAPTICS_INT_TP        0x0002        /* Integrated USB TouchPad */
#define USB_DEVICE_ID_SYNAPTICS_CPAD        0x0003        /* Synaptics cPad */
#define USB_DEVICE_ID_SYNAPTICS_TS        0x0006        /* Synaptics TouchScreen */
#define USB_DEVICE_ID_SYNAPTICS_STICK        0x0007        /* Synaptics USB Styk */
#define USB_DEVICE_ID_SYNAPTICS_WP        0x0008        /* Synaptics USB WheelPad */
#define USB_DEVICE_ID_SYNAPTICS_COMP_TP        0x0009        /* Composite USB TouchPad */
#define USB_DEVICE_ID_SYNAPTICS_WTP        0x0010        /* Wireless TouchPad */
#define USB_DEVICE_ID_SYNAPTICS_DPAD        0x0013        /* DisplayPad */

#define SYNUSB_TOUCHPAD                        (1 << 0)
#define SYNUSB_STICK                        (1 << 1)
#define SYNUSB_TOUCHSCREEN                (1 << 2)
#define SYNUSB_AUXDISPLAY                (1 << 3) /* For cPad */
#define SYNUSB_COMBO                        (1 << 4) /* Composite device (TP + stick) */
#define SYNUSB_IO_ALWAYS                (1 << 5)

#define USB_DEVICE_SYNAPTICS(prod, kind)                \
        USB_DEVICE(USB_VENDOR_ID_SYNAPTICS,                \
                   USB_DEVICE_ID_SYNAPTICS_##prod),        \
        .driver_info = (kind),

#define SYNUSB_RECV_SIZE        8

#define XMIN_NOMINAL                1472
#define XMAX_NOMINAL                5472
#define YMIN_NOMINAL                1408
#define YMAX_NOMINAL                4448

struct synusb {
        struct usb_device *udev;
        struct usb_interface *intf;
        struct urb *urb;
        unsigned char *data;

        /* serialize access to open/suspend */
        struct mutex pm_mutex;
        bool is_open;

        /* input device related data structures */
        struct input_dev *input;
        char name[128];
        char phys[64];

        /* characteristics of the device */
        unsigned long flags;
};

static void synusb_report_buttons(struct synusb *synusb)
{
        struct input_dev *input_dev = synusb->input;

        input_report_key(input_dev, BTN_LEFT, synusb->data[1] & 0x04);
        input_report_key(input_dev, BTN_RIGHT, synusb->data[1] & 0x01);
        input_report_key(input_dev, BTN_MIDDLE, synusb->data[1] & 0x02);
}

static void synusb_report_stick(struct synusb *synusb)
{
        struct input_dev *input_dev = synusb->input;
        int x, y;
        unsigned int pressure;

        pressure = synusb->data[6];
        x = (s16)(be16_to_cpup((__be16 *)&synusb->data[2]) << 3) >> 7;
        y = (s16)(be16_to_cpup((__be16 *)&synusb->data[4]) << 3) >> 7;

        if (pressure > 0) {
                input_report_rel(input_dev, REL_X, x);
                input_report_rel(input_dev, REL_Y, -y);
        }

        input_report_abs(input_dev, ABS_PRESSURE, pressure);

        synusb_report_buttons(synusb);

        input_sync(input_dev);
}

static void synusb_report_touchpad(struct synusb *synusb)
{
        struct input_dev *input_dev = synusb->input;
        unsigned int num_fingers, tool_width;
        unsigned int x, y;
        unsigned int pressure, w;

        pressure = synusb->data[6];
        x = be16_to_cpup((__be16 *)&synusb->data[2]);
        y = be16_to_cpup((__be16 *)&synusb->data[4]);
        w = synusb->data[0] & 0x0f;

        if (pressure > 0) {
                num_fingers = 1;
                tool_width = 5;
                switch (w) {
                case 0 ... 1:
                        num_fingers = 2 + w;
                        break;

                case 2:                        /* pen, pretend its a finger */
                        break;

                case 4 ... 15:
                        tool_width = w;
                        break;
                }
        } else {
                num_fingers = 0;
                tool_width = 0;
        }

        /*
         * Post events
         * BTN_TOUCH has to be first as mousedev relies on it when doing
         * absolute -> relative conversion
         */

        if (pressure > 30)
                input_report_key(input_dev, BTN_TOUCH, 1);
        if (pressure < 25)
                input_report_key(input_dev, BTN_TOUCH, 0);

        if (num_fingers > 0) {
                input_report_abs(input_dev, ABS_X, x);
                input_report_abs(input_dev, ABS_Y,
                                 YMAX_NOMINAL + YMIN_NOMINAL - y);
        }

        input_report_abs(input_dev, ABS_PRESSURE, pressure);
        input_report_abs(input_dev, ABS_TOOL_WIDTH, tool_width);

        input_report_key(input_dev, BTN_TOOL_FINGER, num_fingers == 1);
        input_report_key(input_dev, BTN_TOOL_DOUBLETAP, num_fingers == 2);
        input_report_key(input_dev, BTN_TOOL_TRIPLETAP, num_fingers == 3);

        synusb_report_buttons(synusb);
        if (synusb->flags & SYNUSB_AUXDISPLAY)
                input_report_key(input_dev, BTN_MIDDLE, synusb->data[1] & 0x08);

        input_sync(input_dev);
}

static void synusb_irq(struct urb *urb)
{
        struct synusb *synusb = urb->context;
        int error;

        /* Check our status in case we need to bail out early. */
        switch (urb->status) {
        case 0:
                usb_mark_last_busy(synusb->udev);
                break;

        /* Device went away so don't keep trying to read from it. */
        case -ECONNRESET:
        case -ENOENT:
        case -ESHUTDOWN:
                return;

        default:
                goto resubmit;
                break;
        }

        if (synusb->flags & SYNUSB_STICK)
                synusb_report_stick(synusb);
        else
                synusb_report_touchpad(synusb);

resubmit:
        error = usb_submit_urb(urb, GFP_ATOMIC);
        if (error && error != -EPERM)
                dev_err(&synusb->intf->dev,
                        "%s - usb_submit_urb failed with result: %d",
                        __func__, error);
}

static struct usb_endpoint_descriptor *
synusb_get_in_endpoint(struct usb_host_interface *iface)
{

        struct usb_endpoint_descriptor *endpoint;
        int i;

        for (i = 0; i < iface->desc.bNumEndpoints; ++i) {
                endpoint = &iface->endpoint[i].desc;

                if (usb_endpoint_is_int_in(endpoint)) {
                        /* we found our interrupt in endpoint */
                        return endpoint;
                }
        }

        return NULL;
}

static int synusb_open(struct input_dev *dev)
{
        struct synusb *synusb = input_get_drvdata(dev);
        int retval;

        retval = usb_autopm_get_interface(synusb->intf);
        if (retval) {
                dev_err(&synusb->intf->dev,
                        "%s - usb_autopm_get_interface failed, error: %d\n",
                        __func__, retval);
                return retval;
        }

        mutex_lock(&synusb->pm_mutex);
        retval = usb_submit_urb(synusb->urb, GFP_KERNEL);
        if (retval) {
                dev_err(&synusb->intf->dev,
                        "%s - usb_submit_urb failed, error: %d\n",
                        __func__, retval);
                retval = -EIO;
                goto out;
        }

        synusb->intf->needs_remote_wakeup = 1;
        synusb->is_open = true;

out:
        mutex_unlock(&synusb->pm_mutex);
        usb_autopm_put_interface(synusb->intf);
        return retval;
}

static void synusb_close(struct input_dev *dev)
{
        struct synusb *synusb = input_get_drvdata(dev);
        int autopm_error;

        autopm_error = usb_autopm_get_interface(synusb->intf);

        mutex_lock(&synusb->pm_mutex);
        usb_kill_urb(synusb->urb);
        synusb->intf->needs_remote_wakeup = 0;
        synusb->is_open = false;
        mutex_unlock(&synusb->pm_mutex);

        if (!autopm_error)
                usb_autopm_put_interface(synusb->intf);
}

static int synusb_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        struct usb_device *udev = interface_to_usbdev(intf);
        struct usb_endpoint_descriptor *ep;
        struct synusb *synusb;
        struct input_dev *input_dev;
        unsigned int intf_num = intf->cur_altsetting->desc.bInterfaceNumber;
        unsigned int altsetting = min(intf->num_altsetting, 1U);
        int error;

        error = usb_set_interface(udev, intf_num, altsetting);
        if (error) {
                dev_err(&udev->dev,
                        "Can not set alternate setting to %i, error: %i",
                        altsetting, error);
                return error;
        }

        ep = synusb_get_in_endpoint(intf->cur_altsetting);
        if (!ep)
                return -ENODEV;

        synusb = kzalloc(sizeof(*synusb), GFP_KERNEL);
        input_dev = input_allocate_device();
        if (!synusb || !input_dev) {
                error = -ENOMEM;
                goto err_free_mem;
        }

        synusb->udev = udev;
        synusb->intf = intf;
        synusb->input = input_dev;
        mutex_init(&synusb->pm_mutex);

        synusb->flags = id->driver_info;
        if (synusb->flags & SYNUSB_COMBO) {
                /*
                 * This is a combo device, we need to set proper
                 * capability, depending on the interface.
                 */
                synusb->flags |= intf_num == 1 ?
                                        SYNUSB_STICK : SYNUSB_TOUCHPAD;
        }

        synusb->urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!synusb->urb) {
                error = -ENOMEM;
                goto err_free_mem;
        }

        synusb->data = usb_alloc_coherent(udev, SYNUSB_RECV_SIZE, GFP_KERNEL,
                                          &synusb->urb->transfer_dma);
        if (!synusb->data) {
                error = -ENOMEM;
                goto err_free_urb;
        }

        usb_fill_int_urb(synusb->urb, udev,
                         usb_rcvintpipe(udev, ep->bEndpointAddress),
                         synusb->data, SYNUSB_RECV_SIZE,
                         synusb_irq, synusb,
                         ep->bInterval);
        synusb->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;

        if (udev->manufacturer)
                strscpy(synusb->name, udev->manufacturer,
                        sizeof(synusb->name));

        if (udev->product) {
                if (udev->manufacturer)
                        strlcat(synusb->name, " ", sizeof(synusb->name));
                strlcat(synusb->name, udev->product, sizeof(synusb->name));
        }

        if (!strlen(synusb->name))
                snprintf(synusb->name, sizeof(synusb->name),
                         "USB Synaptics Device %04x:%04x",
                         le16_to_cpu(udev->descriptor.idVendor),
                         le16_to_cpu(udev->descriptor.idProduct));

        if (synusb->flags & SYNUSB_STICK)
                strlcat(synusb->name, " (Stick)", sizeof(synusb->name));

        usb_make_path(udev, synusb->phys, sizeof(synusb->phys));
        strlcat(synusb->phys, "/input0", sizeof(synusb->phys));

        input_dev->name = synusb->name;
        input_dev->phys = synusb->phys;
        usb_to_input_id(udev, &input_dev->id);
        input_dev->dev.parent = &synusb->intf->dev;

        if (!(synusb->flags & SYNUSB_IO_ALWAYS)) {
                input_dev->open = synusb_open;
                input_dev->close = synusb_close;
        }

        input_set_drvdata(input_dev, synusb);

        __set_bit(EV_ABS, input_dev->evbit);
        __set_bit(EV_KEY, input_dev->evbit);

        if (synusb->flags & SYNUSB_STICK) {
                __set_bit(EV_REL, input_dev->evbit);
                __set_bit(REL_X, input_dev->relbit);
                __set_bit(REL_Y, input_dev->relbit);
                __set_bit(INPUT_PROP_POINTING_STICK, input_dev->propbit);
                input_set_abs_params(input_dev, ABS_PRESSURE, 0, 127, 0, 0);
        } else {
                input_set_abs_params(input_dev, ABS_X,
                                     XMIN_NOMINAL, XMAX_NOMINAL, 0, 0);
                input_set_abs_params(input_dev, ABS_Y,
                                     YMIN_NOMINAL, YMAX_NOMINAL, 0, 0);
                input_set_abs_params(input_dev, ABS_PRESSURE, 0, 255, 0, 0);
                input_set_abs_params(input_dev, ABS_TOOL_WIDTH, 0, 15, 0, 0);
                __set_bit(BTN_TOUCH, input_dev->keybit);
                __set_bit(BTN_TOOL_FINGER, input_dev->keybit);
                __set_bit(BTN_TOOL_DOUBLETAP, input_dev->keybit);
                __set_bit(BTN_TOOL_TRIPLETAP, input_dev->keybit);
        }

        if (synusb->flags & SYNUSB_TOUCHSCREEN)
                __set_bit(INPUT_PROP_DIRECT, input_dev->propbit);
        else
                __set_bit(INPUT_PROP_POINTER, input_dev->propbit);

        __set_bit(BTN_LEFT, input_dev->keybit);
        __set_bit(BTN_RIGHT, input_dev->keybit);
        __set_bit(BTN_MIDDLE, input_dev->keybit);

        usb_set_intfdata(intf, synusb);

        if (synusb->flags & SYNUSB_IO_ALWAYS) {
                error = synusb_open(input_dev);
                if (error)
                        goto err_free_dma;
        }

        error = input_register_device(input_dev);
        if (error) {
                dev_err(&udev->dev,
                        "Failed to register input device, error %d\n",
                        error);
                goto err_stop_io;
        }

        return 0;

err_stop_io:
        if (synusb->flags & SYNUSB_IO_ALWAYS)
                synusb_close(synusb->input);
err_free_dma:
        usb_free_coherent(udev, SYNUSB_RECV_SIZE, synusb->data,
                          synusb->urb->transfer_dma);
err_free_urb:
        usb_free_urb(synusb->urb);
err_free_mem:
        input_free_device(input_dev);
        kfree(synusb);
        usb_set_intfdata(intf, NULL);

        return error;
}

static void synusb_disconnect(struct usb_interface *intf)
{
        struct synusb *synusb = usb_get_intfdata(intf);
        struct usb_device *udev = interface_to_usbdev(intf);

        if (synusb->flags & SYNUSB_IO_ALWAYS)
                synusb_close(synusb->input);

        input_unregister_device(synusb->input);

        usb_free_coherent(udev, SYNUSB_RECV_SIZE, synusb->data,
                          synusb->urb->transfer_dma);
        usb_free_urb(synusb->urb);
        kfree(synusb);

        usb_set_intfdata(intf, NULL);
}

static int synusb_suspend(struct usb_interface *intf, pm_message_t message)
{
        struct synusb *synusb = usb_get_intfdata(intf);

        mutex_lock(&synusb->pm_mutex);
        usb_kill_urb(synusb->urb);
        mutex_unlock(&synusb->pm_mutex);

        return 0;
}

static int synusb_resume(struct usb_interface *intf)
{
        struct synusb *synusb = usb_get_intfdata(intf);
        int retval = 0;

        mutex_lock(&synusb->pm_mutex);

        if ((synusb->is_open || (synusb->flags & SYNUSB_IO_ALWAYS)) &&
            usb_submit_urb(synusb->urb, GFP_NOIO) < 0) {
                retval = -EIO;
        }

        mutex_unlock(&synusb->pm_mutex);

        return retval;
}

static int synusb_pre_reset(struct usb_interface *intf)
{
        struct synusb *synusb = usb_get_intfdata(intf);

        mutex_lock(&synusb->pm_mutex);
        usb_kill_urb(synusb->urb);

        return 0;
}

static int synusb_post_reset(struct usb_interface *intf)
{
        struct synusb *synusb = usb_get_intfdata(intf);
        int retval = 0;

        if ((synusb->is_open || (synusb->flags & SYNUSB_IO_ALWAYS)) &&
            usb_submit_urb(synusb->urb, GFP_NOIO) < 0) {
                retval = -EIO;
        }

        mutex_unlock(&synusb->pm_mutex);

        return retval;
}

static int synusb_reset_resume(struct usb_interface *intf)
{
        return synusb_resume(intf);
}

static const struct usb_device_id synusb_idtable[] = {
        { USB_DEVICE_SYNAPTICS(TP, SYNUSB_TOUCHPAD) },
        { USB_DEVICE_SYNAPTICS(INT_TP, SYNUSB_TOUCHPAD) },
        { USB_DEVICE_SYNAPTICS(CPAD,
                SYNUSB_TOUCHPAD | SYNUSB_AUXDISPLAY | SYNUSB_IO_ALWAYS) },
        { USB_DEVICE_SYNAPTICS(TS, SYNUSB_TOUCHSCREEN) },
        { USB_DEVICE_SYNAPTICS(STICK, SYNUSB_STICK) },
        { USB_DEVICE_SYNAPTICS(WP, SYNUSB_TOUCHPAD) },
        { USB_DEVICE_SYNAPTICS(COMP_TP, SYNUSB_COMBO) },
        { USB_DEVICE_SYNAPTICS(WTP, SYNUSB_TOUCHPAD) },
        { USB_DEVICE_SYNAPTICS(DPAD, SYNUSB_TOUCHPAD) },
        { }
};
MODULE_DEVICE_TABLE(usb, synusb_idtable);

static struct usb_driver synusb_driver = {
        .name                = "synaptics_usb",
        .probe                = synusb_probe,
        .disconnect        = synusb_disconnect,
        .id_table        = synusb_idtable,
        .suspend        = synusb_suspend,
        .resume                = synusb_resume,
        .pre_reset        = synusb_pre_reset,
        .post_reset        = synusb_post_reset,
        .reset_resume        = synusb_reset_resume,
        .supports_autosuspend = 1,
};

module_usb_driver(synusb_driver);

MODULE_AUTHOR("Rob Miller <rob@inpharmatica.co.uk>, "
              "Ron Lee <ron@debian.org>, "
              "Jan Steinhoff <cpad@jan-steinhoff.de>");
MODULE_DESCRIPTION("Synaptics USB device driver");
MODULE_LICENSE("GPL");














    5 






  476 
  478 








    5 






    5 






    5 










   86 







   86 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
// SPDX-License-Identifier: GPL-2.0
#include <linux/rtnetlink.h>
#include <linux/notifier.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <net/net_namespace.h>
#include <net/fib_notifier.h>
#include <net/ip_fib.h>

int call_fib4_notifier(struct notifier_block *nb,
                       enum fib_event_type event_type,
                       struct fib_notifier_info *info)
{
        info->family = AF_INET;
        return call_fib_notifier(nb, event_type, info);
}

int call_fib4_notifiers(struct net *net, enum fib_event_type event_type,
                        struct fib_notifier_info *info)
{
        ASSERT_RTNL();

        info->family = AF_INET;
        /* Paired with READ_ONCE() in fib4_seq_read() */
        WRITE_ONCE(net->ipv4.fib_seq, net->ipv4.fib_seq + 1);
        return call_fib_notifiers(net, event_type, info);
}

static unsigned int fib4_seq_read(const struct net *net)
{
        /* Paired with WRITE_ONCE() in call_fib4_notifiers() */
        return READ_ONCE(net->ipv4.fib_seq) + fib4_rules_seq_read(net);
}

static int fib4_dump(struct net *net, struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        int err;

        err = fib4_rules_dump(net, nb, extack);
        if (err)
                return err;

        return fib_notify(net, nb, extack);
}

static const struct fib_notifier_ops fib4_notifier_ops_template = {
        .family                = AF_INET,
        .fib_seq_read        = fib4_seq_read,
        .fib_dump        = fib4_dump,
        .owner                = THIS_MODULE,
};

int __net_init fib4_notifier_init(struct net *net)
{
        struct fib_notifier_ops *ops;

        net->ipv4.fib_seq = 0;

        ops = fib_notifier_ops_register(&fib4_notifier_ops_template, net);
        if (IS_ERR(ops))
                return PTR_ERR(ops);
        net->ipv4.notifier_ops = ops;

        return 0;
}

void __net_exit fib4_notifier_exit(struct net *net)
{
        fib_notifier_ops_unregister(net->ipv4.notifier_ops);
}




































































































































































































































































































   76 












   78 









   78 








   78 



   78 

   78 




















































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
// SPDX-License-Identifier: GPL-2.0-only
/*
 * HT handling
 *
 * Copyright 2003, Jouni Malinen <jkmaline@cc.hut.fi>
 * Copyright 2002-2005, Instant802 Networks, Inc.
 * Copyright 2005-2006, Devicescape Software, Inc.
 * Copyright 2006-2007        Jiri Benc <jbenc@suse.cz>
 * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
 * Copyright 2007-2010, Intel Corporation
 * Copyright(c) 2015-2017 Intel Deutschland GmbH
 * Copyright (C) 2018 - 2023 Intel Corporation
 */

#include <linux/ieee80211.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "driver-ops.h"
#include "wme.h"

/**
 * DOC: TX A-MPDU aggregation
 *
 * Aggregation on the TX side requires setting the hardware flag
 * %IEEE80211_HW_AMPDU_AGGREGATION. The driver will then be handed
 * packets with a flag indicating A-MPDU aggregation. The driver
 * or device is responsible for actually aggregating the frames,
 * as well as deciding how many and which to aggregate.
 *
 * When TX aggregation is started by some subsystem (usually the rate
 * control algorithm would be appropriate) by calling the
 * ieee80211_start_tx_ba_session() function, the driver will be
 * notified via its @ampdu_action function, with the
 * %IEEE80211_AMPDU_TX_START action.
 *
 * In response to that, the driver is later required to call the
 * ieee80211_start_tx_ba_cb_irqsafe() function, which will really
 * start the aggregation session after the peer has also responded.
 * If the peer responds negatively, the session will be stopped
 * again right away. Note that it is possible for the aggregation
 * session to be stopped before the driver has indicated that it
 * is done setting it up, in which case it must not indicate the
 * setup completion.
 *
 * Also note that, since we also need to wait for a response from
 * the peer, the driver is notified of the completion of the
 * handshake by the %IEEE80211_AMPDU_TX_OPERATIONAL action to the
 * @ampdu_action callback.
 *
 * Similarly, when the aggregation session is stopped by the peer
 * or something calling ieee80211_stop_tx_ba_session(), the driver's
 * @ampdu_action function will be called with the action
 * %IEEE80211_AMPDU_TX_STOP. In this case, the call must not fail,
 * and the driver must later call ieee80211_stop_tx_ba_cb_irqsafe().
 * Note that the sta can get destroyed before the BA tear down is
 * complete.
 */

static void ieee80211_send_addba_request(struct sta_info *sta, u16 tid,
                                         u8 dialog_token, u16 start_seq_num,
                                         u16 agg_size, u16 timeout)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        u16 capab;

        skb = dev_alloc_skb(sizeof(*mgmt) +
                            2 + sizeof(struct ieee80211_addba_ext_ie) +
                            local->hw.extra_tx_headroom);
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom);
        mgmt = ieee80211_mgmt_ba(skb, sta->sta.addr, sdata);

        skb_put(skb, 1 + sizeof(mgmt->u.action.u.addba_req));

        mgmt->u.action.category = WLAN_CATEGORY_BACK;
        mgmt->u.action.u.addba_req.action_code = WLAN_ACTION_ADDBA_REQ;

        mgmt->u.action.u.addba_req.dialog_token = dialog_token;
        capab = IEEE80211_ADDBA_PARAM_AMSDU_MASK;
        capab |= IEEE80211_ADDBA_PARAM_POLICY_MASK;
        capab |= u16_encode_bits(tid, IEEE80211_ADDBA_PARAM_TID_MASK);
        capab |= u16_encode_bits(agg_size, IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK);

        mgmt->u.action.u.addba_req.capab = cpu_to_le16(capab);

        mgmt->u.action.u.addba_req.timeout = cpu_to_le16(timeout);
        mgmt->u.action.u.addba_req.start_seq_num =
                                        cpu_to_le16(start_seq_num << 4);

        if (sta->sta.deflink.he_cap.has_he)
                ieee80211_add_addbaext(skb, 0, agg_size);

        ieee80211_tx_skb_tid(sdata, skb, tid, -1);
}

void ieee80211_send_bar(struct ieee80211_vif *vif, u8 *ra, u16 tid, u16 ssn)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_local *local = sdata->local;
        struct sk_buff *skb;
        struct ieee80211_bar *bar;
        u16 bar_control = 0;

        skb = dev_alloc_skb(sizeof(*bar) + local->hw.extra_tx_headroom);
        if (!skb)
                return;

        skb_reserve(skb, local->hw.extra_tx_headroom);
        bar = skb_put_zero(skb, sizeof(*bar));
        bar->frame_control = cpu_to_le16(IEEE80211_FTYPE_CTL |
                                         IEEE80211_STYPE_BACK_REQ);
        memcpy(bar->ra, ra, ETH_ALEN);
        memcpy(bar->ta, sdata->vif.addr, ETH_ALEN);
        bar_control |= (u16)IEEE80211_BAR_CTRL_ACK_POLICY_NORMAL;
        bar_control |= (u16)IEEE80211_BAR_CTRL_CBMTID_COMPRESSED_BA;
        bar_control |= (u16)(tid << IEEE80211_BAR_CTRL_TID_INFO_SHIFT);
        bar->control = cpu_to_le16(bar_control);
        bar->start_seq_num = cpu_to_le16(ssn);

        IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT |
                                        IEEE80211_TX_CTL_REQ_TX_STATUS;
        ieee80211_tx_skb_tid(sdata, skb, tid, -1);
}
EXPORT_SYMBOL(ieee80211_send_bar);

void ieee80211_assign_tid_tx(struct sta_info *sta, int tid,
                             struct tid_ampdu_tx *tid_tx)
{
        lockdep_assert_wiphy(sta->local->hw.wiphy);
        lockdep_assert_held(&sta->lock);
        rcu_assign_pointer(sta->ampdu_mlme.tid_tx[tid], tid_tx);
}

/*
 * When multiple aggregation sessions on multiple stations
 * are being created/destroyed simultaneously, we need to
 * refcount the global queue stop caused by that in order
 * to not get into a situation where one of the aggregation
 * setup or teardown re-enables queues before the other is
 * ready to handle that.
 *
 * These two functions take care of this issue by keeping
 * a global "agg_queue_stop" refcount.
 */
static void __acquires(agg_queue)
ieee80211_stop_queue_agg(struct ieee80211_sub_if_data *sdata, int tid)
{
        int queue = sdata->vif.hw_queue[ieee80211_ac_from_tid(tid)];

        /* we do refcounting here, so don't use the queue reason refcounting */

        if (atomic_inc_return(&sdata->local->agg_queue_stop[queue]) == 1)
                ieee80211_stop_queue_by_reason(
                        &sdata->local->hw, queue,
                        IEEE80211_QUEUE_STOP_REASON_AGGREGATION,
                        false);
        __acquire(agg_queue);
}

static void __releases(agg_queue)
ieee80211_wake_queue_agg(struct ieee80211_sub_if_data *sdata, int tid)
{
        int queue = sdata->vif.hw_queue[ieee80211_ac_from_tid(tid)];

        if (atomic_dec_return(&sdata->local->agg_queue_stop[queue]) == 0)
                ieee80211_wake_queue_by_reason(
                        &sdata->local->hw, queue,
                        IEEE80211_QUEUE_STOP_REASON_AGGREGATION,
                        false);
        __release(agg_queue);
}

static void
ieee80211_agg_stop_txq(struct sta_info *sta, int tid)
{
        struct ieee80211_txq *txq = sta->sta.txq[tid];
        struct ieee80211_sub_if_data *sdata;
        struct fq *fq;
        struct txq_info *txqi;

        if (!txq)
                return;

        txqi = to_txq_info(txq);
        sdata = vif_to_sdata(txq->vif);
        fq = &sdata->local->fq;

        /* Lock here to protect against further seqno updates on dequeue */
        spin_lock_bh(&fq->lock);
        set_bit(IEEE80211_TXQ_STOP, &txqi->flags);
        spin_unlock_bh(&fq->lock);
}

static void
ieee80211_agg_start_txq(struct sta_info *sta, int tid, bool enable)
{
        struct ieee80211_txq *txq = sta->sta.txq[tid];
        struct txq_info *txqi;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        if (!txq)
                return;

        txqi = to_txq_info(txq);

        if (enable)
                set_bit(IEEE80211_TXQ_AMPDU, &txqi->flags);
        else
                clear_bit(IEEE80211_TXQ_AMPDU, &txqi->flags);

        clear_bit(IEEE80211_TXQ_STOP, &txqi->flags);
        local_bh_disable();
        rcu_read_lock();
        schedule_and_wake_txq(sta->sdata->local, txqi);
        rcu_read_unlock();
        local_bh_enable();
}

/*
 * splice packets from the STA's pending to the local pending,
 * requires a call to ieee80211_agg_splice_finish later
 */
static void __acquires(agg_queue)
ieee80211_agg_splice_packets(struct ieee80211_sub_if_data *sdata,
                             struct tid_ampdu_tx *tid_tx, u16 tid)
{
        struct ieee80211_local *local = sdata->local;
        int queue = sdata->vif.hw_queue[ieee80211_ac_from_tid(tid)];
        unsigned long flags;

        ieee80211_stop_queue_agg(sdata, tid);

        if (WARN(!tid_tx,
                 "TID %d gone but expected when splicing aggregates from the pending queue\n",
                 tid))
                return;

        if (!skb_queue_empty(&tid_tx->pending)) {
                spin_lock_irqsave(&local->queue_stop_reason_lock, flags);
                /* copy over remaining packets */
                skb_queue_splice_tail_init(&tid_tx->pending,
                                           &local->pending[queue]);
                spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags);
        }
}

static void __releases(agg_queue)
ieee80211_agg_splice_finish(struct ieee80211_sub_if_data *sdata, u16 tid)
{
        ieee80211_wake_queue_agg(sdata, tid);
}

static void ieee80211_remove_tid_tx(struct sta_info *sta, int tid)
{
        struct tid_ampdu_tx *tid_tx;

        lockdep_assert_wiphy(sta->local->hw.wiphy);
        lockdep_assert_held(&sta->lock);

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);

        /*
         * When we get here, the TX path will not be lockless any more wrt.
         * aggregation, since the OPERATIONAL bit has long been cleared.
         * Thus it will block on getting the lock, if it occurs. So if we
         * stop the queue now, we will not get any more packets, and any
         * that might be being processed will wait for us here, thereby
         * guaranteeing that no packets go to the tid_tx pending queue any
         * more.
         */

        ieee80211_agg_splice_packets(sta->sdata, tid_tx, tid);

        /* future packets must not find the tid_tx struct any more */
        ieee80211_assign_tid_tx(sta, tid, NULL);

        ieee80211_agg_splice_finish(sta->sdata, tid);

        kfree_rcu(tid_tx, rcu_head);
}

int __ieee80211_stop_tx_ba_session(struct sta_info *sta, u16 tid,
                                   enum ieee80211_agg_stop_reason reason)
{
        struct ieee80211_local *local = sta->local;
        struct tid_ampdu_tx *tid_tx;
        struct ieee80211_ampdu_params params = {
                .sta = &sta->sta,
                .tid = tid,
                .buf_size = 0,
                .amsdu = false,
                .timeout = 0,
                .ssn = 0,
        };
        int ret;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        switch (reason) {
        case AGG_STOP_DECLINED:
        case AGG_STOP_LOCAL_REQUEST:
        case AGG_STOP_PEER_REQUEST:
                params.action = IEEE80211_AMPDU_TX_STOP_CONT;
                break;
        case AGG_STOP_DESTROY_STA:
                params.action = IEEE80211_AMPDU_TX_STOP_FLUSH;
                break;
        default:
                WARN_ON_ONCE(1);
                return -EINVAL;
        }

        spin_lock_bh(&sta->lock);

        /* free struct pending for start, if present */
        tid_tx = sta->ampdu_mlme.tid_start_tx[tid];
        kfree(tid_tx);
        sta->ampdu_mlme.tid_start_tx[tid] = NULL;

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);
        if (!tid_tx) {
                spin_unlock_bh(&sta->lock);
                return -ENOENT;
        }

        /*
         * if we're already stopping ignore any new requests to stop
         * unless we're destroying it in which case notify the driver
         */
        if (test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state)) {
                spin_unlock_bh(&sta->lock);
                if (reason != AGG_STOP_DESTROY_STA)
                        return -EALREADY;
                params.action = IEEE80211_AMPDU_TX_STOP_FLUSH_CONT;
                ret = drv_ampdu_action(local, sta->sdata, &params);
                WARN_ON_ONCE(ret);
                return 0;
        }

        if (test_bit(HT_AGG_STATE_WANT_START, &tid_tx->state)) {
                /* not even started yet! */
                ieee80211_assign_tid_tx(sta, tid, NULL);
                spin_unlock_bh(&sta->lock);
                kfree_rcu(tid_tx, rcu_head);
                return 0;
        }

        set_bit(HT_AGG_STATE_STOPPING, &tid_tx->state);

        ieee80211_agg_stop_txq(sta, tid);

        spin_unlock_bh(&sta->lock);

        ht_dbg(sta->sdata, "Tx BA session stop requested for %pM tid %u\n",
               sta->sta.addr, tid);

        del_timer_sync(&tid_tx->addba_resp_timer);
        del_timer_sync(&tid_tx->session_timer);

        /*
         * After this packets are no longer handed right through
         * to the driver but are put onto tid_tx->pending instead,
         * with locking to ensure proper access.
         */
        clear_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state);

        /*
         * There might be a few packets being processed right now (on
         * another CPU) that have already gotten past the aggregation
         * check when it was still OPERATIONAL and consequently have
         * IEEE80211_TX_CTL_AMPDU set. In that case, this code might
         * call into the driver at the same time or even before the
         * TX paths calls into it, which could confuse the driver.
         *
         * Wait for all currently running TX paths to finish before
         * telling the driver. New packets will not go through since
         * the aggregation session is no longer OPERATIONAL.
         */
        if (!local->in_reconfig)
                synchronize_net();

        tid_tx->stop_initiator = reason == AGG_STOP_PEER_REQUEST ?
                                        WLAN_BACK_RECIPIENT :
                                        WLAN_BACK_INITIATOR;
        tid_tx->tx_stop = reason == AGG_STOP_LOCAL_REQUEST;

        ret = drv_ampdu_action(local, sta->sdata, &params);

        /* HW shall not deny going back to legacy */
        if (WARN_ON(ret)) {
                /*
                 * We may have pending packets get stuck in this case...
                 * Not bothering with a workaround for now.
                 */
        }

        /*
         * In the case of AGG_STOP_DESTROY_STA, the driver won't
         * necessarily call ieee80211_stop_tx_ba_cb(), so this may
         * seem like we can leave the tid_tx data pending forever.
         * This is true, in a way, but "forever" is only until the
         * station struct is actually destroyed. In the meantime,
         * leaving it around ensures that we don't transmit packets
         * to the driver on this TID which might confuse it.
         */

        return 0;
}

/*
 * After sending add Block Ack request we activated a timer until
 * add Block Ack response will arrive from the recipient.
 * If this timer expires sta_addba_resp_timer_expired will be executed.
 */
static void sta_addba_resp_timer_expired(struct timer_list *t)
{
        struct tid_ampdu_tx *tid_tx = from_timer(tid_tx, t, addba_resp_timer);
        struct sta_info *sta = tid_tx->sta;
        u8 tid = tid_tx->tid;

        /* check if the TID waits for addBA response */
        if (test_bit(HT_AGG_STATE_RESPONSE_RECEIVED, &tid_tx->state)) {
                ht_dbg(sta->sdata,
                       "timer expired on %pM tid %d not expecting addBA response\n",
                       sta->sta.addr, tid);
                return;
        }

        ht_dbg(sta->sdata, "addBA response timer expired on %pM tid %d\n",
               sta->sta.addr, tid);

        ieee80211_stop_tx_ba_session(&sta->sta, tid);
}

static void ieee80211_send_addba_with_timeout(struct sta_info *sta,
                                              struct tid_ampdu_tx *tid_tx)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sta->local;
        u8 tid = tid_tx->tid;
        u16 buf_size;

        if (WARN_ON_ONCE(test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state) ||
                         test_bit(HT_AGG_STATE_WANT_STOP, &tid_tx->state)))
                return;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        /* activate the timer for the recipient's addBA response */
        mod_timer(&tid_tx->addba_resp_timer, jiffies + ADDBA_RESP_INTERVAL);
        ht_dbg(sdata, "activated addBA response timer on %pM tid %d\n",
               sta->sta.addr, tid);

        spin_lock_bh(&sta->lock);
        sta->ampdu_mlme.last_addba_req_time[tid] = jiffies;
        sta->ampdu_mlme.addba_req_num[tid]++;
        spin_unlock_bh(&sta->lock);

        if (sta->sta.deflink.eht_cap.has_eht) {
                buf_size = local->hw.max_tx_aggregation_subframes;
        } else if (sta->sta.deflink.he_cap.has_he) {
                buf_size = min_t(u16, local->hw.max_tx_aggregation_subframes,
                                 IEEE80211_MAX_AMPDU_BUF_HE);
        } else {
                /*
                 * We really should use what the driver told us it will
                 * transmit as the maximum, but certain APs (e.g. the
                 * LinkSys WRT120N with FW v1.0.07 build 002 Jun 18 2012)
                 * will crash when we use a lower number.
                 */
                buf_size = IEEE80211_MAX_AMPDU_BUF_HT;
        }

        /* send AddBA request */
        ieee80211_send_addba_request(sta, tid, tid_tx->dialog_token,
                                     tid_tx->ssn, buf_size, tid_tx->timeout);

        WARN_ON(test_and_set_bit(HT_AGG_STATE_SENT_ADDBA, &tid_tx->state));
}

void ieee80211_tx_ba_session_handle_start(struct sta_info *sta, int tid)
{
        struct tid_ampdu_tx *tid_tx;
        struct ieee80211_local *local = sta->local;
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_ampdu_params params = {
                .sta = &sta->sta,
                .action = IEEE80211_AMPDU_TX_START,
                .tid = tid,
                .buf_size = 0,
                .amsdu = false,
                .timeout = 0,
        };
        int ret;

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);

        /*
         * Start queuing up packets for this aggregation session.
         * We're going to release them once the driver is OK with
         * that.
         */
        clear_bit(HT_AGG_STATE_WANT_START, &tid_tx->state);

        /*
         * Make sure no packets are being processed. This ensures that
         * we have a valid starting sequence number and that in-flight
         * packets have been flushed out and no packets for this TID
         * will go into the driver during the ampdu_action call.
         */
        synchronize_net();

        params.ssn = sta->tid_seq[tid] >> 4;
        ret = drv_ampdu_action(local, sdata, &params);
        tid_tx->ssn = params.ssn;
        if (ret == IEEE80211_AMPDU_TX_START_DELAY_ADDBA) {
                return;
        } else if (ret == IEEE80211_AMPDU_TX_START_IMMEDIATE) {
                /*
                 * We didn't send the request yet, so don't need to check
                 * here if we already got a response, just mark as driver
                 * ready immediately.
                 */
                set_bit(HT_AGG_STATE_DRV_READY, &tid_tx->state);
        } else if (ret) {
                ht_dbg(sdata,
                       "BA request denied - HW unavailable for %pM tid %d\n",
                       sta->sta.addr, tid);
                spin_lock_bh(&sta->lock);
                ieee80211_agg_splice_packets(sdata, tid_tx, tid);
                ieee80211_assign_tid_tx(sta, tid, NULL);
                ieee80211_agg_splice_finish(sdata, tid);
                spin_unlock_bh(&sta->lock);

                ieee80211_agg_start_txq(sta, tid, false);

                kfree_rcu(tid_tx, rcu_head);
                return;
        }

        ieee80211_send_addba_with_timeout(sta, tid_tx);
}

void ieee80211_refresh_tx_agg_session_timer(struct ieee80211_sta *pubsta,
                                            u16 tid)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct tid_ampdu_tx *tid_tx;

        if (WARN_ON_ONCE(tid >= IEEE80211_NUM_TIDS))
                return;

        tid_tx = rcu_dereference(sta->ampdu_mlme.tid_tx[tid]);
        if (!tid_tx)
                return;

        tid_tx->last_tx = jiffies;
}
EXPORT_SYMBOL(ieee80211_refresh_tx_agg_session_timer);

/*
 * After accepting the AddBA Response we activated a timer,
 * resetting it after each frame that we send.
 */
static void sta_tx_agg_session_timer_expired(struct timer_list *t)
{
        struct tid_ampdu_tx *tid_tx = from_timer(tid_tx, t, session_timer);
        struct sta_info *sta = tid_tx->sta;
        u8 tid = tid_tx->tid;
        unsigned long timeout;

        if (test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state)) {
                return;
        }

        timeout = tid_tx->last_tx + TU_TO_JIFFIES(tid_tx->timeout);
        if (time_is_after_jiffies(timeout)) {
                mod_timer(&tid_tx->session_timer, timeout);
                return;
        }

        ht_dbg(sta->sdata, "tx session timer expired on %pM tid %d\n",
               sta->sta.addr, tid);

        ieee80211_stop_tx_ba_session(&sta->sta, tid);
}

int ieee80211_start_tx_ba_session(struct ieee80211_sta *pubsta, u16 tid,
                                  u16 timeout)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct tid_ampdu_tx *tid_tx;
        int ret = 0;

        trace_api_start_tx_ba_session(pubsta, tid);

        if (WARN(sta->reserved_tid == tid,
                 "Requested to start BA session on reserved tid=%d", tid))
                return -EINVAL;

        if (!pubsta->deflink.ht_cap.ht_supported &&
            !pubsta->deflink.vht_cap.vht_supported &&
            !pubsta->deflink.he_cap.has_he &&
            !pubsta->deflink.eht_cap.has_eht)
                return -EINVAL;

        if (WARN_ON_ONCE(!local->ops->ampdu_action))
                return -EINVAL;

        if ((tid >= IEEE80211_NUM_TIDS) ||
            !ieee80211_hw_check(&local->hw, AMPDU_AGGREGATION) ||
            ieee80211_hw_check(&local->hw, TX_AMPDU_SETUP_IN_HW))
                return -EINVAL;

        if (WARN_ON(tid >= IEEE80211_FIRST_TSPEC_TSID))
                return -EINVAL;

        ht_dbg(sdata, "Open BA session requested for %pM tid %u\n",
               pubsta->addr, tid);

        if (sdata->vif.type != NL80211_IFTYPE_STATION &&
            sdata->vif.type != NL80211_IFTYPE_MESH_POINT &&
            sdata->vif.type != NL80211_IFTYPE_AP_VLAN &&
            sdata->vif.type != NL80211_IFTYPE_AP &&
            sdata->vif.type != NL80211_IFTYPE_ADHOC)
                return -EINVAL;

        if (test_sta_flag(sta, WLAN_STA_BLOCK_BA)) {
                ht_dbg(sdata,
                       "BA sessions blocked - Denying BA session request %pM tid %d\n",
                       sta->sta.addr, tid);
                return -EINVAL;
        }

        if (test_sta_flag(sta, WLAN_STA_MFP) &&
            !test_sta_flag(sta, WLAN_STA_AUTHORIZED)) {
                ht_dbg(sdata,
                       "MFP STA not authorized - deny BA session request %pM tid %d\n",
                       sta->sta.addr, tid);
                return -EINVAL;
        }

        /*
         * 802.11n-2009 11.5.1.1: If the initiating STA is an HT STA, is a
         * member of an IBSS, and has no other existing Block Ack agreement
         * with the recipient STA, then the initiating STA shall transmit a
         * Probe Request frame to the recipient STA and shall not transmit an
         * ADDBA Request frame unless it receives a Probe Response frame
         * from the recipient within dot11ADDBAFailureTimeout.
         *
         * The probe request mechanism for ADDBA is currently not implemented,
         * but we only build up Block Ack session with HT STAs. This information
         * is set when we receive a bss info from a probe response or a beacon.
         */
        if (sta->sdata->vif.type == NL80211_IFTYPE_ADHOC &&
            !sta->sta.deflink.ht_cap.ht_supported) {
                ht_dbg(sdata,
                       "BA request denied - IBSS STA %pM does not advertise HT support\n",
                       pubsta->addr);
                return -EINVAL;
        }

        spin_lock_bh(&sta->lock);

        /* we have tried too many times, receiver does not want A-MPDU */
        if (sta->ampdu_mlme.addba_req_num[tid] > HT_AGG_MAX_RETRIES) {
                ret = -EBUSY;
                goto err_unlock_sta;
        }

        /*
         * if we have tried more than HT_AGG_BURST_RETRIES times we
         * will spread our requests in time to avoid stalling connection
         * for too long
         */
        if (sta->ampdu_mlme.addba_req_num[tid] > HT_AGG_BURST_RETRIES &&
            time_before(jiffies, sta->ampdu_mlme.last_addba_req_time[tid] +
                        HT_AGG_RETRIES_PERIOD)) {
                ht_dbg(sdata,
                       "BA request denied - %d failed requests on %pM tid %u\n",
                       sta->ampdu_mlme.addba_req_num[tid], sta->sta.addr, tid);
                ret = -EBUSY;
                goto err_unlock_sta;
        }

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);
        /* check if the TID is not in aggregation flow already */
        if (tid_tx || sta->ampdu_mlme.tid_start_tx[tid]) {
                ht_dbg(sdata,
                       "BA request denied - session is not idle on %pM tid %u\n",
                       sta->sta.addr, tid);
                ret = -EAGAIN;
                goto err_unlock_sta;
        }

        /* prepare A-MPDU MLME for Tx aggregation */
        tid_tx = kzalloc(sizeof(struct tid_ampdu_tx), GFP_ATOMIC);
        if (!tid_tx) {
                ret = -ENOMEM;
                goto err_unlock_sta;
        }

        skb_queue_head_init(&tid_tx->pending);
        __set_bit(HT_AGG_STATE_WANT_START, &tid_tx->state);

        tid_tx->timeout = timeout;
        tid_tx->sta = sta;
        tid_tx->tid = tid;

        /* response timer */
        timer_setup(&tid_tx->addba_resp_timer, sta_addba_resp_timer_expired, 0);

        /* tx timer */
        timer_setup(&tid_tx->session_timer,
                    sta_tx_agg_session_timer_expired, TIMER_DEFERRABLE);

        /* assign a dialog token */
        sta->ampdu_mlme.dialog_token_allocator++;
        tid_tx->dialog_token = sta->ampdu_mlme.dialog_token_allocator;

        /*
         * Finally, assign it to the start array; the work item will
         * collect it and move it to the normal array.
         */
        sta->ampdu_mlme.tid_start_tx[tid] = tid_tx;

        wiphy_work_queue(local->hw.wiphy, &sta->ampdu_mlme.work);

        /* this flow continues off the work */
 err_unlock_sta:
        spin_unlock_bh(&sta->lock);
        return ret;
}
EXPORT_SYMBOL(ieee80211_start_tx_ba_session);

static void ieee80211_agg_tx_operational(struct ieee80211_local *local,
                                         struct sta_info *sta, u16 tid)
{
        struct tid_ampdu_tx *tid_tx;
        struct ieee80211_ampdu_params params = {
                .sta = &sta->sta,
                .action = IEEE80211_AMPDU_TX_OPERATIONAL,
                .tid = tid,
                .timeout = 0,
                .ssn = 0,
        };

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);
        params.buf_size = tid_tx->buf_size;
        params.amsdu = tid_tx->amsdu;

        ht_dbg(sta->sdata, "Aggregation is on for %pM tid %d\n",
               sta->sta.addr, tid);

        drv_ampdu_action(local, sta->sdata, &params);

        /*
         * synchronize with TX path, while splicing the TX path
         * should block so it won't put more packets onto pending.
         */
        spin_lock_bh(&sta->lock);

        ieee80211_agg_splice_packets(sta->sdata, tid_tx, tid);
        /*
         * Now mark as operational. This will be visible
         * in the TX path, and lets it go lock-free in
         * the common case.
         */
        set_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state);
        ieee80211_agg_splice_finish(sta->sdata, tid);

        spin_unlock_bh(&sta->lock);

        ieee80211_agg_start_txq(sta, tid, true);
}

void ieee80211_start_tx_ba_cb(struct sta_info *sta, int tid,
                              struct tid_ampdu_tx *tid_tx)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        if (WARN_ON(test_and_set_bit(HT_AGG_STATE_DRV_READY, &tid_tx->state)))
                return;

        if (test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state) ||
            test_bit(HT_AGG_STATE_WANT_STOP, &tid_tx->state))
                return;

        if (!test_bit(HT_AGG_STATE_SENT_ADDBA, &tid_tx->state)) {
                ieee80211_send_addba_with_timeout(sta, tid_tx);
                /* RESPONSE_RECEIVED state would trigger the flow again */
                return;
        }

        if (test_bit(HT_AGG_STATE_RESPONSE_RECEIVED, &tid_tx->state))
                ieee80211_agg_tx_operational(local, sta, tid);
}

static struct tid_ampdu_tx *
ieee80211_lookup_tid_tx(struct ieee80211_sub_if_data *sdata,
                        const u8 *ra, u16 tid, struct sta_info **sta)
{
        struct tid_ampdu_tx *tid_tx;

        if (tid >= IEEE80211_NUM_TIDS) {
                ht_dbg(sdata, "Bad TID value: tid = %d (>= %d)\n",
                       tid, IEEE80211_NUM_TIDS);
                return NULL;
        }

        *sta = sta_info_get_bss(sdata, ra);
        if (!*sta) {
                ht_dbg(sdata, "Could not find station: %pM\n", ra);
                return NULL;
        }

        tid_tx = rcu_dereference((*sta)->ampdu_mlme.tid_tx[tid]);

        if (WARN_ON(!tid_tx))
                ht_dbg(sdata, "addBA was not requested!\n");

        return tid_tx;
}

void ieee80211_start_tx_ba_cb_irqsafe(struct ieee80211_vif *vif,
                                      const u8 *ra, u16 tid)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        struct tid_ampdu_tx *tid_tx;

        trace_api_start_tx_ba_cb(sdata, ra, tid);

        rcu_read_lock();
        tid_tx = ieee80211_lookup_tid_tx(sdata, ra, tid, &sta);
        if (!tid_tx)
                goto out;

        set_bit(HT_AGG_STATE_START_CB, &tid_tx->state);
        wiphy_work_queue(local->hw.wiphy, &sta->ampdu_mlme.work);
 out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(ieee80211_start_tx_ba_cb_irqsafe);

int ieee80211_stop_tx_ba_session(struct ieee80211_sta *pubsta, u16 tid)
{
        struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        struct ieee80211_local *local = sdata->local;
        struct tid_ampdu_tx *tid_tx;
        int ret = 0;

        trace_api_stop_tx_ba_session(pubsta, tid);

        if (!local->ops->ampdu_action)
                return -EINVAL;

        if (tid >= IEEE80211_NUM_TIDS)
                return -EINVAL;

        spin_lock_bh(&sta->lock);
        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);

        if (!tid_tx) {
                ret = -ENOENT;
                goto unlock;
        }

        WARN(sta->reserved_tid == tid,
             "Requested to stop BA session on reserved tid=%d", tid);

        if (test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state)) {
                /* already in progress stopping it */
                ret = 0;
                goto unlock;
        }

        set_bit(HT_AGG_STATE_WANT_STOP, &tid_tx->state);
        wiphy_work_queue(local->hw.wiphy, &sta->ampdu_mlme.work);

 unlock:
        spin_unlock_bh(&sta->lock);
        return ret;
}
EXPORT_SYMBOL(ieee80211_stop_tx_ba_session);

void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid,
                             struct tid_ampdu_tx *tid_tx)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        bool send_delba = false;
        bool start_txq = false;

        ht_dbg(sdata, "Stopping Tx BA session for %pM tid %d\n",
               sta->sta.addr, tid);

        spin_lock_bh(&sta->lock);

        if (!test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state)) {
                ht_dbg(sdata,
                       "unexpected callback to A-MPDU stop for %pM tid %d\n",
                       sta->sta.addr, tid);
                goto unlock_sta;
        }

        if (tid_tx->stop_initiator == WLAN_BACK_INITIATOR && tid_tx->tx_stop)
                send_delba = true;

        ieee80211_remove_tid_tx(sta, tid);
        start_txq = true;

 unlock_sta:
        spin_unlock_bh(&sta->lock);

        if (start_txq)
                ieee80211_agg_start_txq(sta, tid, false);

        if (send_delba)
                ieee80211_send_delba(sdata, sta->sta.addr, tid,
                        WLAN_BACK_INITIATOR, WLAN_REASON_QSTA_NOT_USE);
}

void ieee80211_stop_tx_ba_cb_irqsafe(struct ieee80211_vif *vif,
                                     const u8 *ra, u16 tid)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
        struct ieee80211_local *local = sdata->local;
        struct sta_info *sta;
        struct tid_ampdu_tx *tid_tx;

        trace_api_stop_tx_ba_cb(sdata, ra, tid);

        rcu_read_lock();
        tid_tx = ieee80211_lookup_tid_tx(sdata, ra, tid, &sta);
        if (!tid_tx)
                goto out;

        set_bit(HT_AGG_STATE_STOP_CB, &tid_tx->state);
        wiphy_work_queue(local->hw.wiphy, &sta->ampdu_mlme.work);
 out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(ieee80211_stop_tx_ba_cb_irqsafe);


void ieee80211_process_addba_resp(struct ieee80211_local *local,
                                  struct sta_info *sta,
                                  struct ieee80211_mgmt *mgmt,
                                  size_t len)
{
        struct tid_ampdu_tx *tid_tx;
        struct ieee80211_txq *txq;
        u16 capab, tid, buf_size;
        bool amsdu;

        lockdep_assert_wiphy(sta->local->hw.wiphy);

        capab = le16_to_cpu(mgmt->u.action.u.addba_resp.capab);
        amsdu = capab & IEEE80211_ADDBA_PARAM_AMSDU_MASK;
        tid = u16_get_bits(capab, IEEE80211_ADDBA_PARAM_TID_MASK);
        buf_size = u16_get_bits(capab, IEEE80211_ADDBA_PARAM_BUF_SIZE_MASK);

        ieee80211_retrieve_addba_ext_data(sta,
                                          mgmt->u.action.u.addba_resp.variable,
                                          len - offsetof(typeof(*mgmt),
                                                         u.action.u.addba_resp.variable),
                                          &buf_size);

        buf_size = min(buf_size, local->hw.max_tx_aggregation_subframes);

        txq = sta->sta.txq[tid];
        if (!amsdu && txq)
                set_bit(IEEE80211_TXQ_NO_AMSDU, &to_txq_info(txq)->flags);

        tid_tx = rcu_dereference_protected_tid_tx(sta, tid);
        if (!tid_tx)
                return;

        if (mgmt->u.action.u.addba_resp.dialog_token != tid_tx->dialog_token) {
                ht_dbg(sta->sdata, "wrong addBA response token, %pM tid %d\n",
                       sta->sta.addr, tid);
                return;
        }

        del_timer_sync(&tid_tx->addba_resp_timer);

        ht_dbg(sta->sdata, "switched off addBA timer for %pM tid %d\n",
               sta->sta.addr, tid);

        /*
         * addba_resp_timer may have fired before we got here, and
         * caused WANT_STOP to be set. If the stop then was already
         * processed further, STOPPING might be set.
         */
        if (test_bit(HT_AGG_STATE_WANT_STOP, &tid_tx->state) ||
            test_bit(HT_AGG_STATE_STOPPING, &tid_tx->state)) {
                ht_dbg(sta->sdata,
                       "got addBA resp for %pM tid %d but we already gave up\n",
                       sta->sta.addr, tid);
                return;
        }

        /*
         * IEEE 802.11-2007 7.3.1.14:
         * In an ADDBA Response frame, when the Status Code field
         * is set to 0, the Buffer Size subfield is set to a value
         * of at least 1.
         */
        if (le16_to_cpu(mgmt->u.action.u.addba_resp.status)
                        == WLAN_STATUS_SUCCESS && buf_size) {
                if (test_and_set_bit(HT_AGG_STATE_RESPONSE_RECEIVED,
                                     &tid_tx->state)) {
                        /* ignore duplicate response */
                        return;
                }

                tid_tx->buf_size = buf_size;
                tid_tx->amsdu = amsdu;

                if (test_bit(HT_AGG_STATE_DRV_READY, &tid_tx->state))
                        ieee80211_agg_tx_operational(local, sta, tid);

                sta->ampdu_mlme.addba_req_num[tid] = 0;

                tid_tx->timeout =
                        le16_to_cpu(mgmt->u.action.u.addba_resp.timeout);

                if (tid_tx->timeout) {
                        mod_timer(&tid_tx->session_timer,
                                  TU_TO_EXP_TIME(tid_tx->timeout));
                        tid_tx->last_tx = jiffies;
                }

        } else {
                __ieee80211_stop_tx_ba_session(sta, tid, AGG_STOP_DECLINED);
        }
}



































































































































































































 13848 







 3794 





































  419 



  570 




















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
/* SPDX-License-Identifier: GPL-2.0 */
/* rwsem.h: R/W semaphores, public interface
 *
 * Written by David Howells (dhowells@redhat.com).
 * Derived from asm-i386/semaphore.h
 */

#ifndef _LINUX_RWSEM_H
#define _LINUX_RWSEM_H

#include <linux/linkage.h>

#include <linux/types.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/atomic.h>
#include <linux/err.h>
#include <linux/cleanup.h>

#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define __RWSEM_DEP_MAP_INIT(lockname)                        \
        .dep_map = {                                        \
                .name = #lockname,                        \
                .wait_type_inner = LD_WAIT_SLEEP,        \
        },
#else
# define __RWSEM_DEP_MAP_INIT(lockname)
#endif

#ifndef CONFIG_PREEMPT_RT

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
#include <linux/osq_lock.h>
#endif

/*
 * For an uncontended rwsem, count and owner are the only fields a task
 * needs to touch when acquiring the rwsem. So they are put next to each
 * other to increase the chance that they will share the same cacheline.
 *
 * In a contended rwsem, the owner is likely the most frequently accessed
 * field in the structure as the optimistic waiter that holds the osq lock
 * will spin on owner. For an embedded rwsem, other hot fields in the
 * containing structure should be moved further away from the rwsem to
 * reduce the chance that they will share the same cacheline causing
 * cacheline bouncing problem.
 */
struct rw_semaphore {
        atomic_long_t count;
        /*
         * Write owner or one of the read owners as well flags regarding
         * the current state of the rwsem. Can be used as a speculative
         * check to see if the write owner is running on the cpu.
         */
        atomic_long_t owner;
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
        struct optimistic_spin_queue osq; /* spinner MCS lock */
#endif
        raw_spinlock_t wait_lock;
        struct list_head wait_list;
#ifdef CONFIG_DEBUG_RWSEMS
        void *magic;
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map        dep_map;
#endif
};

#define RWSEM_UNLOCKED_VALUE                0UL
#define RWSEM_WRITER_LOCKED                (1UL << 0)
#define __RWSEM_COUNT_INIT(name)        .count = ATOMIC_LONG_INIT(RWSEM_UNLOCKED_VALUE)

static inline int rwsem_is_locked(struct rw_semaphore *sem)
{
        return atomic_long_read(&sem->count) != RWSEM_UNLOCKED_VALUE;
}

static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem)
{
        WARN_ON(atomic_long_read(&sem->count) == RWSEM_UNLOCKED_VALUE);
}

static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem)
{
        WARN_ON(!(atomic_long_read(&sem->count) & RWSEM_WRITER_LOCKED));
}

/* Common initializer macros and functions */

#ifdef CONFIG_DEBUG_RWSEMS
# define __RWSEM_DEBUG_INIT(lockname) .magic = &lockname,
#else
# define __RWSEM_DEBUG_INIT(lockname)
#endif

#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
#define __RWSEM_OPT_INIT(lockname) .osq = OSQ_LOCK_UNLOCKED,
#else
#define __RWSEM_OPT_INIT(lockname)
#endif

#define __RWSEM_INITIALIZER(name)                                \
        { __RWSEM_COUNT_INIT(name),                                \
          .owner = ATOMIC_LONG_INIT(0),                                \
          __RWSEM_OPT_INIT(name)                                \
          .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
          .wait_list = LIST_HEAD_INIT((name).wait_list),        \
          __RWSEM_DEBUG_INIT(name)                                \
          __RWSEM_DEP_MAP_INIT(name) }

#define DECLARE_RWSEM(name) \
        struct rw_semaphore name = __RWSEM_INITIALIZER(name)

extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
                         struct lock_class_key *key);

#define init_rwsem(sem)                                                \
do {                                                                \
        static struct lock_class_key __key;                        \
                                                                \
        __init_rwsem((sem), #sem, &__key);                        \
} while (0)

/*
 * This is the same regardless of which rwsem implementation that is being used.
 * It is just a heuristic meant to be called by somebody already holding the
 * rwsem to see if somebody from an incompatible type is wanting access to the
 * lock.
 */
static inline int rwsem_is_contended(struct rw_semaphore *sem)
{
        return !list_empty(&sem->wait_list);
}

#else /* !CONFIG_PREEMPT_RT */

#include <linux/rwbase_rt.h>

struct rw_semaphore {
        struct rwbase_rt        rwbase;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map        dep_map;
#endif
};

#define __RWSEM_INITIALIZER(name)                                \
        {                                                        \
                .rwbase = __RWBASE_INITIALIZER(name),                \
                __RWSEM_DEP_MAP_INIT(name)                        \
        }

#define DECLARE_RWSEM(lockname) \
        struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname)

extern void  __init_rwsem(struct rw_semaphore *rwsem, const char *name,
                          struct lock_class_key *key);

#define init_rwsem(sem)                                                \
do {                                                                \
        static struct lock_class_key __key;                        \
                                                                \
        __init_rwsem((sem), #sem, &__key);                        \
} while (0)

static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem)
{
        return rw_base_is_locked(&sem->rwbase);
}

static __always_inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem)
{
        WARN_ON(!rwsem_is_locked(sem));
}

static __always_inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem)
{
        WARN_ON(!rw_base_is_write_locked(&sem->rwbase));
}

static __always_inline int rwsem_is_contended(struct rw_semaphore *sem)
{
        return rw_base_is_contended(&sem->rwbase);
}

#endif /* CONFIG_PREEMPT_RT */

/*
 * The functions below are the same for all rwsem implementations including
 * the RT specific variant.
 */

static inline void rwsem_assert_held(const struct rw_semaphore *sem)
{
        if (IS_ENABLED(CONFIG_LOCKDEP))
                lockdep_assert_held(sem);
        else
                rwsem_assert_held_nolockdep(sem);
}

static inline void rwsem_assert_held_write(const struct rw_semaphore *sem)
{
        if (IS_ENABLED(CONFIG_LOCKDEP))
                lockdep_assert_held_write(sem);
        else
                rwsem_assert_held_write_nolockdep(sem);
}

/*
 * lock for reading
 */
extern void down_read(struct rw_semaphore *sem);
extern int __must_check down_read_interruptible(struct rw_semaphore *sem);
extern int __must_check down_read_killable(struct rw_semaphore *sem);

/*
 * trylock for reading -- returns 1 if successful, 0 if contention
 */
extern int down_read_trylock(struct rw_semaphore *sem);

/*
 * lock for writing
 */
extern void down_write(struct rw_semaphore *sem);
extern int __must_check down_write_killable(struct rw_semaphore *sem);

/*
 * trylock for writing -- returns 1 if successful, 0 if contention
 */
extern int down_write_trylock(struct rw_semaphore *sem);

/*
 * release a read lock
 */
extern void up_read(struct rw_semaphore *sem);

/*
 * release a write lock
 */
extern void up_write(struct rw_semaphore *sem);

DEFINE_GUARD(rwsem_read, struct rw_semaphore *, down_read(_T), up_read(_T))
DEFINE_GUARD_COND(rwsem_read, _try, down_read_trylock(_T))
DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T) == 0)

DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T))
DEFINE_GUARD_COND(rwsem_write, _try, down_write_trylock(_T))

/*
 * downgrade write lock to read lock
 */
extern void downgrade_write(struct rw_semaphore *sem);

#ifdef CONFIG_DEBUG_LOCK_ALLOC
/*
 * nested locking. NOTE: rwsems are not allowed to recurse
 * (which occurs if the same task tries to acquire the same
 * lock instance multiple times), but multiple locks of the
 * same lock class might be taken, if the order of the locks
 * is always the same. This ordering rule can be expressed
 * to lockdep via the _nested() APIs, but enumerating the
 * subclasses that are used. (If the nesting relationship is
 * static then another method for expressing nested locking is
 * the explicit definition of lock class keys and the use of
 * lockdep_set_class() at lock initialization time.
 * See Documentation/locking/lockdep-design.rst for more details.)
 */
extern void down_read_nested(struct rw_semaphore *sem, int subclass);
extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass);
extern void down_write_nested(struct rw_semaphore *sem, int subclass);
extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass);
extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock);

# define down_write_nest_lock(sem, nest_lock)                        \
do {                                                                \
        typecheck(struct lockdep_map *, &(nest_lock)->dep_map);        \
        _down_write_nest_lock(sem, &(nest_lock)->dep_map);        \
} while (0)

/*
 * Take/release a lock when not the owner will release it.
 *
 * [ This API should be avoided as much as possible - the
 *   proper abstraction for this case is completions. ]
 */
extern void down_read_non_owner(struct rw_semaphore *sem);
extern void up_read_non_owner(struct rw_semaphore *sem);
#else
# define down_read_nested(sem, subclass)                down_read(sem)
# define down_read_killable_nested(sem, subclass)        down_read_killable(sem)
# define down_write_nest_lock(sem, nest_lock)        down_write(sem)
# define down_write_nested(sem, subclass)        down_write(sem)
# define down_write_killable_nested(sem, subclass)        down_write_killable(sem)
# define down_read_non_owner(sem)                down_read(sem)
# define up_read_non_owner(sem)                        up_read(sem)
#endif

#endif /* _LINUX_RWSEM_H */















































































   23 










    7 


























   92 
   93 






   93 







   72 




    3 

   63 



    1 


    3 









    2 





    7 
    2 




    1 





    6 



    3 





    7 



   11 



   17 








   11 




   11 











   34 





























    2 






    3 







    6 































   36 
   36 




   36 



   36 
   35 

    2 






   34 



   34 





    9 









   25 

































    1 







    1 

    1 





    1 
































































































    1 











    1 






    1 


















































































































































   29 
   29 














   29 
   28 

    2 




   27 

    3 

    3 







    1 


    1 





   17 







    1 










   16 














    1 



























    2 







    1 


    2 
    1 













    3 














































    1 































































    4 



    2 














    1 



    2 




























































    2 







    2 




    2 




    1 





















    2 


    2 


    7 










   63 










   62 
   58 
   58 





   59 




   34 
   24 































   10 






























    8 









    2 





    9 




    9 











   10 



   10 


   11 


















   70 
   49 
   21 




   16 
    5 

   20 
   49 



   29 

   29 





    4 






    2 
    2 
    3 
    1 
    3 
    1 
    3